diff --git a/.github/workflows/build-cpu.yml b/.github/workflows/build-cpu.yml index 8f62e1a177..30b07ce788 100644 --- a/.github/workflows/build-cpu.yml +++ b/.github/workflows/build-cpu.yml @@ -121,7 +121,7 @@ jobs: env: OPENBLAS_VERSION: 0.3.23 SDE_VERSION: 9.33.0-2024-01-07 - VULKAN_VERSION: 1.4.313.2 + VULKAN_VERSION: 1.4.357.0 strategy: matrix: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 778574cf9f..3460485af3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -759,7 +759,7 @@ jobs: env: OPENBLAS_VERSION: 0.3.23 - VULKAN_VERSION: 1.4.313.2 + VULKAN_VERSION: 1.4.357.0 strategy: matrix: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 91fa381dd0..0031334788 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -73,6 +73,7 @@ For more info, please refer to the [AGENTS.md](AGENTS.md) file. - When merging a PR, make sure you have a good understanding of the changes - If a PR does not warrant a new release, add `[no release]` in the squashed commit to spare CI resources - Be mindful of maintenance: most of the work going into a feature happens after the PR is merged. If the PR author is not committed to contribute long-term, someone else needs to take responsibility (you) +- Add the ["merge ready"](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr+is%3Aopen+draft%3Ano+sort%3Aupdated-desc+label%3A%22merge+ready%22+) label to a PR to indicate when a PR can be fast-merged without waiting for 2 independent reviews. [(more info)](https://github.com/ggml-org/llama.cpp/pull/26178) Maintainers reserve the right to decline review or close pull requests for any reason, without any questions, particularly under any of the following conditions: - The proposed change is already mentioned in the roadmap or an existing issue, and it has been assigned to someone. diff --git a/README.md b/README.md index ee70581e37..57436327ec 100644 --- a/README.md +++ b/README.md @@ -2,65 +2,56 @@ ![llama](https://raw.githubusercontent.com/ggml-org/llama.brand/refs/heads/master/cover/llama-cpp/cover-llama-cpp-dark.svg) +
+ +LLM inference in C/C++ + [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT) [![Release](https://img.shields.io/github/v/release/ggml-org/llama.cpp)](https://github.com/ggml-org/llama.cpp/releases) [![Server](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml/badge.svg)](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml) [![Docker](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml/badge.svg)](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml) [![Winget](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml/badge.svg)](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml) -[Manifesto](https://github.com/ggml-org/llama.cpp/discussions/205) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Abartowski1182%20OR%20author%3Ahipudding%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3A0cc4m%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) +[manifesto](https://github.com/ggml-org/llama.cpp/discussions/205) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/issues?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Abartowski1182%20OR%20author%3Ahipudding%20OR%20author%3AServeurpersoCom%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3A0cc4m%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3Aallozaur%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Alhez%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming)%20sort%3Aupdated-desc) / [dev branches](https://github.com/ggml-org/llama.cpp-dev/blob/master/README-features.md) / [compile times](https://github.com/ggml-org/llama.cpp-dev/blob/master/README-compile-times.md) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291) -LLM inference in C/C++ - -## Recent API changes - -- [Changelog for `libllama` API](https://github.com/ggml-org/llama.cpp/issues/9289) -- [Changelog for `llama-server` REST API](https://github.com/ggml-org/llama.cpp/issues/9291) - -## Hot topics - -- **Hugging Face cache migration: models downloaded with `-hf` are now stored in the standard Hugging Face cache directory, enabling sharing with other HF tools.** -- **[guide : using the new WebUI of llama.cpp](https://github.com/ggml-org/llama.cpp/discussions/16938)** -- [guide : running gpt-oss with llama.cpp](https://github.com/ggml-org/llama.cpp/discussions/15396) -- [[FEEDBACK] Better packaging for llama.cpp to support downstream consumers 🤗](https://github.com/ggml-org/llama.cpp/discussions/15313) -- Support for the `gpt-oss` model with native MXFP4 format has been added | [PR](https://github.com/ggml-org/llama.cpp/pull/15091) | [Collaboration with NVIDIA](https://blogs.nvidia.com/blog/rtx-ai-garage-openai-oss) | [Comment](https://github.com/ggml-org/llama.cpp/discussions/15095) -- Multimodal support arrived in `llama-server`: [#12898](https://github.com/ggml-org/llama.cpp/pull/12898) | [documentation](./docs/multimodal.md) -- VS Code extension for FIM completions: https://github.com/ggml-org/llama.vscode -- Vim/Neovim plugin for FIM completions: https://github.com/ggml-org/llama.vim -- Hugging Face Inference Endpoints now support GGUF out of the box! https://github.com/ggml-org/llama.cpp/discussions/9669 -- Hugging Face GGUF editor: [discussion](https://github.com/ggml-org/llama.cpp/discussions/9268) | [tool](https://huggingface.co/spaces/CISCai/gguf-editor) -- WebGPU support is now available in the browser, see a blog/demo introducing it [here](https://reeselevine.github.io/llamas-on-the-web/). - ----- +
## Quick start -Getting started with llama.cpp is straightforward. Here are several ways to install it on your machine: +A few options to get `llama.cpp` installed on your machine: -- Install `llama.cpp` using [brew, nix, winget, or conda-forge](docs/install.md) +- Visit https://llama.app and follow the instructions - Run with Docker - see our [Docker documentation](docs/docker.md) - Download pre-built binaries from the [releases page](https://github.com/ggml-org/llama.cpp/releases) - Build from source by cloning this repository - check out [our build guide](docs/build.md) -Once installed, you'll need a model to work with. Head to the [Obtaining and quantizing models](#obtaining-and-quantizing-models) section to learn more. - -Example command: +Once installed: ```sh -# Use a local model file -llama-cli -m my_model.gguf - -# Or download and run a model directly from Hugging Face -llama-cli -hf ggml-org/gemma-3-1b-it-GGUF +# Download and run a model directly from Hugging Face +llama cli -hf ggml-org/Qwen3.5-0.8B-GGUF # Launch OpenAI-compatible API server -llama-server -hf ggml-org/gemma-3-1b-it-GGUF +llama serve -hf ggml-org/Qwen3.5-0.8B-GGUF ``` + + + + + +
+ VLM session with `llama cli` + VLM session with llama cli + + Built-in web UI against `llama serve` running Qwen 3.6 + Built-in web UI against llama serve +
+ ## Description -The main goal of `llama.cpp` is to enable LLM inference with minimal setup and state-of-the-art performance on a wide -range of hardware - locally and in the cloud. +The main goal of `llama.cpp` is to enable LLM (and VLM) inference with minimal setup and state-of-the-art performance on +a wide range of hardware - locally and in the cloud. - Plain C/C++ implementation without any dependencies - Apple silicon is a first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks @@ -71,467 +62,40 @@ range of hardware - locally and in the cloud. - Vulkan and SYCL backend support - CPU+GPU hybrid inference to partially accelerate models larger than the total VRAM capacity -The `llama.cpp` project is the main playground for developing new features for the [ggml](https://github.com/ggml-org/ggml) library. - -
-Models - -Typically finetunes of the base models below are supported as well. - -Instructions for adding support for new models: [HOWTO-add-model.md](docs/development/HOWTO-add-model.md) - -#### Text-only - -- [X] LLaMA 🦙 -- [x] LLaMA 2 🦙🦙 -- [x] LLaMA 3 🦙🦙🦙 -- [X] [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.1) -- [x] [Mixtral MoE](https://huggingface.co/models?search=mistral-ai/Mixtral) -- [x] [DBRX](https://huggingface.co/databricks/dbrx-instruct) -- [x] [Jamba](https://huggingface.co/ai21labs) -- [X] [Falcon](https://huggingface.co/models?search=tiiuae/falcon) -- [X] [Chinese LLaMA / Alpaca](https://github.com/ymcui/Chinese-LLaMA-Alpaca) and [Chinese LLaMA-2 / Alpaca-2](https://github.com/ymcui/Chinese-LLaMA-Alpaca-2) -- [X] [Vigogne (French)](https://github.com/bofenghuang/vigogne) -- [X] [BERT](https://github.com/ggml-org/llama.cpp/pull/5423) -- [X] [Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/) -- [X] [Baichuan 1 & 2](https://huggingface.co/models?search=baichuan-inc/Baichuan) + [derivations](https://huggingface.co/hiyouga/baichuan-7b-sft) -- [X] [Aquila 1 & 2](https://huggingface.co/models?search=BAAI/Aquila) -- [X] [Starcoder models](https://github.com/ggml-org/llama.cpp/pull/3187) -- [X] [Refact](https://huggingface.co/smallcloudai/Refact-1_6B-fim) -- [X] [MPT](https://github.com/ggml-org/llama.cpp/pull/3417) -- [X] [Bloom](https://github.com/ggml-org/llama.cpp/pull/3553) -- [x] [Yi models](https://huggingface.co/models?search=01-ai/Yi) -- [X] [StableLM models](https://huggingface.co/stabilityai) -- [x] [Deepseek models](https://huggingface.co/models?search=deepseek-ai/deepseek) -- [x] [Qwen models](https://huggingface.co/models?search=Qwen/Qwen) -- [x] [PLaMo-13B](https://github.com/ggml-org/llama.cpp/pull/3557) -- [x] [Phi models](https://huggingface.co/models?search=microsoft/phi) -- [x] [PhiMoE](https://github.com/ggml-org/llama.cpp/pull/11003) -- [x] [GPT-2](https://huggingface.co/gpt2) -- [x] [Orion 14B](https://github.com/ggml-org/llama.cpp/pull/5118) -- [x] [InternLM2](https://huggingface.co/models?search=internlm2) -- [x] [CodeShell](https://github.com/WisdomShell/codeshell) -- [x] [Gemma](https://ai.google.dev/gemma) -- [x] [Mamba](https://github.com/state-spaces/mamba) -- [x] [Grok-1](https://huggingface.co/keyfan/grok-1-hf) -- [x] [Xverse](https://huggingface.co/models?search=xverse) -- [x] [Command-R models](https://huggingface.co/models?search=CohereForAI/c4ai-command-r) -- [x] [SEA-LION](https://huggingface.co/models?search=sea-lion) -- [x] [GritLM-7B](https://huggingface.co/GritLM/GritLM-7B) + [GritLM-8x7B](https://huggingface.co/GritLM/GritLM-8x7B) -- [x] [OLMo](https://allenai.org/olmo) -- [x] [OLMo 2](https://allenai.org/olmo) -- [x] [OLMoE](https://huggingface.co/allenai/OLMoE-1B-7B-0924) -- [x] [Granite models](https://huggingface.co/collections/ibm-granite/granite-code-models-6624c5cec322e4c148c8b330) -- [x] [GPT-NeoX](https://github.com/EleutherAI/gpt-neox) + [Pythia](https://github.com/EleutherAI/pythia) -- [x] [Snowflake-Arctic MoE](https://huggingface.co/collections/Snowflake/arctic-66290090abe542894a5ac520) -- [x] [Smaug](https://huggingface.co/models?search=Smaug) -- [x] [Poro 34B](https://huggingface.co/LumiOpen/Poro-34B) -- [x] [Bitnet b1.58 models](https://huggingface.co/1bitLLM) -- [x] [Flan T5](https://huggingface.co/models?search=flan-t5) -- [x] [Open Elm models](https://huggingface.co/collections/apple/openelm-instruct-models-6619ad295d7ae9f868b759ca) -- [x] [ChatGLM3-6b](https://huggingface.co/THUDM/chatglm3-6b) + [ChatGLM4-9b](https://huggingface.co/THUDM/glm-4-9b) + [GLMEdge-1.5b](https://huggingface.co/THUDM/glm-edge-1.5b-chat) + [GLMEdge-4b](https://huggingface.co/THUDM/glm-edge-4b-chat) -- [x] [GLM-4-0414](https://huggingface.co/collections/THUDM/glm-4-0414-67f3cbcb34dd9d252707cb2e) -- [x] [SmolLM](https://huggingface.co/collections/HuggingFaceTB/smollm-6695016cad7167254ce15966) -- [x] [EXAONE-3.0-7.8B-Instruct](https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct) -- [x] [FalconMamba Models](https://huggingface.co/collections/tiiuae/falconmamba-7b-66b9a580324dd1598b0f6d4a) -- [x] [Jais](https://huggingface.co/inceptionai/jais-13b-chat) -- [x] [Bielik-11B-v2.3](https://huggingface.co/collections/speakleash/bielik-11b-v23-66ee813238d9b526a072408a) -- [x] [RWKV-7](https://huggingface.co/collections/shoumenchougou/rwkv7-gxx-gguf) -- [x] [RWKV-6](https://github.com/BlinkDL/RWKV-LM) -- [x] [QRWKV-6](https://huggingface.co/recursal/QRWKV6-32B-Instruct-Preview-v0.1) -- [x] [GigaChat-20B-A3B](https://huggingface.co/ai-sage/GigaChat-20B-A3B-instruct) -- [X] [Trillion-7B-preview](https://huggingface.co/trillionlabs/Trillion-7B-preview) -- [x] [Ling models](https://huggingface.co/collections/inclusionAI/ling-67c51c85b34a7ea0aba94c32) -- [x] [Liquid LFM2 models](https://huggingface.co/collections/LiquidAI/lfm2) -- [x] [Liquid LFM2.5 models](https://huggingface.co/collections/LiquidAI/lfm25) -- [x] [Liquid Nanos](https://huggingface.co/collections/LiquidAI/liquid-nanos) -- [x] [Hunyuan models](https://huggingface.co/collections/tencent/hunyuan-dense-model-6890632cda26b19119c9c5e7) -- [x] [BailingMoeV2 (Ring/Ling 2.0) models](https://huggingface.co/collections/inclusionAI/ling-v2-68bf1dd2fc34c306c1fa6f86) -- [x] [Mellum models](https://huggingface.co/JetBrains/models?search=mellum) - -#### Multimodal - -- [x] [LLaVA 1.5 models](https://huggingface.co/collections/liuhaotian/llava-15-653aac15d994e992e2677a7e), [LLaVA 1.6 models](https://huggingface.co/collections/liuhaotian/llava-16-65b9e40155f60fd046a5ccf2) -- [x] [BakLLaVA](https://huggingface.co/models?search=SkunkworksAI/Bakllava) -- [x] [Obsidian](https://huggingface.co/NousResearch/Obsidian-3B-V0.5) -- [x] [ShareGPT4V](https://huggingface.co/models?search=Lin-Chen/ShareGPT4V) -- [x] [MobileVLM 1.7B/3B models](https://huggingface.co/models?search=mobileVLM) -- [x] [Yi-VL](https://huggingface.co/models?search=Yi-VL) -- [x] [Mini CPM](https://huggingface.co/models?search=MiniCPM) -- [x] [Moondream](https://huggingface.co/vikhyatk/moondream2) -- [x] [Bunny](https://github.com/BAAI-DCAI/Bunny) -- [x] [GLM-EDGE](https://huggingface.co/models?search=glm-edge) -- [x] [Qwen2-VL](https://huggingface.co/collections/Qwen/qwen2-vl-66cee7455501d7126940800d) -- [x] [LFM2-VL](https://huggingface.co/collections/LiquidAI/lfm2-vl-68963bbc84a610f7638d5ffa) - -
- -
-Bindings - -- Python: [ddh0/easy-llama](https://github.com/ddh0/easy-llama) -- Python: [abetlen/llama-cpp-python](https://github.com/abetlen/llama-cpp-python) -- Go: [go-skynet/go-llama.cpp](https://github.com/go-skynet/go-llama.cpp) -- Node.js: [withcatai/node-llama-cpp](https://github.com/withcatai/node-llama-cpp) -- JS/TS (llama.cpp server client): [lgrammel/modelfusion](https://modelfusion.dev/integration/model-provider/llamacpp) -- JS/TS (Programmable Prompt Engine CLI): [offline-ai/cli](https://github.com/offline-ai/cli) -- JavaScript/Wasm (works in browser): [tangledgroup/llama-cpp-wasm](https://github.com/tangledgroup/llama-cpp-wasm) -- Typescript/Wasm (nicer API, available on npm): [ngxson/wllama](https://github.com/ngxson/wllama) -- Ruby: [yoshoku/llama_cpp.rb](https://github.com/yoshoku/llama_cpp.rb) -- Ruby: [docusealco/rllama](https://github.com/docusealco/rllama) -- Rust (more features): [edgenai/llama_cpp-rs](https://github.com/edgenai/llama_cpp-rs) -- Rust (nicer API): [mdrokz/rust-llama.cpp](https://github.com/mdrokz/rust-llama.cpp) -- Rust (more direct bindings): [utilityai/llama-cpp-rs](https://github.com/utilityai/llama-cpp-rs) -- Rust (automated build from crates.io): [ShelbyJenkins/llm_client](https://github.com/ShelbyJenkins/llm_client) -- C#/.NET: [SciSharp/LLamaSharp](https://github.com/SciSharp/LLamaSharp) -- C#/VB.NET (more features - community license): [LM-Kit.NET](https://docs.lm-kit.com/lm-kit-net/index.html) -- Scala 3: [donderom/llm4s](https://github.com/donderom/llm4s) -- Clojure: [phronmophobic/llama.clj](https://github.com/phronmophobic/llama.clj) -- React Native: [mybigday/llama.rn](https://github.com/mybigday/llama.rn) -- Java: [kherud/java-llama.cpp](https://github.com/kherud/java-llama.cpp) -- Java: [QuasarByte/llama-cpp-jna](https://github.com/QuasarByte/llama-cpp-jna) -- Zig: [deins/llama.cpp.zig](https://github.com/Deins/llama.cpp.zig) -- Flutter/Dart: [netdur/llama_cpp_dart](https://github.com/netdur/llama_cpp_dart) -- Flutter: [xuegao-tzx/Fllama](https://github.com/xuegao-tzx/Fllama) -- PHP (API bindings and features built on top of llama.cpp): [distantmagic/resonance](https://github.com/distantmagic/resonance) [(more info)](https://github.com/ggml-org/llama.cpp/pull/6326) -- Guile Scheme: [guile_llama_cpp](https://savannah.nongnu.org/projects/guile-llama-cpp) -- Swift [srgtuszy/llama-cpp-swift](https://github.com/srgtuszy/llama-cpp-swift) -- Swift [ShenghaiWang/SwiftLlama](https://github.com/ShenghaiWang/SwiftLlama) -- Delphi [Embarcadero/llama-cpp-delphi](https://github.com/Embarcadero/llama-cpp-delphi) -- Go (no CGo needed): [hybridgroup/yzma](https://github.com/hybridgroup/yzma) -- Android: [llama.android](/examples/llama.android) - -
- -
-UIs - -*(to have a project listed here, it should clearly state that it depends on `llama.cpp`)* - -- [AI Sublime Text plugin](https://github.com/yaroslavyaroslav/OpenAI-sublime-text) (MIT) -- [BonzAI App](https://apps.apple.com/us/app/bonzai-your-local-ai-agent/id6752847988) (proprietary) -- [cztomsik/ava](https://github.com/cztomsik/ava) (MIT) -- [Dot](https://github.com/alexpinel/Dot) (GPL) -- [eva](https://github.com/ylsdamxssjxxdd/eva) (MIT) -- [iohub/collama](https://github.com/iohub/coLLaMA) (Apache-2.0) -- [janhq/jan](https://github.com/janhq/jan) (AGPL) -- [johnbean393/Sidekick](https://github.com/johnbean393/Sidekick) (MIT) -- [KanTV](https://github.com/zhouwg/kantv?tab=readme-ov-file) (Apache-2.0) -- [KodiBot](https://github.com/firatkiral/kodibot) (GPL) -- [llama.vim](https://github.com/ggml-org/llama.vim) (MIT) -- [LARS](https://github.com/abgulati/LARS) (AGPL) -- [Llama Assistant](https://github.com/vietanhdev/llama-assistant) (GPL) -- [LlamaLib](https://github.com/undreamai/LlamaLib) (Apache-2.0) -- [LLMFarm](https://github.com/guinmoon/LLMFarm?tab=readme-ov-file) (MIT) -- [LLMUnity](https://github.com/undreamai/LLMUnity) (MIT) -- [LMStudio](https://lmstudio.ai/) (proprietary) -- [LocalAI](https://github.com/mudler/LocalAI) (MIT) -- [LostRuins/koboldcpp](https://github.com/LostRuins/koboldcpp) (AGPL) -- [MindMac](https://mindmac.app) (proprietary) -- [MindWorkAI/AI-Studio](https://github.com/MindWorkAI/AI-Studio) (FSL-1.1-MIT) -- [Mobile-Artificial-Intelligence/maid](https://github.com/Mobile-Artificial-Intelligence/maid) (MIT) -- [Mozilla-Ocho/llamafile](https://github.com/Mozilla-Ocho/llamafile) (Apache-2.0) -- [nat/openplayground](https://github.com/nat/openplayground) (MIT) -- [nomic-ai/gpt4all](https://github.com/nomic-ai/gpt4all) (MIT) -- [ollama/ollama](https://github.com/ollama/ollama) (MIT) -- [oobabooga/text-generation-webui](https://github.com/oobabooga/text-generation-webui) (AGPL) -- [PocketPal AI](https://github.com/a-ghorbani/pocketpal-ai) (MIT) -- [psugihara/FreeChat](https://github.com/psugihara/FreeChat) (MIT) -- [ptsochantaris/emeltal](https://github.com/ptsochantaris/emeltal) (MIT) -- [pythops/tenere](https://github.com/pythops/tenere) (AGPL) -- [ramalama](https://github.com/containers/ramalama) (MIT) -- [semperai/amica](https://github.com/semperai/amica) (MIT) -- [withcatai/catai](https://github.com/withcatai/catai) (MIT) -- [Autopen](https://github.com/blackhole89/autopen) (GPL) - -
- -
-Tools - -- [akx/ggify](https://github.com/akx/ggify) – download PyTorch models from Hugging Face Hub and convert them to GGML -- [akx/ollama-dl](https://github.com/akx/ollama-dl) – download models from the Ollama library to be used directly with llama.cpp -- [crashr/gppm](https://github.com/crashr/gppm) – launch llama.cpp instances utilizing NVIDIA Tesla P40 or P100 GPUs with reduced idle power consumption -- [gpustack/gguf-parser](https://github.com/gpustack/gguf-parser-go/tree/main/cmd/gguf-parser) - review/check the GGUF file and estimate the memory usage -- [Styled Lines](https://marketplace.unity.com/packages/tools/generative-ai/styled-lines-llama-cpp-model-292902) (proprietary licensed, async wrapper of inference part for game development in Unity3d with pre-built Mobile and Web platform wrappers and a model example) -- [unslothai/unsloth](https://github.com/unslothai/unsloth) – 🦥 exports/saves fine-tuned and trained models to GGUF (Apache-2.0) - -
- -
-Infrastructure - -- [Paddler](https://github.com/intentee/paddler) - Open-source LLMOps platform for hosting and scaling AI in your own infrastructure -- [GPUStack](https://github.com/gpustack/gpustack) - Manage GPU clusters for running LLMs -- [llama_cpp_canister](https://github.com/onicai/llama_cpp_canister) - llama.cpp as a smart contract on the Internet Computer, using WebAssembly -- [llama-swap](https://github.com/mostlygeek/llama-swap) - transparent proxy that adds automatic model switching with llama-server -- [Kalavai](https://github.com/kalavai-net/kalavai-client) - Crowdsource end to end LLM deployment at any scale -- [llmaz](https://github.com/InftyAI/llmaz) - ☸️ Easy, advanced inference platform for large language models on Kubernetes. -- [LLMKube](https://github.com/defilantech/llmkube) - Kubernetes operator for llama.cpp with multi-GPU and Apple Silicon Metal - support" -
- -
-Games - -- [Lucy's Labyrinth](https://github.com/MorganRO8/Lucys_Labyrinth) - A simple maze game where agents controlled by an AI model will try to trick you. - -
- +The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-org/ggml) library. ## Supported backends | Backend | Target devices | | --- | --- | -| [Metal](docs/build.md#metal-build) | Apple Silicon | | [BLAS](docs/build.md#blas-build) | All | | [BLIS](docs/backend/BLIS.md) | All | -| [SYCL](docs/backend/SYCL.md) | Intel GPU | -| [OpenVINO [In Progress]](docs/backend/OPENVINO.md) | Intel CPUs, GPUs, and NPUs | -| [MUSA](docs/build.md#musa) | Moore Threads GPU | +| [CANN](docs/build.md#cann) | Ascend NPU | | [CUDA](docs/build.md#cuda) | Nvidia GPU | | [HIP](docs/build.md#hip) | AMD GPU | -| [ZenDNN](docs/build.md#zendnn) | AMD CPU | -| [Vulkan](docs/build.md#vulkan) | GPU | -| [CANN](docs/build.md#cann) | Ascend NPU | -| [OpenCL](docs/backend/OPENCL.md) | Adreno GPU | -| [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE | -| [WebGPU](docs/build.md#webgpu) | All | -| [RPC](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) | All | | [Hexagon [In Progress]](docs/backend/snapdragon/README.md) | Snapdragon | +| [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE | +| [MUSA](docs/build.md#musa) | Moore Threads GPU | +| [Metal](docs/build.md#metal-build) | Apple Silicon | +| [OpenCL](docs/backend/OPENCL.md) | Adreno GPU | +| [OpenVINO [In Progress]](docs/backend/OPENVINO.md) | Intel CPUs, GPUs, and NPUs | +| [RPC](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) | All | +| [SYCL](docs/backend/SYCL.md) | Intel GPU | | [VirtGPU](docs/backend/VirtGPU.md) | VirtGPU APIR | +| [Vulkan](docs/build.md#vulkan) | GPU | +| [WebGPU](docs/build.md#webgpu) | All | +| [ZenDNN](docs/build.md#zendnn) | AMD CPU | -## Obtaining and quantizing models +## Documentation -The [Hugging Face](https://huggingface.co) platform hosts a [number of LLMs](https://huggingface.co/models?library=gguf&sort=trending) compatible with `llama.cpp`: - -- [Trending](https://huggingface.co/models?library=gguf&sort=trending) -- [LLaMA](https://huggingface.co/models?sort=trending&search=llama+gguf) - -You can either manually download the GGUF file or directly use any `llama.cpp`-compatible models from [Hugging Face](https://huggingface.co/) or other model hosting sites, by using this CLI argument: `-hf /[:quant]`. For example: - -```sh -llama-cli -hf ggml-org/gemma-3-1b-it-GGUF -``` - -By default, the CLI would download from Hugging Face, you can switch to other options with the environment variable `MODEL_ENDPOINT`. The `MODEL_ENDPOINT` must point to a Hugging Face compatible API endpoint. - -After downloading a model, use the CLI tools to run it locally - see below. - -`llama.cpp` requires the model to be stored in the [GGUF](https://github.com/ggml-org/ggml/blob/master/docs/gguf.md) file format. Models in other data formats can be converted to GGUF using the `convert_*.py` Python scripts in this repo. - -The Hugging Face platform provides a variety of online tools for converting, quantizing and hosting models with `llama.cpp`: - -- Use the [GGUF-my-repo space](https://huggingface.co/spaces/ggml-org/gguf-my-repo) to convert to GGUF format and quantize model weights to smaller sizes -- Use the [GGUF-my-LoRA space](https://huggingface.co/spaces/ggml-org/gguf-my-lora) to convert LoRA adapters to GGUF format (more info: https://github.com/ggml-org/llama.cpp/discussions/10123) -- Use the [GGUF-editor space](https://huggingface.co/spaces/CISCai/gguf-editor) to edit GGUF meta data in the browser (more info: https://github.com/ggml-org/llama.cpp/discussions/9268) -- Use the [Inference Endpoints](https://ui.endpoints.huggingface.co/) to directly host `llama.cpp` in the cloud (more info: https://github.com/ggml-org/llama.cpp/discussions/9669) - -To learn more about model quantization, [read this documentation](tools/quantize/README.md) - -## [`llama-cli`](tools/cli) - -#### A CLI tool for accessing and experimenting with most of `llama.cpp`'s functionality. - --
- Run in conversation mode - - Models with a built-in chat template will automatically activate conversation mode. If this doesn't occur, you can manually enable it by adding `-cnv` and specifying a suitable chat template with `--chat-template NAME` - - ```bash - llama-cli -m model.gguf - - # > hi, who are you? - # Hi there! I'm your helpful assistant! I'm an AI-powered chatbot designed to assist and provide information to users like you. I'm here to help answer your questions, provide guidance, and offer support on a wide range of topics. I'm a friendly and knowledgeable AI, and I'm always happy to help with anything you need. What's on your mind, and how can I assist you today? - # - # > what is 1+1? - # Easy peasy! The answer to 1+1 is... 2! - ``` - -
- --
- Run in conversation mode with custom chat template - - ```bash - # use the "chatml" template (use -h to see the list of supported templates) - llama-cli -m model.gguf -cnv --chat-template chatml - - # use a custom template - llama-cli -m model.gguf -cnv --in-prefix 'User: ' --reverse-prompt 'User:' - ``` - -
- --
- Constrain the output with a custom grammar - - ```bash - llama-cli -m model.gguf -n 256 --grammar-file grammars/json.gbnf -p 'Request: schedule a call at 8pm; Command:' - - # {"appointmentTime": "8pm", "appointmentDetails": "schedule a a call"} - ``` - - The [grammars/](grammars/) folder contains a handful of sample grammars. To write your own, check out the [GBNF Guide](grammars/README.md). - - For authoring more complex JSON grammars, check out https://grammar.intrinsiclabs.ai/ - -
- - -## [`llama-server`](tools/server) - -#### A lightweight, [OpenAI API](https://github.com/openai/openai-openapi) compatible, HTTP server for serving LLMs. - --
- Start a local HTTP server with default configuration on port 8080 - - ```bash - llama-server -m model.gguf --port 8080 - - # Basic web UI can be accessed via browser: http://localhost:8080 - # Chat completion endpoint: http://localhost:8080/v1/chat/completions - ``` - -
- --
- Support multiple-users and parallel decoding - - ```bash - # up to 4 concurrent requests, each with 4096 max context - llama-server -m model.gguf -c 16384 -np 4 - ``` - -
- --
- Enable speculative decoding - - ```bash - # the draft.gguf model should be a small variant of the target model.gguf - llama-server -m model.gguf -md draft.gguf - ``` - -
- --
- Serve an embedding model - - ```bash - # use the /embedding endpoint - llama-server -m model.gguf --embedding --pooling cls -ub 8192 - ``` - -
- --
- Serve a reranking model - - ```bash - # use the /reranking endpoint - llama-server -m model.gguf --reranking - ``` - -
- --
- Constrain all outputs with a grammar - - ```bash - # custom grammar - llama-server -m model.gguf --grammar-file grammar.gbnf - - # JSON - llama-server -m model.gguf --grammar-file grammars/json.gbnf - ``` - -
- - -## [`llama-perplexity`](tools/perplexity) - -#### A tool for measuring the [perplexity](tools/perplexity/README.md) [^1] (and other quality metrics) of a model over a given text. - --
- Measure the perplexity over a text file - - ```bash - llama-perplexity -m model.gguf -f file.txt - - # [1]15.2701,[2]5.4007,[3]5.3073,[4]6.2965,[5]5.8940,[6]5.6096,[7]5.7942,[8]4.9297, ... - # Final estimate: PPL = 5.4007 +/- 0.67339 - ``` - -
- --
- Measure KL divergence - - ```bash - # TODO - ``` - -
- -[^1]: [https://huggingface.co/docs/transformers/perplexity](https://huggingface.co/docs/transformers/perplexity) - -## [`llama-bench`](tools/llama-bench) - -#### Benchmark the performance of the inference for various parameters. - --
- Run default benchmark - - ```bash - llama-bench -m model.gguf - - # Output: - # | model | size | params | backend | threads | test | t/s | - # | ------------------- | ---------: | ---------: | ---------- | ------: | ------------: | -------------------: | - # | qwen2 1.5B Q4_0 | 885.97 MiB | 1.54 B | Metal,BLAS | 16 | pp512 | 5765.41 ± 20.55 | - # | qwen2 1.5B Q4_0 | 885.97 MiB | 1.54 B | Metal,BLAS | 16 | tg128 | 197.71 ± 0.81 | - # - # build: 3e0ba0e60 (4229) - ``` - -
- -## [`llama-simple`](examples/simple) - -#### A minimal example for implementing apps with `llama.cpp`. Useful for developers. - --
- Basic text completion - - ```bash - llama-simple -m model.gguf - - # Hello my name is Kaitlyn and I am a 16 year old girl. I am a junior in high school and I am currently taking a class called "The Art of - ``` - -
- - -## Contributing - -- Contributors can open PRs -- Collaborators will be invited based on contributions -- Maintainers can push to branches in the `llama.cpp` repo and merge PRs into the `master` branch -- Any help with managing issues, PRs and projects is very appreciated! -- See [good first issues](https://github.com/ggml-org/llama.cpp/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) for tasks suitable for first contributions -- Read the [CONTRIBUTING.md](CONTRIBUTING.md) for more information -- Make sure to read this: [Inference at the edge](https://github.com/ggml-org/llama.cpp/discussions/205) -- A bit of backstory for those who are interested: [Changelog podcast](https://changelog.com/podcast/532) - -## Other documentation +#### Tools - [cli](tools/cli/README.md) - [completion](tools/completion/README.md) - [server](tools/server/README.md) - [GBNF grammars](grammars/README.md) -#### Development documentation +#### Development - [How to build](docs/build.md) - [Running on Docker](docs/docker.md) @@ -539,63 +103,19 @@ To learn more about model quantization, [read this documentation](tools/quantize - [Multi-GPU usage](docs/multi-gpu.md) - [Performance troubleshooting](docs/development/token_generation_performance_tips.md) - [GGML tips & tricks](https://github.com/ggml-org/llama.cpp/wiki/GGML-Tips-&-Tricks) +- [XCFramework](docs/xcframework.md) +- [Completions](docs/completions.md) +- [Models](docs/models.md) -#### Seminal papers and background on the models +## Contributing -If your issue is with model generation quality, then please at least scan the following links and papers to understand the limitations of LLaMA models. This is especially important when choosing an appropriate model size and appreciating both the significant and subtle differences between LLaMA models and ChatGPT: -- LLaMA: - - [Introducing LLaMA: A foundational, 65-billion-parameter large language model](https://ai.facebook.com/blog/large-language-model-llama-meta-ai/) - - [LLaMA: Open and Efficient Foundation Language Models](https://arxiv.org/abs/2302.13971) -- GPT-3 - - [Language Models are Few-Shot Learners](https://arxiv.org/abs/2005.14165) -- GPT-3.5 / InstructGPT / ChatGPT: - - [Aligning language models to follow instructions](https://openai.com/research/instruction-following) - - [Training language models to follow instructions with human feedback](https://arxiv.org/abs/2203.02155) +- Contributors can open PRs +- Collaborators will be invited based on contributions +- Maintainers can push to branches in the `llama.cpp` repo and merge PRs into the `master` branch +- Any help with managing issues, PRs and projects is very appreciated! +- Read the [CONTRIBUTING.md](CONTRIBUTING.md) for more information -## XCFramework -The XCFramework is a precompiled version of the library for iOS, visionOS, tvOS, -and macOS. It can be used in Swift projects without the need to compile the -library from source. For example: -```swift -// swift-tools-version: 5.10 -// The swift-tools-version declares the minimum version of Swift required to build this package. - -import PackageDescription - -let package = Package( - name: "MyLlamaPackage", - targets: [ - .executableTarget( - name: "MyLlamaPackage", - dependencies: [ - "LlamaFramework" - ]), - .binaryTarget( - name: "LlamaFramework", - url: "https://github.com/ggml-org/llama.cpp/releases/download/b5046/llama-b5046-xcframework.zip", - checksum: "c19be78b5f00d8d29a25da41042cb7afa094cbf6280a225abe614b03b20029ab" - ) - ] -) -``` -The above example is using an intermediate build `b5046` of the library. This can be modified -to use a different version by changing the URL and checksum. - -## Completions -Command-line completion is available for some environments. - -#### Bash Completion -```bash -$ build/bin/llama-cli --completion-bash > ~/.llama-completion.bash -$ source ~/.llama-completion.bash -``` -Optionally this can be added to your `.bashrc` or `.bash_profile` to load it -automatically. For example: -```console -$ echo "source ~/.llama-completion.bash" >> ~/.bashrc -``` - -## Dependencies +## Acknowledgements - [yhirose/cpp-httplib](https://github.com/yhirose/cpp-httplib) - Single-header HTTP server, used by `llama-server` - MIT license - [stb-image](https://github.com/nothings/stb) - Single-header image format decoder, used by multimodal subsystem - Public domain diff --git a/common/arg.cpp b/common/arg.cpp index 9753441313..79480e06f9 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -539,6 +539,13 @@ void common_models_handler_apply(common_models_handler & handler, common_params } }; + // an explicit draft file selection (e.g. -md with -hfd) disables the sidecar resolution of the draft repo + if (!params.speculative.draft.mparams.hf_file.empty()) { + plan_spec.mtp = {}; + plan_spec.dflash = {}; + plan_spec.eagle3 = {}; + } + // infer the speculative type from the sidecar shipped by the draft repo when none is requested if (spec_types_is_default(params)) { if (!plan_spec.mtp.local_path.empty()) { @@ -588,6 +595,11 @@ void common_models_handler_apply(common_models_handler & handler, common_params }); } + // a wired draft sidecar counts as an explicit draft for the main plan fallback below + if (spec_sidecar_found) { + had_spec_url = true; + } + // handle plan_spec (e.g. --spec-draft-hf) if (!plan_spec.model_files.empty() && !had_spec_url && !spec_sidecar_found) { add_tasks(plan_spec.model_files, plan_spec.primary, params.speculative.draft.mparams); @@ -1049,6 +1061,31 @@ static std::vector parse_device_list(const std::string & val return devices; } +void common_print_available_devices() { + constexpr size_t MiB = 1024 * 1024; + std::vector devices; + + ggml_backend_load_all(); + + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + auto * dev = ggml_backend_dev_get(i); + if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) { + devices.push_back(dev); + } + } + printf("Available devices:\n"); + + if (devices.empty()) { + printf(" (none)\n"); + return; + } + for (auto * dev : devices) { + size_t free, total; + ggml_backend_dev_memory(dev, &free, &total); + printf(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / MiB, free / MiB); + } +} + static void add_rpc_devices(const std::string & servers) { auto rpc_servers = string_split(servers, ','); if (rpc_servers.empty()) { @@ -2508,7 +2545,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } add_opt(common_arg( {"--mlock"}, - "DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing", + "DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing", [](common_params & params) { LOG_WRN("DEPRECATED: --mlock is deprecated. use --load-mode mlock instead\n"); params.load_mode = LLAMA_LOAD_MODE_MLOCK; @@ -2537,13 +2574,15 @@ common_params_context common_params_parser_init(common_params & params, llama_ex "model loading mode (default: mmap)\n" "- none: no special loading mode\n" "- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)\n" - "- mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n" + "- mlock: force system to keep model in RAM rather than swapping or compressing\n" + "- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n" "- dio: use DirectIO if available\n", [](common_params & params, const std::string & value) { - /**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; } - else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; } - else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; } - else if (value == "dio") { params.load_mode = LLAMA_LOAD_MODE_DIRECT_IO; } + /**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; } + else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; } + else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; } + else if (value == "mmap+mlock") { params.load_mode = LLAMA_LOAD_MODE_MMAP_MLOCK; } + else if (value == "dio") { params.load_mode = LLAMA_LOAD_MODE_DIRECT_IO; } else { throw std::invalid_argument("invalid value"); } } ).set_env("LLAMA_ARG_LOAD_MODE")); @@ -2574,20 +2613,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--list-devices"}, "print list of available devices and exit", [](common_params &) { - ggml_backend_load_all(); - std::vector devices; - for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { - auto * dev = ggml_backend_dev_get(i); - if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) { - devices.push_back(dev); - } - } - printf("Available devices:\n"); - for (auto * dev : devices) { - size_t free, total; - ggml_backend_dev_memory(dev, &free, &total); - printf(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024); - } + common_print_available_devices(); exit(0); } )); diff --git a/common/arg.h b/common/arg.h index 54a38b9cce..8f609e356f 100644 --- a/common/arg.h +++ b/common/arg.h @@ -123,6 +123,9 @@ struct common_params_context { // if one argument has invalid value, it will automatically display usage of the specific argument (and not the full usage message) bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr); +// load all backends and print the list of available (non-CPU) devices to stdout +void common_print_available_devices(); + // parse input arguments from CLI into a map bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map & out_map); diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index a309f02765..f786f5ff23 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -1056,3 +1056,141 @@ void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, co visit(arena, child_id); } } + +static void minimax_m3_collect(const common_peg_ast_arena & arena, + const common_peg_ast_node & node, + const std::string & tag, + std::vector & out) { + for (auto child_id : node.children) { + const auto & child = arena.get(child_id); + if (child.tag == tag) { + out.push_back(child_id); + } else { + minimax_m3_collect(arena, child, tag, out); + } + } +} + +static common_peg_ast_id minimax_m3_value_of(const common_peg_ast_arena & arena, const common_peg_ast_node & node) { + for (auto child_id : node.children) { + const auto & tag = arena.get(child_id).tag; + if (tag == common_chat_peg_builder::TOOL_ARG_VALUE || + tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE || + tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_OBJECT || + tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_ARRAY) { + return child_id; + } + } + return COMMON_PEG_INVALID_AST_ID; +} + +static std::string minimax_m3_value_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id, bool closed); + +static std::string minimax_m3_member_to_json(const common_peg_ast_arena & arena, const common_peg_ast_node & node) { + auto name_id = arena.find_by_tag(node, common_chat_peg_builder::TOOL_ARG_NAME); + if (name_id == COMMON_PEG_INVALID_AST_ID) { + return ""; + } + + return ordered_json(arena.get(name_id).text).dump() + ":" + + minimax_m3_value_to_json(arena, minimax_m3_value_of(arena, node), !node.is_partial); +} + +static std::string minimax_m3_container_to_json(const common_peg_ast_arena & arena, + const common_peg_ast_node & node, + bool is_object, + bool closed) { + const std::string tag = is_object ? common_chat_peg_builder::TOOL_ARG + : common_chat_peg_minimax_m3_mapper::TOOL_ARG_ITEM; + + std::vector entries; + minimax_m3_collect(arena, node, tag, entries); + + std::string result = is_object ? "{" : "["; + + bool add_comma = false; + for (auto entry_id : entries) { + const auto & entry = arena.get(entry_id); + + std::string text; + if (is_object) { + text = minimax_m3_member_to_json(arena, entry); + } else { + text = minimax_m3_value_to_json(arena, minimax_m3_value_of(arena, entry), !entry.is_partial); + } + + if (text.empty()) { + continue; + } + + if (add_comma) { + result += ","; + } + add_comma = true; + result += text; + } + + if (closed) { + result += is_object ? "}" : "]"; + } + return result; +} + +static std::string minimax_m3_value_to_json(const common_peg_ast_arena & arena, common_peg_ast_id id, bool closed) { + if (id == COMMON_PEG_INVALID_AST_ID) { + return ""; + } + + const auto & node = arena.get(id); + + if (node.tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_OBJECT) { + return minimax_m3_container_to_json(arena, node, /* is_object = */ true, closed); + } + + if (node.tag == common_chat_peg_minimax_m3_mapper::TOOL_ARG_ARRAY) { + return minimax_m3_container_to_json(arena, node, /* is_object = */ false, closed); + } + + if (node.tag == common_chat_peg_builder::TOOL_ARG_STRING_VALUE) { + return "\"" + escape_json_string_inner(std::string(node.text)) + (closed ? "\"" : ""); + } + + // Numbers and booleans are written verbatim by the template + return std::string(node.text); +} + +void common_chat_peg_minimax_m3_mapper::from_ast(const common_peg_ast_arena & arena, + const common_peg_parse_result & result) { + for (const auto & node : result.nodes) { + visit(arena, node); + } +} + +void common_chat_peg_minimax_m3_mapper::visit(const common_peg_ast_arena & arena, common_peg_ast_id id) { + const auto & node = arena.get(id); + + if (node.tag == common_chat_peg_builder::REASONING) { + result.reasoning_content += std::string(node.text); + return; + } + + if (node.tag == common_chat_peg_builder::CONTENT) { + result.content += std::string(node.text); + return; + } + + if (node.tag == common_chat_peg_builder::TOOL) { + auto name_id = arena.find_by_tag(node, common_chat_peg_builder::TOOL_NAME); + if (name_id != COMMON_PEG_INVALID_AST_ID) { + common_chat_tool_call call; + call.name = std::string(arena.get(name_id).text); + call.arguments = minimax_m3_container_to_json(arena, node, /* is_object = */ true, !node.is_partial); + result.tool_calls.push_back(call); + } + return; + } + + for (auto child_id : node.children) { + visit(arena, child_id); + } +} diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index b3ffd7de2d..cd14f2c117 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -40,6 +40,18 @@ class common_chat_peg_gemma4_mapper : public common_chat_peg_mapper { void visit(const common_peg_ast_arena & arena, common_peg_ast_id id); }; +class common_chat_peg_minimax_m3_mapper : public common_chat_peg_mapper { + public: + static constexpr const char * TOOL_ARG_OBJECT = "tool-arg-object"; + static constexpr const char * TOOL_ARG_ARRAY = "tool-arg-array"; + static constexpr const char * TOOL_ARG_ITEM = "tool-arg-item"; + + common_chat_peg_minimax_m3_mapper(common_chat_msg & msg) : common_chat_peg_mapper(msg) {} + virtual void from_ast(const common_peg_ast_arena & arena, const common_peg_parse_result & result); + private: + void visit(const common_peg_ast_arena & arena, common_peg_ast_id id); +}; + struct content_structure; struct tool_call_structure; diff --git a/common/chat.cpp b/common/chat.cpp index 7a6e7238cf..7740f35c0e 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -816,6 +816,8 @@ const char * common_chat_format_name(common_chat_format format) { return "peg-native"; case COMMON_CHAT_FORMAT_PEG_GEMMA4: return "peg-gemma4"; + case COMMON_CHAT_FORMAT_PEG_MINIMAX_M3: + return "peg-minimax-m3"; default: throw std::runtime_error("Unknown chat format"); } @@ -2270,6 +2272,264 @@ static common_chat_params common_chat_params_init_cohere2moe(const common_chat_t return data; } +static common_chat_params common_chat_params_init_minimax_m3(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_MINIMAX_M3; + data.supports_thinking = true; + data.thinking_start_tag = ""; + data.thinking_end_tags = {""}; + + // M3 prefixes every tool tag with the namespace token "]<]minimax[>["; + // params use the parameter name as the tag (...). + const std::string NS = "]<]minimax[>["; + const std::string THINK_START = ""; + const std::string THINK_END = ""; + const std::string FC_START = NS + ""; + const std::string FC_END = NS + ""; + const std::string INVOKE_END = NS + ""; + + data.preserved_tokens = { + NS, + "", + "", + THINK_START, + THINK_END, + }; + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "]~b]ai" }, + { COMMON_CHAT_ROLE_USER, "]~b]user" }, + { COMMON_CHAT_ROLE_TOOL, "]~b]tool" }, + { COMMON_CHAT_ROLE_SYSTEM, "]~b]developer" }, + { COMMON_CHAT_ROLE_SYSTEM, "]~b]system" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + const std::string GEN_PROMPT = data.generation_prompt; + + using mm3 = common_chat_peg_minimax_m3_mapper; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START); + auto end = p.end(); + + auto reasoning = p.eps(); + if (extract_reasoning) { + auto block = inputs.enable_thinking + ? p.literal(THINK_START) + p.space() + + p.ac(p.reasoning(p.until(THINK_END)) + p.literal(THINK_END), THINK_END) + : p.literal(THINK_START) + p.ac(p.until(THINK_END) + p.literal(THINK_END), THINK_END); + + // A turn without reasoning is prefixed with a bare , written either by the + // generation prompt (thinking_mode = "disabled") or by the model itself. + reasoning = p.optional(p.choice({ block, p.literal(THINK_END) })); + } + + if (has_response_format) { + auto response_format = p.rule("response-format", + p.literal("```json") + p.space() + + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)) + + p.space() + p.literal("```")); + return generation_prompt + reasoning + response_format + end; + } + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return generation_prompt + reasoning + p.content(p.rest()) + end; + } + + auto alternatives_of = [](const json & schema) -> std::optional { + for (const auto * keyword : { "oneOf", "anyOf" }) { + if (schema.contains(keyword) && schema.at(keyword).is_array() && !schema.at(keyword).empty()) { + return schema.at(keyword); + } + } + return std::nullopt; + }; + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + auto params = function.contains("parameters") ? function.at("parameters") : json::object(); + + auto schema_info = common_schema_info(); + schema_info.resolve_refs(params); + + // The template expands argument values recursively in XML (see the to_xml() macro) + std::function value_of; + std::function members_of; + + auto element_of = [&](const std::string & tag, const json & schema, const std::string & rule_name) { + const std::string close = NS + ""; + return p.rule(rule_name, + p.tool_arg( + p.tool_arg_open( + p.literal(NS + "<") + + p.tool_arg_name(p.literal(tag)) + + p.literal(">")) + + value_of(schema, rule_name, close))); + }; + + value_of = [&](const json & schema, + const std::string & rule_name, + const std::string & close) -> common_peg_parser { + auto close_tag = p.tool_arg_close(p.literal(close)); + + // A string accepts anything, so a union with a string alternative is a string + if (schema_info.resolves_to_string(schema)) { + return p.ac(p.tool_arg_string_value(p.until(close)) + close_tag, close); + } + + if (auto alternatives = alternatives_of(schema)) { + std::vector choices; + + size_t index = 0; + for (const auto & alternative : *alternatives) { + const std::string alt_name = rule_name + "-" + std::to_string(index++); + + // There is a risk that this breaks streaming deltas, but that's a risk we + // assume to provide tool arg streaming. + choices.push_back(value_of(alternative, alt_name, close)); + } + + return p.choice(choices); + } + + const std::string type = schema.contains("type") && schema.at("type").is_string() + ? schema.at("type").get() + : ""; + + if (type == "object" && schema.contains("properties")) { + return p.tag(mm3::TOOL_ARG_OBJECT, members_of(schema, rule_name)) + p.space() + close_tag; + } + + if (type == "array" && schema.contains("items")) { + const std::string item_close = NS + ""; + auto item = p.rule(rule_name + "-item", + p.tag(mm3::TOOL_ARG_ITEM, + p.literal(NS + "") + + value_of(schema.at("items"), rule_name + "-item", item_close))); + return p.tag(mm3::TOOL_ARG_ARRAY, p.repeat(p.space() + item, 0, -1)) + p.space() + close_tag; + } + + return p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", schema, false)) + close_tag; + }; + + // Required properties in schema order, then any number of optional ones in any order. + members_of = [&](const json & schema, const std::string & rule_prefix) -> common_peg_parser { + const auto & props = schema.at("properties"); + + std::set required; + if (schema.contains("required")) { + schema.at("required").get_to(required); + } + + std::vector required_elements; + std::vector optional_elements; + for (const auto & [key, key_schema] : props.items()) { + auto element = element_of(key, key_schema, rule_prefix + "-" + key); + if (required.find(key) != required.end()) { + required_elements.push_back(element); + } else { + optional_elements.push_back(element); + } + } + + common_peg_parser members = p.eps(); + for (size_t i = 0; i < required_elements.size(); i++) { + if (i > 0) { + members = members + p.space(); + } + members = members + required_elements[i]; + } + + if (!optional_elements.empty()) { + common_peg_parser any_optional = p.choice(); + for (const auto & element : optional_elements) { + any_optional |= element; + } + members = members + p.repeat(p.space() + any_optional, 0, -1); + } + + return members; + }; + + common_peg_parser invoke_body = + params.contains("properties") ? members_of(params, "tool-" + name + "-arg") : p.eps(); + + auto func_parser = p.tool( + p.tool_open(p.literal(NS + "")) + + p.space() + invoke_body + p.space() + + p.tool_close(p.literal(INVOKE_END))); + + tool_choice |= p.rule("tool-" + name, func_parser); + }); + + auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; + + common_peg_parser tool_calls = p.eps(); + if (inputs.parallel_tool_calls) { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + + p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END)); + } else { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); + } + + if (!require_tools) { + tool_calls = p.optional(tool_calls); + } + + auto content_before_tools = p.content(p.until(FC_START)); + return generation_prompt + reasoning + content_before_tools + tool_calls + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); + builder.resolve_refs(schema); + }); + if (has_response_format) { + auto schema = inputs.json_schema; + builder.resolve_refs(schema); + } + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, FC_START }, + }; + } + + return data; +} + namespace workaround { static void map_developer_role_to_system(json & messages) { @@ -2707,6 +2967,15 @@ std::optional common_chat_try_specialized_template( return common_chat_params_init_gigachat_v3(tmpl, params); } + // MiniMax-M3: the namespace token "]<]minimax[>[" collides with the autoparser's + // markup delimiters, so detect the template and use a dedicated parser. + if (src.find("]<]minimax[>[") != std::string::npos && + src.find("") != std::string::npos && + src.find(" mapper; if (params.format == COMMON_CHAT_FORMAT_PEG_GEMMA4) { mapper = std::make_unique(msg); + } else if (params.format == COMMON_CHAT_FORMAT_PEG_MINIMAX_M3) { + mapper = std::make_unique(msg); } else { mapper = std::make_unique(msg); } @@ -3020,6 +3291,8 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars std::unique_ptr mapper; if (params.format == COMMON_CHAT_FORMAT_PEG_GEMMA4) { mapper = std::make_unique(msg); + } else if (params.format == COMMON_CHAT_FORMAT_PEG_MINIMAX_M3) { + mapper = std::make_unique(msg); } else { mapper = std::make_unique(msg); } diff --git a/common/chat.h b/common/chat.h index d79f4ecd77..6d5b220aeb 100644 --- a/common/chat.h +++ b/common/chat.h @@ -233,6 +233,7 @@ enum common_chat_format { COMMON_CHAT_FORMAT_PEG_SIMPLE, COMMON_CHAT_FORMAT_PEG_NATIVE, COMMON_CHAT_FORMAT_PEG_GEMMA4, + COMMON_CHAT_FORMAT_PEG_MINIMAX_M3, COMMON_CHAT_FORMAT_COUNT, // Not a format, just the # formats }; diff --git a/common/common.cpp b/common/common.cpp index 82dd780fd8..ff27d392fb 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1476,6 +1476,20 @@ std::string common_get_model_endpoint() { return model_endpoint; } +char * common_get_model_or_exit(int argc, char * argv[]) { + if (argc > 1) { + return argv[1]; + } + + char * path = getenv("LLAMACPP_TEST_MODELFILE"); + if (!path || strlen(path) == 0) { + fprintf(stderr, "\033[33mWARNING: No model file provided. Skipping this test. Set LLAMACPP_TEST_MODELFILE= to silence this warning and run this test.\n\033[0m"); + exit(EXIT_SUCCESS); + } + + return path; +} + common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) { auto * mem = llama_get_memory(ctx); if (mem == nullptr) { @@ -1518,23 +1532,49 @@ done: return res; } -void common_context_seq_rm(llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1) { +static void common_context_seq_rm(llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1) { auto * mem = llama_get_memory(ctx); if (!llama_memory_seq_rm(mem, seq_id, p0, p1)) { GGML_ABORT("%s", string_format("failed to remove sequence %d with p0=%d, p1=%d\n", seq_id, p0, p1).c_str()); } } -void common_context_seq_cp(llama_context * ctx, llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { +static void common_context_seq_cp(llama_context * ctx, llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) { auto * mem = llama_get_memory(ctx); llama_memory_seq_cp(mem, seq_id_src, seq_id_dst, p0, p1); } -void common_context_seq_add(llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) { +static void common_context_seq_add(llama_context * ctx, llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) { auto * mem = llama_get_memory(ctx); llama_memory_seq_add(mem, seq_id, p0, p1, delta); } +void common_memory::init(llama_context * ctx_tgt, llama_context * ctx_dft) { + this->ctx_tgt = ctx_tgt; + this->ctx_dft = ctx_dft; +} + +void common_memory::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) const { + common_context_seq_rm(ctx_tgt, seq_id, p0, p1); + if (ctx_dft) { + common_context_seq_rm(ctx_dft, seq_id, p0, p1); + } +} + +void common_memory::seq_cp(llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) const { + common_context_seq_cp(ctx_tgt, seq_id_src, seq_id_dst, p0, p1); + if (ctx_dft) { + common_context_seq_cp(ctx_dft, seq_id_src, seq_id_dst, p0, p1); + } +} + +void common_memory::seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) const { + common_context_seq_add(ctx_tgt, seq_id, p0, p1, delta); + if (ctx_dft) { + common_context_seq_add(ctx_dft, seq_id, p0, p1, delta); + } +} + void common_set_adapter_lora(struct llama_context * ctx, std::vector & lora) { std::vector loras; std::vector scales; @@ -1580,6 +1620,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) { mparams.progress_callback = params.load_progress_callback; mparams.progress_callback_user_data = params.load_progress_callback_user_data; mparams.no_alloc = params.no_alloc; + mparams.load_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); return mparams; } diff --git a/common/common.h b/common/common.h index 2792521836..919c0ea103 100644 --- a/common/common.h +++ b/common/common.h @@ -173,6 +173,7 @@ enum common_speculative_type { COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding + COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head) COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values @@ -293,10 +294,6 @@ struct common_params_sampling { bool backend_sampling = false; - bool has_logit_bias() const { - return !logit_bias.empty(); - } - // print the parameters into a string std::string print() const; }; @@ -388,7 +385,7 @@ struct common_params_speculative { uint32_t need_n_rs_seq() const { bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) { - return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH; + return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; }); return needs_rs_seq ? draft.n_max : 0u; @@ -933,6 +930,9 @@ void common_set_adapter_lora(struct llama_context * ctx, std::vector(model, '/'); auto model_dir = model_parts.end() - 1; @@ -600,10 +614,19 @@ static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files, auto bits = extract_quant_bits(f.path); auto diff = std::abs(bits - model_bits); - if (!found || depth > best_depth || (depth == best_depth && diff < best_diff)) { + std::string path_upper = f.path; + for (char & c : path_upper) { + c = (char) std::toupper((unsigned char) c); + } + bool exact = !tag_upper.empty() && path_upper.find("-" + tag_upper + ".") != std::string::npos; + + if (!found || depth > best_depth || + (depth == best_depth && exact && !best_exact) || + (depth == best_depth && exact == best_exact && diff < best_diff)) { best = f; best_depth = depth; best_diff = diff; + best_exact = exact; found = true; } } @@ -616,18 +639,21 @@ static hf_cache::hf_file find_best_mmproj(const hf_cache::hf_files & files, } static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files, - const std::string & model) { - return find_best_sibling(files, model, "mtp-"); + const std::string & model, + const std::string & tag = "") { + return find_best_sibling(files, model, "mtp-", tag); } static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files, - const std::string & model) { - return find_best_sibling(files, model, "eagle3-"); + const std::string & model, + const std::string & tag = "") { + return find_best_sibling(files, model, "eagle3-", tag); } static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files, - const std::string & model) { - return find_best_sibling(files, model, "dflash-"); + const std::string & model, + const std::string & tag = "") { + return find_best_sibling(files, model, "dflash-", tag); } static bool gguf_filename_is_model(const std::string & filepath) { @@ -736,27 +762,36 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & } } else { primary = find_best_model(all, tag); - if (primary.path.empty()) { + // a requested sidecar can resolve on its own, without a full model of the same tag + if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); return plan; } } - plan.primary = primary; - plan.model_files = get_split_files(all, primary); + if (!primary.path.empty()) { + plan.primary = primary; + plan.model_files = get_split_files(all, primary); + } - if (opts.download_mmproj) { + if (opts.download_mmproj && !primary.path.empty()) { plan.mmproj = find_best_mmproj(all, primary.path); } if (opts.download_mtp) { - plan.mtp = find_best_mtp(all, primary.path); + plan.mtp = find_best_mtp(all, primary.path, tag); } if (opts.download_dflash) { - plan.dflash = find_best_dflash(all, primary.path); + plan.dflash = find_best_dflash(all, primary.path, tag); } if (opts.download_eagle3) { - plan.eagle3 = find_best_eagle3(all, primary.path); + plan.eagle3 = find_best_eagle3(all, primary.path, tag); + } + + if (primary.path.empty() && + plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty()) { + LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); + list_available_gguf_files(all); } return plan; diff --git a/common/fit.cpp b/common/fit.cpp index c79221cb00..c82d066ad4 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -136,7 +136,7 @@ static std::vector common_get_device_memory_data_impl( devs.push_back(llama_model_get_device(model, i)); } - hp_ngl = llama_model_n_layer(model); + hp_ngl = llama_model_n_layer(model) + llama_model_n_layer_nextn(model); hp_n_ctx_train = llama_model_n_ctx_train(model); hp_n_expert = llama_model_n_expert(model); diff --git a/common/sampling.cpp b/common/sampling.cpp index 7b241e34f7..256ac161e2 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -310,8 +310,19 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, st } } - if (params.has_logit_bias()) { - samplers.push_back(llama_sampler_init_logit_bias(llama_vocab_n_tokens(vocab), params.logit_bias.size(), params.logit_bias.data())); + // logit bias: user biases + model suppress tokens (-INFINITY) + { + std::vector merged = params.logit_bias; + + int32_t n_suppress = 0; + const llama_token * suppress = llama_vocab_get_suppress_tokens(vocab, &n_suppress); + for (int32_t i = 0; i < n_suppress; ++i) { + merged.push_back({ suppress[i], -INFINITY }); + } + + if (!merged.empty()) { + samplers.push_back(llama_sampler_init_logit_bias(llama_vocab_n_tokens(vocab), merged.size(), merged.data())); + } } if (params.mirostat == 0) { diff --git a/common/speculative.cpp b/common/speculative.cpp index 3cb08767bd..5653a90b88 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -34,6 +34,7 @@ const std::map common_speculative_type_fro {"draft-eagle3", COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3}, {"draft-mtp", COMMON_SPECULATIVE_TYPE_DRAFT_MTP}, {"draft-dflash", COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH}, + {"draft-dspark", COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK}, {"ngram-simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE}, {"ngram-map-k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K}, {"ngram-map-k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V}, @@ -437,6 +438,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { int32_t n_embd_dec = 0; // draft hidden size int32_t n_embd_enc = 0; // target_layer_ids_n * target_hidden_size int32_t n_embd_tgt = 0; // target model hidden size + int32_t n_layer_tgt = 0; // target model layer count const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices uint32_t target_layer_ids_n = 0; @@ -478,6 +480,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { n_embd_tgt = llama_model_n_embd(model_tgt); n_embd_dec = llama_model_n_embd(model_dft); n_embd_enc = (int32_t) target_layer_ids_n * n_embd_tgt; + n_layer_tgt = llama_model_n_layer(model_tgt); const int32_t n_b = (int32_t) llama_n_batch(ctx_dft); batch = llama_batch_init(/*n_tokens=*/ n_b, /*embd=*/ n_embd_dec, /*n_seq_max=*/ 1); @@ -510,9 +513,15 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { } } - // turn on extraction of the target layers' input embeddings + // turn on extraction of the target layers' hidden states for (uint32_t k = 0; k < target_layer_ids_n; ++k) { - llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true); + if (target_layer_ids[k] < n_layer_tgt) { + llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true); + } else if (target_layer_ids[k] == n_layer_tgt) { + llama_set_embeddings_nextn(ctx_tgt, true, /*masked*/ false); + } else { + GGML_ABORT("EAGLE3: target layer id %d exceeds target n_layer %d", target_layer_ids[k], n_layer_tgt); + } } // turn on extraction of the draft model's pre-norm hidden state @@ -600,7 +609,9 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { features_buf.resize((size_t) n_tokens * n_embd_enc, 0.0f); for (uint32_t k = 0; k < target_layer_ids_n; ++k) { - const float * layer = llama_get_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k]); + const float * layer = target_layer_ids[k] < n_layer_tgt + ? llama_get_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k]) + : llama_get_embeddings_nextn(ctx_tgt); if (!layer) { GGML_ABORT("EAGLE3: target layer %d input not extracted.", target_layer_ids[k]); } @@ -918,15 +929,20 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { int32_t block_size = 0; llama_token mask_token_id = 0; + // draft-dspark: the draft carries a Markov head and uses an anchor-first block layout + const bool is_dspark; + const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices uint32_t target_layer_ids_n = 0; // scratch buffer for concatenated target features [n_tokens, n_embd_enc] std::vector features_buf; - common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, n_seq) + common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq, + common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH) + : common_speculative_impl(type, n_seq) , params(params.draft) + , is_dspark(type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK) { auto * ctx_tgt = this->params.ctx_tgt; auto * ctx_dft = this->params.ctx_dft; @@ -953,16 +969,18 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { } mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft)); - LOG_INF("%s: adding speculative implementation 'draft-dflash'\n", __func__); + LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(type).c_str()); LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min); LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u\n", __func__, block_size, mask_token_id, target_layer_ids_n); - // DFlash input is [id_last, * (block_size-1)], so it can draft at most block_size-1 tokens per step - if (this->params.n_max > block_size - 1 || this->params.n_min > block_size - 1) { - LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained DFlash block size %d -- clamping to %d\n", - __func__, this->params.n_max, this->params.n_min, block_size, block_size - 1); - this->params.n_max = std::min(this->params.n_max, block_size - 1); - this->params.n_min = std::min(this->params.n_min, block_size - 1); + // DFlash input is [id_last, * (block_size-1)]: in-place denoising yields at most + // block_size-1 draft tokens, DSpark yield a full block_size draft tokens + const int32_t n_draft_max = is_dspark ? block_size : block_size - 1; + if (this->params.n_max > n_draft_max || this->params.n_min > n_draft_max) { + LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained block size %d -- clamping to %d\n", + __func__, this->params.n_max, this->params.n_min, block_size, n_draft_max); + this->params.n_max = std::min(this->params.n_max, n_draft_max); + this->params.n_min = std::min(this->params.n_min, n_draft_max); } batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq); @@ -1126,12 +1144,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { const int32_t n = (int32_t) dp.n_past; - int32_t n_draft = params.n_max; - if (dp.n_max > 0) { - n_draft = std::min(n_draft, dp.n_max); - } + const int32_t n_draft = params.n_max; - const int32_t n_block_tokens = n_draft + 1; // id_last + n_draft * + const int32_t n_block_tokens = n_draft + (is_dspark ? 0 : 1); i_block_beg[seq_id] = batch.n_tokens; n_block [seq_id] = n_block_tokens; for (int32_t i = 0; i < n_block_tokens; ++i) { @@ -1163,27 +1178,57 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { auto & result = *dp.result; - // greedily read the predicted block at this sequence's noise positions 1..n_block_tokens-1 - for (int32_t i = 1; i < n_block_tokens; ++i) { - common_sampler_sample(smpl, ctx_dft, beg + i, true); + if (is_dspark) { + // DSpark predicts the next token from position 0 and optionally truncates + // at the first position below the confidence threshold. + const float * conf = params.p_min > 0.0f ? llama_get_embeddings_nextn(ctx_dft) : nullptr; - const auto * cur_p = common_sampler_get_candidates(smpl, true); + for (int32_t i = 0; i < n_block_tokens; ++i) { + const int32_t idx = beg + i; - for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", - seq_id, k, i - 1, cur_p->data[k].id, cur_p->data[k].p, - common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); + if (conf && conf[(size_t) idx * n_embd_dec] < params.p_min) { + break; + } + + common_sampler_sample(smpl, ctx_dft, idx, true); + + const auto * cur_p = common_sampler_get_candidates(smpl, true); + + for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { + LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", + seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, + common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); + } + + const llama_token id = cur_p->data[0].id; + + common_sampler_accept(smpl, id, true); + + result.push_back(id); } + } else { + // greedily read the predicted block at this sequence's noise positions 1..n_block_tokens-1 + for (int32_t i = 1; i < n_block_tokens; ++i) { + common_sampler_sample(smpl, ctx_dft, beg + i, true); - const llama_token id = cur_p->data[0].id; + const auto * cur_p = common_sampler_get_candidates(smpl, true); - if (cur_p->data[0].p < params.p_min) { - break; + for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { + LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", + seq_id, k, i - 1, cur_p->data[k].id, cur_p->data[k].p, + common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); + } + + const llama_token id = cur_p->data[0].id; + + if (cur_p->data[0].p < params.p_min) { + break; + } + + common_sampler_accept(smpl, id, true); + + result.push_back(id); } - - common_sampler_accept(smpl, id, true); - - result.push_back(id); } if (result.size() < (size_t) params.n_min) { @@ -2145,6 +2190,7 @@ std::string common_speculative_type_to_str(common_speculative_type type) { case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: return "draft-eagle3"; case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: return "draft-mtp"; case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: return "draft-dflash"; + case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: return "draft-dspark"; case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram-simple"; case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: return "ngram-map-k"; case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram-map-k4v"; @@ -2198,6 +2244,7 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) { case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: + case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: n_max = std::max(n_max, std::max(0, spec->draft.n_max)); break; case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: @@ -2284,7 +2331,7 @@ common_speculative_init_result::common_speculative_init_result( std::string model_path; if (has_draft) { model_path = params.speculative.draft.mparams.path; - LOG_TRC("%s: loading draft model '%s'\n", __func__, model_path.c_str()); + LOG_INF("%s: loading draft model '%s'\n", __func__, model_path.c_str()); llama_model * model_dft = llama_model_load_from_file(params.model.path.c_str(), mparams); if (model_dft == NULL) { @@ -2304,7 +2351,7 @@ common_speculative_init_result::common_speculative_init_result( } else if (spec_mtp) { model_path = params.model.path; - LOG_TRC("%s: creating MTP draft context against the target model '%s'\n", __func__, model_path.c_str()); + LOG_INF("%s: creating MTP draft context against the target model '%s'\n", __func__, model_path.c_str()); llama_context * ctx_dft = llama_init_from_model(model_tgt, cparams); if (ctx_dft == nullptr) { @@ -2342,6 +2389,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, bool has_draft_eagle3 = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3)) && params.draft.ctx_dft != nullptr; bool has_draft_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; bool has_draft_dflash = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)) && params.draft.ctx_dft != nullptr; + bool has_draft_dspark = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)) && params.draft.ctx_dft != nullptr; @@ -2352,7 +2400,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD)); // when adding a new type - update here the logic above - static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 10); + static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 11); // this list here defines the priority of the speculators // the one with highest priority are listed first @@ -2385,6 +2433,9 @@ common_speculative * common_speculative_init(common_params_speculative & params, if (has_draft_dflash) { configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params)); } + if (has_draft_dspark) { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params)); + } } std::vector> impls = {}; @@ -2409,6 +2460,11 @@ common_speculative * common_speculative_init(common_params_speculative & params, impls.push_back(std::make_unique(config.params, n_seq)); break; } + case COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK: { + impls.push_back(std::make_unique( + config.params, n_seq, COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)); + break; + } case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: { common_ngram_map ngram_map = get_common_ngram_map(config.type, config.params.ngram_simple); diff --git a/conversion/__init__.py b/conversion/__init__.py index 5a8b46baf4..88e7bbe1ab 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -53,6 +53,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "DeepseekV3ForCausalLM": "deepseek", "DeepseekV32ForCausalLM": "deepseek", "DFlashDraftModel": "qwen", + "Qwen3DSparkModel": "qwen", "DeepseekV4ForCausalLM": "deepseek", "DistilBertForMaskedLM": "bert", "DistilBertForSequenceClassification": "bert", @@ -167,6 +168,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "ModernBertForMaskedLM": "bert", "ModernBertForSequenceClassification": "bert", "ModernBertModel": "bert", + "NanbeigeForCausalLM": "nanbeige", "NemotronForCausalLM": "nemotron", "NemotronHForCausalLM": "nemotron", "NeoBERT": "bert", @@ -289,6 +291,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "LlavaForConditionalGeneration": "llava", "MERaLiON2ForConditionalGeneration": "ultravox", "MiMoV2ForCausalLM": "mimo", + "MiniMaxM3SparseForConditionalGeneration": "minimax", "MiniCPMV4_6ForConditionalGeneration": "minicpm", "Mistral3ForConditionalGeneration": "llava", "NemotronH_Nano_VL_V2": "nemotron", diff --git a/conversion/glm.py b/conversion/glm.py index d85268a621..cc34cddbf8 100644 --- a/conversion/glm.py +++ b/conversion/glm.py @@ -1,6 +1,8 @@ from __future__ import annotations -from typing import Iterable, TYPE_CHECKING +import re + +from typing import Callable, Iterable, TYPE_CHECKING import torch @@ -213,12 +215,47 @@ class Glm4MoeLiteModel(DeepseekV2Model): class GlmMoeDsaModel(DeepseekV2Model): model_arch = gguf.MODEL_ARCH.GLM_DSA skip_mtp = False + supports_mtp_export = True + + # Trunk layer count, stashed before indexing so the classmethod + # filter_tensors can identify the appended NextN/MTP block (mirrors + # HYV3Model / Step35Model). + _n_main_layers: int | None = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0) + self.block_count = self.hparams["num_hidden_layers"] + if not self.no_mtp: + self.block_count += self.hparams.get("num_nextn_predict_layers", 0) self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + def index_tensors(self, remote_hf_model_id: str | None = None): + type(self)._n_main_layers = self.hparams["num_hidden_layers"] + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + if (titem := super().filter_tensors(item)) is None: + return None + name, gen = titem + + # GLM-5.2 appends the NextN/MTP block past num_hidden_layers + # (model.layers.78 -> blk.78 in the 79-block file). + assert cls._n_main_layers is not None + is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers + + # --no-mtp: drop the appended NextN block entirely. + if is_mtp and cls.no_mtp: + return None + # --mtp: keep ONLY NextN-block tensors plus the shared embeddings/ + # norm/lm_head (so the resulting GGUF carries just the draft head). + if cls.mtp_only and not is_mtp and name not in ( + "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", + ): + return None + + return name, gen + def set_vocab(self): return self._set_vocab_glm() @@ -230,7 +267,7 @@ class GlmMoeDsaModel(DeepseekV2Model): self.gguf_writer.add_rope_dimension_count(int(rope_dim * partial_rotary_factor)) # NextN/MTP prediction layers - if (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None: + if not self.no_mtp and (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None: self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers) # DSA indexer parameters diff --git a/conversion/llama.py b/conversion/llama.py index 315a619c9c..9b3373f911 100644 --- a/conversion/llama.py +++ b/conversion/llama.py @@ -69,9 +69,14 @@ class LlamaModel(TextModel): target_config = {**target_config, **target_config["text_config"]} self.target_vocab_size = target_config["vocab_size"] - # target_layers: derived from target model layer count (low/mid/high) + # target_layers: use the eagle3 config's explicit aux hidden-state layer ids + # if present, else derive from the target layer count. target_num_layers = target_config["num_hidden_layers"] - target_layers = [2, target_num_layers // 2, target_num_layers - 3] + aux_layer_ids = eagle3_raw_config.get("eagle_aux_hidden_state_layer_ids") + if aux_layer_ids: + target_layers = aux_layer_ids + else: + target_layers = [2, target_num_layers // 2, target_num_layers - 3] logger.info(f"EAGLE-3: target_layers = {target_layers} (target model has {target_num_layers} layers)") self.gguf_writer.add_target_layers(target_layers) @@ -90,6 +95,12 @@ class LlamaModel(TextModel): logger.info(f"EAGLE-3: norm_before_residual = {norm_before_residual}") self.gguf_writer.add_norm_before_residual(norm_before_residual) + # norm_before_fc: RMSNorm applied to the fused target features before the + # fc projection (e.g. nvidia/gpt-oss-120b-Eagle3-v3) + norm_before_fc = eagle3_raw_config.get("norm_before_fc", False) + logger.info(f"EAGLE-3: norm_before_fc = {norm_before_fc}") + self.gguf_writer.add_norm_before_fc(norm_before_fc) + def set_vocab(self): # eagle3: use tokenizer from target model if provided original_dir_model = None @@ -222,6 +233,9 @@ class LlamaModel(TextModel): if name == "fc.weight": yield (name, data_torch) return + if name == "input_norm.weight": + yield (self.format_tensor_name(gguf.MODEL_TENSOR.ENC_OUTPUT_NORM), data_torch) + return if name == "d2t": # store for manual int64 handling in prepare_tensors (avoid F32 conversion) if not hasattr(self, '_eagle3_int_tensors'): diff --git a/conversion/mimo.py b/conversion/mimo.py index 11ec286794..ca2ed28ad3 100644 --- a/conversion/mimo.py +++ b/conversion/mimo.py @@ -1,8 +1,9 @@ from __future__ import annotations +import json import re -from typing import Callable, TYPE_CHECKING +from typing import Any, Callable, Iterable, TYPE_CHECKING import torch @@ -229,7 +230,13 @@ class MimoV2Model(TextModel): @ModelBase.register("MiMoV2ForCausalLM") -class MiMoV2VisionModel(MmprojModel): +class MiMoV2VisionAudioModel(MmprojModel): + has_audio_encoder = True + + _audio_tok_hparams: dict[str, Any] | None = None + _rvq_codebook_sizes: list[int] | None = None + _code_embd: dict[int, Tensor] | None = None + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) assert self.hparams_vision is not None @@ -253,10 +260,22 @@ class MiMoV2VisionModel(MmprojModel): self.visual_token_window_size = int(hp.get("visual_token_window_size", -1)) self.use_sink = bool(hp.get("use_sink", False)) + def get_audio_config(self) -> dict[str, Any] | None: + if self._audio_tok_hparams is None: + path = self.dir_model / "audio_tokenizer" / "config.json" + with open(path, "r", encoding="utf-8") as f: + cfg = json.load(f) + # aliases so MmprojModel.find_aparam() / n_block_keys can resolve them + cfg["hidden_size"] = cfg["d_model"] + cfg["intermediate_size"] = cfg["encoder_ffn_dim"] + cfg["num_attention_heads"] = cfg["encoder_attention_heads"] + self._audio_tok_hparams = cfg + return self._audio_tok_hparams + def set_gguf_parameters(self): super().set_gguf_parameters() - self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MIMOVL) + self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.MIMOVL) self.gguf_writer.add_vision_use_silu(True) self.gguf_writer.add_vision_head_count_kv(self.num_kv_heads) self.gguf_writer.add_vision_spatial_merge_size(self.spatial_merge_size) @@ -266,19 +285,45 @@ class MiMoV2VisionModel(MmprojModel): self.gguf_writer.add_vision_min_pixels(int(self.preprocessor_config["min_pixels"])) self.gguf_writer.add_vision_max_pixels(int(self.preprocessor_config["max_pixels"])) + assert self.hparams_audio is not None + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.MIMO_AUDIO) + self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["n_mels"]) + self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams_audio.get("layer_norm_eps", 1e-5)) + + assert self._rvq_codebook_sizes is not None + self.gguf_writer.add_audio_rvq_num_quantizers(len(self._rvq_codebook_sizes)) + self.gguf_writer.add_audio_rvq_codebook_size(self._rvq_codebook_sizes) + + n_layer = self.hparams_audio["encoder_layers"] + swa_per_block = self.hparams_audio.get("swa_per_block", 1) + if self.hparams_audio.get("hybrid_attention") and swa_per_block > 1: + wa_pattern = [0 if i % swa_per_block < swa_per_block - 1 else -1 for i in range(n_layer)] + else: + wa_pattern = [-1] * n_layer + self.gguf_writer.add_audio_wa_pattern_mode(wa_pattern) + self.gguf_writer.add_audio_window_size(int(self.hparams_audio["encoder_attn_window_size"][0])) + + audio_cfg = self.global_config["audio_config"] + self.gguf_writer.add_audio_local_block_count(int(audio_cfg["input_local_layers"])) + self.gguf_writer.add_audio_local_group_size(int(audio_cfg["group_size"])) + def tensor_force_quant(self, name, new_name, bid, n_dims): - # Sinks must be F32: any sink-style softmax/mask add in ggml requires - # F32, and we fold sinks into a host-built F32 mask at encode time. - if new_name.endswith(".attn_sinks"): + # for audio encoder: keep codebook in F32 + if new_name in ( + gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_RVQ_CODEBOOK] + ".weight", + gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_MM_CODE_EMBD] + ".weight", + ): + return gguf.GGMLQuantizationType.F32 + if ("encoder.conv" in name or "encoder.down_sample_layer" in name) and name.endswith(".weight"): return gguf.GGMLQuantizationType.F32 return super().tensor_force_quant(name, new_name, bid, n_dims) @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, _ = item - if not name.startswith("visual."): - return None - return super().filter_tensors(item) + if name.startswith("visual.") or name.startswith("speech_embeddings.") or name.startswith("audio_encoder."): + return super().filter_tensors(item) + return None def modify_tensors(self, data_torch, name, bid): # Conv3D patch embed: split along the temporal axis (kt=2) into two Conv2D @@ -292,4 +337,64 @@ class MiMoV2VisionModel(MmprojModel): yield (embd_name + ".weight.1", data_torch[:, :, 1, ...]) return + if m := re.match(r"^speech_embeddings\.(\d+)\.weight$", name): + if self._code_embd is None: + self._code_embd = {} + self._code_embd[int(m.group(1))] = data_torch + + n_channels = int(self.global_config["audio_config"]["audio_channels"]) + if len(self._code_embd) < n_channels: + return + merged = torch.stack([self._code_embd.pop(i) for i in range(n_channels)], dim=0) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MM_CODE_EMBD), merged) + return + + if "conv1.bias" in name or "conv2.bias" in name: + # transpose conv1/conv2 bias so it broadcasts against [n_frames, C_out, 1] + data_torch = data_torch.unsqueeze(-1) + + if name == "audio_encoder.projection.mlp.0.weight": + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MMPROJ, 1), data_torch) + return + if name == "audio_encoder.projection.mlp.2.weight": + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_MMPROJ, 2), data_torch) + return + yield from super().modify_tensors(data_torch, name, bid) + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + # note: audio encoder is in its own subdir "audio_tokenizer" + from safetensors.torch import load_file + + tok_dir = self.dir_model / "audio_tokenizer" + state_dict = load_file(tok_dir / "model.safetensors") + + codebook_re = re.compile(r"^encoder\.quantizer\.vq\.layers\.(\d+)\._codebook\.embed$") + codebooks: dict[int, Tensor] = {} + + # EMA/training-only RVQ buffers - not needed for inference (nearest-codebook + # lookup only reads "_codebook.embed") + skip_suffixes = ( + "_codebook.cluster_size", + "_codebook.embed_avg", + "_codebook.inited", + ) + for name, tensor in state_dict.items(): + if name.endswith(skip_suffixes): + continue + if m := codebook_re.match(name): + codebooks[int(m.group(1))] = tensor + continue + yield name, tensor + + # gather codebooks and merge into 3D tensor, similar to MoE MLP tensors + n_q = len(codebooks) + ordered = [codebooks[i] for i in range(n_q)] + self._rvq_codebook_sizes = [int(cb.shape[0]) for cb in ordered] + max_bins = max(self._rvq_codebook_sizes) + dim = ordered[0].shape[1] + merged = ordered[0].new_zeros(n_q, max_bins, dim) + for i, cb in enumerate(ordered): + merged[i, : cb.shape[0], :] = cb + + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_RVQ_CODEBOOK), merged) diff --git a/conversion/minimax.py b/conversion/minimax.py index cbbdfe3ae8..c2175cc932 100644 --- a/conversion/minimax.py +++ b/conversion/minimax.py @@ -7,7 +7,7 @@ import torch if TYPE_CHECKING: from torch import Tensor -from .base import ModelBase, TextModel, gguf +from .base import ModelBase, TextModel, MmprojModel, gguf @ModelBase.register("MiniMaxM2ForCausalLM") @@ -58,6 +58,11 @@ class MiniMaxM2Model(TextModel): class MiniMaxM3Model(MiniMaxM2Model): model_arch = gguf.MODEL_ARCH.MINIMAXM3 + def tensor_force_quant(self, name, new_name, bid, n_dims): + if ".indexer." in new_name: + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) + def set_gguf_parameters(self): super().set_gguf_parameters() @@ -87,3 +92,78 @@ class MiniMaxM3Model(MiniMaxM2Model): data_torch = data_torch + 1.0 yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("MiniMaxM3SparseForConditionalGeneration", "MiniMaxM3VLForConditionalGeneration") +class MiniMaxM3VisionModel(MmprojModel): + @classmethod + def filter_tensors(cls, item): + name, gen = item + # keep only the vision-side tensors; text / mtp / sparse-index are dropped + if not name.startswith(("vision_tower.", "multi_modal_projector.", "patch_merge_mlp.")): + return None + return super().filter_tensors((name, gen)) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + assert self.hparams_vision is not None + + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINIMAXM3) + self.gguf_writer.add_vision_use_gelu(True) + + # the ViT carries its own LayerNorm eps (text tower uses a different one) + self.gguf_writer.add_vision_attention_layernorm_eps( + self.hparams_vision.get("layer_norm_eps", 1e-5) + ) + + comp = self.hparams_vision.get("img_token_compression_config", {}) + merge_size = comp.get("spatial_merge_size", 2) + self.gguf_writer.add_vision_spatial_merge_size(int(merge_size)) + + def modify_tensors(self, data_torch, name, bid): + assert self.hparams_vision is not None + + # Conv3d patch embed -> Conv2d slices + if name == "vision_tower.vision_model.embeddings.patch_embedding.weight": + if data_torch.ndim != 5: + raise ValueError(f"unexpected patch_embedding rank {data_torch.ndim} for {name}") + kt = data_torch.shape[2] + base = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.V_ENC_EMBD_PATCH] + for t in range(kt): + suffix = ".weight" if t == 0 else f".weight.{t}" + yield (base + suffix, data_torch[:, :, t, ...]) + return + + # Permute ViT q/k. HF [Ta Ha Wa | Tb Hb Wb | pad] reorder to [Ta Tb | Ha Hb | Wa Wb | pad]. + for new_name, tensor in super().modify_tensors(data_torch, name, bid): + if ".attn_q." in new_name or ".attn_k." in new_name: + tensor = self._permute_vit_qk(tensor, new_name) + yield new_name, tensor + + def _permute_vit_qk(self, t: "Tensor", new_name: str) -> "Tensor": + assert self.hparams_vision is not None + n_head = self.hparams_vision["num_attention_heads"] + d_head = t.shape[0] // n_head + axis_dim = 2 * ((2 * (d_head // 2) // 3) // 2) + ah = axis_dim // 2 + half = 3 * ah + perm = [] + perm += list(range(0, ah)) + perm += list(range(half, half + ah)) + perm += list(range(ah, 2 * ah)) + perm += list(range(half + ah, half + 2 * ah)) + perm += list(range(2 * ah, 3 * ah)) + perm += list(range(half + 2 * ah, half + 3 * ah)) + perm += list(range(2 * half, d_head)) + + assert axis_dim % 2 == 0 + assert 3 * axis_dim <= d_head + assert len(perm) == d_head + assert sorted(perm) == list(range(d_head)), "perm is not a bijection of d_head" + assert t.shape[0] == n_head * d_head, f"{new_name}: {t.shape[0]} != {n_head}*{d_head}" + assert d_head == 80 + + idx = torch.tensor(perm, dtype=torch.long) + if t.ndim == 2: + return t.reshape(n_head, d_head, t.shape[1])[:, idx, :].reshape(t.shape) + return t.reshape(n_head, d_head)[:, idx].reshape(t.shape) diff --git a/conversion/nanbeige.py b/conversion/nanbeige.py new file mode 100644 index 0000000000..f1fc425b3a --- /dev/null +++ b/conversion/nanbeige.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from .base import ModelBase, gguf, logger +from .llama import LlamaModel + + +@ModelBase.register("NanbeigeForCausalLM") +class NanbeigeModel(LlamaModel): + model_arch = gguf.MODEL_ARCH.NANBEIGE + undo_permute = True + + def set_gguf_parameters(self): + super().set_gguf_parameters() + hparams = self.hparams + + n_loops = int(hparams.get("num_loops", 1) or 1) + if n_loops < 1: + n_loops = 1 + self.gguf_writer.add_num_loops(n_loops) + logger.info(f"gguf: num_loops = {n_loops}") + + skip_loop_final_norm = bool(hparams.get("skip_loop_final_norm", False)) + self.gguf_writer.add_skip_loop_final_norm(skip_loop_final_norm) + logger.info(f"gguf: skip_loop_final_norm = {skip_loop_final_norm}") diff --git a/conversion/nemotron.py b/conversion/nemotron.py index e44688a788..0572b42ca2 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -39,28 +39,48 @@ class NemotronNanoV2VLModel(MmprojModel): } return vision_config + def get_audio_config(self) -> dict[str, Any] | None: + return self.global_config.get("sound_config") + def set_gguf_parameters(self): if "image_mean" not in self.preprocessor_config: self.preprocessor_config["image_mean"] = [0.485, 0.456, 0.406] if "image_std" not in self.preprocessor_config: self.preprocessor_config["image_std"] = [0.229, 0.224, 0.225] + if self.hparams_audio is not None: + self.has_vision_encoder = True + self.has_audio_encoder = True + self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["num_mel_bins"]) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) + self.gguf_writer.add_audio_subsampling_factor(self.hparams_audio["subsampling_factor"]) + self.gguf_writer.add_audio_conv_kernel_size(self.hparams_audio["conv_kernel_size"]) + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.PARAKEET) + self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.NEMOTRON_V2_VL) + else: + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_V2_VL) + super().set_gguf_parameters() hparams = self.global_config - self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_V2_VL) self.gguf_writer.add_vision_attention_layernorm_eps(1e-6) self.gguf_writer.add_vision_use_gelu(True) downsample_ratio = hparams.get("downsample_ratio", 0.5) self.gguf_writer.add_vision_projector_scale_factor(int(1.0 / downsample_ratio)) def tensor_force_quant(self, name, new_name, bid, n_dims): - if ".position_embd." in new_name or "pos_embed" in new_name: - return gguf.GGMLQuantizationType.F32 + if "sound_encoder" in name or new_name.startswith("mm.a."): + if "bias" in new_name or "norm" in new_name: + return gguf.GGMLQuantizationType.F32 + if "conv" in new_name and "weight" in new_name: + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, gen = item + if (titem := super().filter_tensors(item)) is None: + return None + name, gen = titem if "input_conditioner" in name: return None @@ -69,14 +89,18 @@ class NemotronNanoV2VLModel(MmprojModel): if "radio_model.model.patch_generator.video_embedder" in name: return None - if not name.startswith("vision_model.radio_model.model.") and not name.startswith("mlp1."): + if not name.startswith(("vision_model.radio_model.model.", "mlp1.", "sound_encoder.", "sound_projection.")): return None if "patch_generator.pos_embed" in name: if not name.endswith(".weight"): name += ".weight" - return super().filter_tensors((name, gen)) + # num_batches is only used for training not inference. + if "conv.norm" in name and "num_batches" in name: + return None + + return name, gen def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: # RADIO's pos_embed doesn't have .weight suffix, but clip.cpp expects it @@ -104,7 +128,26 @@ class NemotronNanoV2VLModel(MmprojModel): n_embd = self.hparams["hidden_size"] data_torch = data_torch.reshape(n_embd, 3, patch_size, patch_size) - yield from super().modify_tensors(data_torch, name, bid) + if "depthwise_conv.weight" in name: + data_torch = data_torch.unsqueeze(-1) + data_torch = data_torch.permute(3, 1, 0, 2).contiguous() + + if "pointwise_conv" in name and name.endswith(".weight"): + if len(data_torch.shape) == 3 and data_torch.shape[2] == 1: + data_torch = data_torch.reshape(data_torch.shape[0], data_torch.shape[1]) + + if "subsampling.layers" in name and name.endswith(".bias"): + if len(data_torch.shape) == 1: + data_torch = data_torch.reshape(1, -1, 1, 1) + + if "pointwise_conv" in name and name.endswith(".bias"): + if len(data_torch.shape) == 1: + data_torch = data_torch.reshape(1, -1, 1, 1) + + for mapped_name, tensor in super().modify_tensors(data_torch, name, bid): + if name.startswith("sound_projection.") and mapped_name.startswith("mm.model.mlp."): + mapped_name = mapped_name.replace("mm.model.mlp.", "mm.a.mlp.") + yield mapped_name, tensor @ModelBase.register("NemotronForCausalLM") diff --git a/conversion/qwen.py b/conversion/qwen.py index 9bc2b99fde..d1127f7431 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -688,3 +688,23 @@ class DFlashModel(Qwen3Model): if not name.startswith("model."): name = "model." + name return super().filter_tensors((name, gen)) + + +@ModelBase.register("Qwen3DSparkModel") +class DSparkModel(DFlashModel): + # DSpark = DFlash + a semi-autoregressive Markov head + model_arch = gguf.MODEL_ARCH.DFLASH + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # normalize the flat DeepSpec schema to DFlash's nested dflash_config + self.hparams.setdefault("dflash_config", { + k: self.hparams[k] for k in ("target_layer_ids", "mask_token_id") if k in self.hparams + }) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + if name.endswith(("embed_tokens.weight", "lm_head.weight")): + return None + return super().filter_tensors((name, gen)) diff --git a/conversion/qwenvl.py b/conversion/qwenvl.py index 7befd0c8d8..202a47961b 100644 --- a/conversion/qwenvl.py +++ b/conversion/qwenvl.py @@ -179,12 +179,12 @@ class Qwen25OmniModel(Qwen2VLVisionModel, Qwen25AudioModel): def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: name, gen = item - if not name.startswith("visual.") and not name.startswith("audio_tower."): - return None - if name.startswith("thinker."): name = name.replace("thinker.", "") + if not name.startswith("visual.") and not name.startswith("audio_tower."): + return None + if "audio_bos_eos_token" in name: # this tensor is left unused in transformers code # https://github.com/huggingface/transformers/blob/6e3063422c4b1c014aa60c32b9254fd2902f0f28/src/transformers/models/qwen2_5_omni/modular_qwen2_5_omni.py#L1809 diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index 0814ceb60f..814e541e1a 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -788,13 +788,18 @@ use 1 SYCL GPUs: [0] with Max compute units:512 | Name | Value | Function | |-------------------|------------------|---------------------------------------------------------------------------------------------------------------------------| | GGML_SYCL_DEBUG | 0 (default) or 1 | Enable log function by macro: GGML_SYCL_DEBUG | -| GGML_SYCL_DEV2DEV_MEMCPY | 0 (default) or 1 | Choose the SYCL or L0 API in dev2dev memory copy.
Value:
* 0: SYCL API (default)
* 1: L0 API -- L0 API is found to lead to abnormal crash in some case. This debug flag is used to check the issue.| +| GGML_SYCL_DEV2DEV_MEMCPY | 0 (default), 1, 2 | Choose the method of dev2dev memory copy.
Value:
* 0: SYCL API (default), only support dGPUs.
* 1: L0 API -- Better performance, only support dGPUs, found to lead to abnormal crash in some case.
* 2: Host Forward -- Most stable method for all cases (including iGPU + dGPU*N), but with lower performance (-2% to -5%).
SYCL & L0 API are easy to be impacted by Intel GPU driver issue. When you meet the garbled output or crash issues in multiple GPUs case, try with this debug flag to work around or check the issue.| | GGML_SYCL_ENABLE_FLASH_ATTN | 1 (default) or 0| Enable Flash-Attention. It can reduce memory usage. The performance impact depends on the LLM.| | GGML_SYCL_ENABLE_OPT | 0 or 1 (default)| Enable optimize features for Intel GPUs. (Recommended to 0 for Intel devices older than Gen 10) | | GGML_SYCL_ENABLE_GRAPH | 0 (default) or 1 | Enable running computations through SYCL Graphs feature. Disabled by default because SYCL Graph is still on development, no better performance. | | GGML_SYCL_USE_LEVEL_ZERO_API | 1 (default) or 0 | Use Level Zero API for device memory allocation instead of SYCL. Reduces system RAM usage on Intel dGPUs by avoiding DMA-buf/TTM host memory staging. Requires GGML_SYCL_SUPPORT_LEVEL_ZERO_API=ON at build time. SYCL backend always runs on Level Zero running time even if it's set as OFF (The SYCL api will be usage for memory allocation).| | GGML_SYCL_ENABLE_DNN | 0 or 1 (default)| Enable running computations through oneDNN and always use oneMKL. | +| GGML_SYCL_FA_ONEDNN | 1 (default) or 0 | Enable the oneDNN fused SDPA (flash-attention) path on supported GPUs. Set to 0 to always use the native SYCL flash-attention kernel. | +| GGML_SYCL_FA_ONEDNN_MAX_KV | 0 (default, disabled) or positive integer | By default (0), all sequences are handled by the oneDNN fused SDPA path, regardless of KV length; a positive value caps that length, past which sequences fall back to the native kernel. If GPU driver watchdog resets (DEVICE_LOST) occur during long-context inference, set this near the context depth where they start, e.g. 24576. | | GGML_SYCL_ENABLE_VMM | 0 or 1 (default) | Enable the virtual-memory device pool. | +| GGML_SYCL_ENABLE_MKL_FA | 1 (default) or 0 | Enable oneMKL GEMM flash attention for XMX-accelerated prompt processing with quantized KV cache. Automatically activates during prefill (prompt processing) when all conditions are met: (1) flash-attn enabled (`-fa` or `--flash-attn on`), (2) KV cache quantized (`--cache-type-k q8_0 --cache-type-v q8_0` or other `*_0/*_1` types), (3) batch size ≥ 1024 (`--batch-size 1024`), (4) prompt length ≥ 1024 tokens. Set to 0 to force the TILE kernel for A/B testing. Example minimum command: `llama-cli -m model.gguf -fa -ngl 99 --cache-type-k q8_0 --cache-type-v q8_0 --batch-size 1024 -p "your prompt"` | +| GGML_SYCL_MKL_FA_DEBUG | 0 (default) or 1 | Enable per-call diagnostic logging for MKL flash attention: GEMM/softmax timings, interleaved-head detection, and buffer memory usage. | +| GGML_SYCL_MKL_FA_DIAG | 0 (default) or 1 | Enable output fingerprinting for MKL flash attention. Dumps the first 64 float output values for the first 6 FA calls with n_kv ≥ 1024, labeled with kernel type (MKL/TILE/VEC) for cross-kernel comparison. | | GGML_SYCL_ENABLE_FUSION | 0 or 1 (default) | Enable fused-kernel dispatch in graph compute (currently top-k MoE gating). | | ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.
Recommended to use when --split-mode = layer | | UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS | 0 (default) or 1 | Allow SYCL/Unified Runtime Level Zero device allocations larger than 4 GiB. llama.cpp's direct Level Zero allocation path requests the relaxed maximum-size limit itself when GGML_SYCL_ENABLE_LEVEL_ZERO=1. | diff --git a/docs/completions.md b/docs/completions.md new file mode 100644 index 0000000000..5376644c0a --- /dev/null +++ b/docs/completions.md @@ -0,0 +1,17 @@ +# Completions + +Command-line completion is available for some environments. + +## Bash Completion + +```bash +$ build/bin/llama-cli --completion-bash > ~/.llama-completion.bash +$ source ~/.llama-completion.bash +``` + +Optionally this can be added to your `.bashrc` or `.bash_profile` to load it +automatically. For example: + +```console +$ echo "source ~/.llama-completion.bash" >> ~/.bashrc +``` diff --git a/docs/development/HOWTO-add-model.md b/docs/development/HOWTO-add-model.md index 632e79881a..102f479eb0 100644 --- a/docs/development/HOWTO-add-model.md +++ b/docs/development/HOWTO-add-model.md @@ -144,6 +144,8 @@ Examples: - Gemma 3 folds the `1 +` of its `norm(1 + weight)` normalization into the weights at conversion time, so the graph just does a plain RMS norm. - Qwen3-Next applies its tensor permutation during conversion (in `modify_tensors`), so the graph can consume the already-permuted weights directly. +Exception: a plain `weight * scale` with a constant scale is usually better left to inference time rather than folded into the weight at conversion. The scale conceptually applies to the activation, not the weight, so folding it into the weight can hurt numerical stability, and it shifts the weight's value range in a way that can make quantization worse. In this case, write the scale to GGUF as its own metadata key (e.g. `%s.attention.output_scale`, `%s.attention.value_scale`, `%s.embedding_scale`) and apply it in the graph, instead of pre-multiplying the weight tensor during conversion. + ### Working with ggml_rope_ext PyTorch implementations usually prefer explicitly calculating `freq_cis`/`sin`/`cos` components. However, in llama.cpp, most RoPE operations can be handled via `ggml_rope_ext`, which does not require a sin/cos matrix. This saves memory while allowing the GGML RoPE kernel to be fused with other ops. diff --git a/docs/install.md b/docs/install.md index 7198e61bf3..b36b0be267 100644 --- a/docs/install.md +++ b/docs/install.md @@ -16,22 +16,22 @@ conda-forge provides builds for: - Apple Metal (macOS) ```sh -conda install -c conda-forge llama-cpp +conda install -c conda-forge llama.cpp ``` ```sh -mamba install -c conda-forge llama-cpp +mamba install -c conda-forge llama.cpp ``` ```sh # Project-local installation -pixi add llama-cpp +pixi add llama.cpp # Global installation -pixi global install llama-cpp +pixi global install llama.cpp ``` -This distribution is managed on [`conda-forge/llama-cpp-feedstock`](https://github.com/conda-forge/llama.cpp-feedstock/). +This distribution is managed on [`conda-forge/llama.cpp-feedstock`](https://github.com/conda-forge/llama.cpp-feedstock/). Shall you have any problems, please open an issue on [its issue tracker](https://github.com/conda-forge/llama.cpp-feedstock/issues). diff --git a/docs/models.md b/docs/models.md new file mode 100644 index 0000000000..eee5952634 --- /dev/null +++ b/docs/models.md @@ -0,0 +1,26 @@ +# Obtaining and quantizing models + +The [Hugging Face](https://huggingface.co) platform hosts [thousands of models](https://huggingface.co/models?library=gguf&sort=trending) compatible with `llama.cpp`: + +- [Trending](https://huggingface.co/models?library=gguf&sort=trending) + +You can use any `llama.cpp`-compatible model from [Hugging Face](https://huggingface.co/) using this CLI argument: `-hf /[:quant]`. For example: + +```sh +llama cli -hf ggml-org/gemma-3-1b-it-GGUF +``` + +You can use the same CLI invocation to download from other sites, by pointing the `MODEL_ENDPOINT` environment variable to an endpoint compatible with the Hugging Face API. +`llama.cpp` can also run models you have downloaded locally to your filesystem. + +After downloading a model, use the CLI tools to run it locally - see below. + +`llama.cpp` requires the model to be stored in the [GGUF](https://github.com/ggml-org/ggml/blob/master/docs/gguf.md) file format. Models in other data formats can be converted to GGUF using the `convert_*.py` Python scripts in this repo. +To learn more about model quantization, [read this documentation](../tools/quantize/README.md) + +The Hugging Face platform provides a variety of online tools for converting, quantizing and hosting models with `llama.cpp`: + +- Use the [GGUF-my-repo space](https://huggingface.co/spaces/ggml-org/gguf-my-repo) to convert to GGUF format and quantize model weights to smaller sizes +- Use the [GGUF-my-LoRA space](https://huggingface.co/spaces/ggml-org/gguf-my-lora) to convert LoRA adapters to GGUF format (more info: https://github.com/ggml-org/llama.cpp/discussions/10123) +- Use the [GGUF-editor space](https://huggingface.co/spaces/CISCai/gguf-editor) to edit GGUF meta data in the browser (more info: https://github.com/ggml-org/llama.cpp/discussions/9268) +- Use the [Inference Endpoints](https://ui.endpoints.huggingface.co/) to directly host `llama.cpp` in the cloud (more info: https://github.com/ggml-org/llama.cpp/discussions/9669) diff --git a/docs/speculative.md b/docs/speculative.md index 4100b92f8f..3957db85c9 100644 --- a/docs/speculative.md +++ b/docs/speculative.md @@ -78,6 +78,38 @@ See: - #22105 +### DSpark (`draft-dspark`) + +DSpark extends DFlash with a semi-autoregressive _Markov head_: the draft still emits a whole +block per forward pass, but each block position's logits are biased by a low-rank term keyed on +the previous token, chained in-graph across the block. This keeps drafting at one decode per +block while recovering some of the left-to-right signal that pure block diffusion loses. + +The draft is a small DeepSpec checkpoint trained for a specific target (for example +[`deepseek-ai/dspark_qwen3_4b_block7`](https://huggingface.co/deepseek-ai/dspark_qwen3_4b_block7) +for `Qwen/Qwen3-4B`). Convert it with `--target-model-dir` so it inherits the target's tokenizer +and token embeddings: + +```bash +python convert_hf_to_gguf.py deepseek-ai/dspark_qwen3_4b_block7 \ + --target-model-dir Qwen/Qwen3-4B --outtype bf16 --outfile Qwen3-4B-DSpark.gguf + +llama-server -m Qwen3-4B.gguf -md Qwen3-4B-DSpark.gguf \ + --spec-type draft-dspark --spec-draft-n-max 7 -fa on --jinja +``` + +`--spec-draft-n-max` is clamped to the draft model's trained block size. + +`--spec-draft-conf-min P` truncates each drafted block at the first position whose predicted +acceptance (from the draft's confidence head, if present) falls below `P` (default 0 = disabled). + +Currently only drafts with a Qwen3 backbone are supported; support for other backbones +(e.g. Gemma4) is planned. + +See: + +- #25173 + ### n-gram Cache (`ngram-cache`) An n-gram is a sequence of n tokens. The n-gram cache implementation maintains statistics about short n-gram sequences. @@ -173,7 +205,7 @@ If a draft model is combined with a draftless decoding the draftless decoding ha ### General Speculative Parameters ``` ---spec-type [none|draft-simple|draft-eagle3|draft-dflash|draft-mtp|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod] +--spec-type [none|draft-simple|draft-eagle3|draft-dflash|draft-dspark|draft-mtp|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod] comma-separated list of types of speculative decoding to use (default: none) (env: LLAMA_ARG_SPEC_TYPE) @@ -314,6 +346,7 @@ Specifies a comma-separated list of speculative decoding types to use. | `draft-simple` | Use a simple draft model for speculation | | `draft-eagle3` | Use an EAGLE-3 draft model that reads the target's hidden states | | `draft-dflash` | Use a DFlash block-diffusion draft model that emits a block per step | +| `draft-dspark` | Use a DSpark draft model (DFlash backbone + semi-autoregressive Markov head) | | `draft-mtp` | Use Multi Token Prediction (MTP) heads from the main model | | `ngram-cache` | Use n-gram cache lookup | | `ngram-simple` | Use simple n-gram pattern matching | diff --git a/docs/xcframework.md b/docs/xcframework.md new file mode 100644 index 0000000000..83c8b50044 --- /dev/null +++ b/docs/xcframework.md @@ -0,0 +1,31 @@ +# XCFramework + +The XCFramework is a precompiled version of the library for iOS, visionOS, tvOS, +and macOS. It can be used in Swift projects without the need to compile the +library from source. For example: + +```swift +// swift-tools-version: 5.10 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +import PackageDescription + +let package = Package( + name: "MyLlamaPackage", + targets: [ + .executableTarget( + name: "MyLlamaPackage", + dependencies: [ + "LlamaFramework" + ]), + .binaryTarget( + name: "LlamaFramework", + url: "https://github.com/ggml-org/llama.cpp/releases/download/b5046/llama-b5046-xcframework.zip", + checksum: "c19be78b5f00d8d29a25da41042cb7afa094cbf6280a225abe614b03b20029ab" + ) + ] +) +``` + +The above example is using an intermediate build `b5046` of the library. This can be modified +to use a different version by changing the URL and checksum. diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index a766e49ea1..159da3afa0 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -4,7 +4,7 @@ project("ggml" C CXX ASM) ### GGML Version set(GGML_VERSION_MAJOR 0) -set(GGML_VERSION_MINOR 17) +set(GGML_VERSION_MINOR 18) set(GGML_VERSION_PATCH 0) set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}") diff --git a/ggml/include/ggml-rpc.h b/ggml/include/ggml-rpc.h index 16ca33947a..276aea00ea 100644 --- a/ggml/include/ggml-rpc.h +++ b/ggml/include/ggml-rpc.h @@ -6,9 +6,9 @@ extern "C" { #endif -#define RPC_PROTO_MAJOR_VERSION 4 +#define RPC_PROTO_MAJOR_VERSION 5 #define RPC_PROTO_MINOR_VERSION 0 -#define RPC_PROTO_PATCH_VERSION 3 +#define RPC_PROTO_PATCH_VERSION 0 #ifdef __cplusplus static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION"); diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 87615921c0..7f4e252dca 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -906,26 +906,35 @@ static int ggml_backend_sched_backend_id_from_cur(ggml_backend_sched_t sched, st } // operations with weights are preferably run on the same backend as the weights - for (int i = 0; i < GGML_MAX_SRC; i++) { - const struct ggml_tensor * src = tensor->src[i]; - if (src == NULL) { - continue; - } - // skip ROPE since the rope freqs tensor is too small to choose a backend based on it - // not an ideal solution - if (tensor->op != GGML_OP_ROPE && src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { - int src_backend_id = ggml_backend_sched_backend_from_buffer(sched, src, tensor); - // check if a backend with higher prio wants to offload the op - if (sched->op_offload && src_backend_id == sched->n_backends - 1 && ggml_backend_buffer_is_host(src->buffer)) { - for (int b = 0; b < src_backend_id; b++) { - if (ggml_backend_supports_op(sched->backends[b], tensor) && ggml_backend_offload_op(sched->backends[b], tensor)) { - SET_CAUSE(tensor, "1.off"); - return b; + // TODO: there are exceptions (see below) - not an ideal solution + bool allow = true; + + // skip ROPE since the rope freqs tensor is too small to choose a backend based on it + allow = allow && tensor->op != GGML_OP_ROPE; + + // skip FLASH_ATTN_EXT since the sinks tensor is too small to choose a based based on it + allow = allow && tensor->op != GGML_OP_FLASH_ATTN_EXT; + + if (allow) { + for (int i = 0; i < GGML_MAX_SRC; i++) { + const struct ggml_tensor * src = tensor->src[i]; + if (src == NULL) { + continue; + } + if (src->buffer != NULL && src->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) { + int src_backend_id = ggml_backend_sched_backend_from_buffer(sched, src, tensor); + // check if a backend with higher prio wants to offload the op + if (sched->op_offload && src_backend_id == sched->n_backends - 1 && ggml_backend_buffer_is_host(src->buffer)) { + for (int b = 0; b < src_backend_id; b++) { + if (ggml_backend_supports_op(sched->backends[b], tensor) && ggml_backend_offload_op(sched->backends[b], tensor)) { + SET_CAUSE(tensor, "1.off"); + return b; + } } } + SET_CAUSE(tensor, "1.wgt%d", i); + return src_backend_id; } - SET_CAUSE(tensor, "1.wgt%d", i); - return src_backend_id; } } diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 74631c2857..16cc5116c5 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -469,6 +469,8 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st return (src0->type == GGML_TYPE_F32 || ((src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type)) && src0->ne[2] == src1->ne[2] && src0->ne[3] == src1->ne[3])) && src1->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32; + case GGML_OP_CONV_2D: + return ggml_is_contiguous(op->src[0]); default: return true; } diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index 23bcd54c12..99b7d5afa2 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -1797,14 +1797,6 @@ class tinyBLAS_Q0_AVX { //PPC Implementation #if defined(__MMA__) -#define SAVE_ACC(ACC, ii, jj) \ - __builtin_mma_disassemble_acc(vec_C, ACC); \ - for (int I = 0; I < 4; I++) { \ - for (int J = 0; J < 4; J++) { \ - *((float*)(C+ii+((jj+J)*ldc)+I)) = *((float*)&vec_C[I]+J); \ - } \ - } \ - template struct mma_instr; @@ -1834,10 +1826,49 @@ class tinyBLAS_HP16_PPC { } void matmul(int64_t m, int64_t n) { - mnpack(0, m, 0, n); + int64_t mc = 256; + int64_t nc = 256; + int64_t kc = 256; + #if defined(_AIX) || defined(__BIG_ENDIAN__) + mc = 128; + nc = 128; + kc = 128; + #endif + if (k < kc) { + kc = k; + } + bool can_use_tiled = (m % mc == 0) && (n % nc == 0) && (k % kc == 0); + if (can_use_tiled) { + matmul_tiled(m, n, mc, nc, kc); + } else { + mnpack(0, m, 0, n); + } } private: + __attribute__((always_inline)) + inline void save_acc(acc_t * ACC, int64_t ii, int64_t jj) { + vec_t vec_C[4]; + __builtin_mma_disassemble_acc(vec_C, ACC); + for (int I = 0; I < 4; I++) { + for (int J = 0; J < 4; J++) { + *((float *)(C+ii+((jj+J)*ldc)+I)) = *((float *)&vec_C[I]+J); + } + } + } + + __attribute__((always_inline)) + inline void add_save_acc(acc_t * ACC, int64_t ii, int64_t jj) { + vec_t vec_C[4]; + __builtin_mma_disassemble_acc(vec_C, ACC); + for (int I = 0; I < 4; I++) { + for (int J = 0; J < 4; J++) { + float * c_ptr = (float *)(C+ii+((jj+J)*ldc)+I); + *c_ptr += *((float *)&vec_C[I]+J); + } + } + } + void vector_permute_store(vec_t *c, int numVec, unsigned char *vecOffset) { vec_t t[8], s[8]; vec_t swiz1 = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23}; @@ -1896,6 +1927,7 @@ class tinyBLAS_HP16_PPC { j = (rows >> 3); if (j > 0) { do { + aoffsets[0] = aoffset; if (cols == 4) { aoffsets[0] = aoffset; for (int it = 1; it < 4; ++it) @@ -1910,17 +1942,17 @@ class tinyBLAS_HP16_PPC { } i = (cols >> 3); if (i > 0) { - aoffsets[0] = aoffset; for (int it = 1; it < 8; ++it) { aoffsets[it] = aoffsets[it-1] + lda; } aoffset += 8 * lda; + do { for (int it = 0; it < 8; ++it) c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]); vector_permute_store(c_arr, 8, vecOffset); for (int it = 0; it < 8; ++it) - aoffsets[it] = aoffsets[it] + 8*lda; + aoffsets[it] = aoffsets[it] + 8; vecOffset += 128; i--; } while(i > 0); @@ -2147,8 +2179,8 @@ class tinyBLAS_HP16_PPC { mma_instr::outer_product(&acc_1, vec_A[x], vec_B[x+4]); } } - SAVE_ACC(&acc_0, ii, jj); - SAVE_ACC(&acc_1, ii, jj+4); + save_acc(&acc_0, ii, jj); + save_acc(&acc_1, ii, jj+4); } void KERNEL_8x4(int64_t ii, int64_t jj) { @@ -2164,8 +2196,8 @@ class tinyBLAS_HP16_PPC { mma_instr::outer_product(&acc_1, vec_A[x+4], vec_B[x]); } } - SAVE_ACC(&acc_0, ii, jj); - SAVE_ACC(&acc_1, ii+4, jj); + save_acc(&acc_0, ii, jj); + save_acc(&acc_1, ii+4, jj); } @@ -2186,13 +2218,64 @@ class tinyBLAS_HP16_PPC { mma_instr::outer_product(&acc_3, vec_A[x+4], vec_B[x+4]); } } - - SAVE_ACC(&acc_0, ii, jj); - SAVE_ACC(&acc_1, ii, jj+4); - SAVE_ACC(&acc_2, ii+4, jj); - SAVE_ACC(&acc_3, ii+4, jj+4); + save_acc(&acc_0, ii, jj); + save_acc(&acc_1, ii, jj+4); + save_acc(&acc_2, ii+4, jj); + save_acc(&acc_3, ii+4, jj+4); } + inline void MMA_16x8(vec_t * vec_A0, vec_t * vec_A1, vec_t * vec_B, acc_t * acc) { + for (int x = 0; x < 4; x ++) { + mma_instr::outer_product(&acc[0], vec_A0[x], vec_B[x]); + mma_instr::outer_product(&acc[1], vec_A0[x], vec_B[x+4]); + mma_instr::outer_product(&acc[2], vec_A0[x+4], vec_B[x]); + mma_instr::outer_product(&acc[3], vec_A0[x+4], vec_B[x+4]); + mma_instr::outer_product(&acc[4], vec_A1[x], vec_B[x]); + mma_instr::outer_product(&acc[5], vec_A1[x], vec_B[x+4]); + mma_instr::outer_product(&acc[6], vec_A1[x+4], vec_B[x]); + mma_instr::outer_product(&acc[7], vec_A1[x+4], vec_B[x+4]); + } + } + void KERNEL(int64_t ii, int64_t jj, int64_t mc, int64_t nc, int64_t kc, vec_t * vec_A, vec_t * vec_B, int64_t kk) { + for (int64_t i = 0; i < mc; i += 16) { + int A_base_addr = (mc / 8) * (i / 8) * 8; + for (int64_t j = 0; j < nc; j += 8) { + int B_base_addr = (nc / 8) * (j / 8) * 8; + acc_t acc[8]; + vec_t A0_block[8]; vec_t A1_block[8]; + for (int x = 0; x < 8; x++) + __builtin_mma_xxsetaccz(&acc[x]); + for (int64_t l = 0; l < kc; l += 8) { + int A0_block_idx = A_base_addr + (l / 8) * 8; + int A1_block_idx = A0_block_idx + (mc / 8) * 8; + int B_block_idx = B_base_addr + (l / 8) * 8; + vec_t* A0_block = &vec_A[A0_block_idx]; + vec_t* A1_block = &vec_A[A1_block_idx]; + vec_t* B_block = &vec_B[B_block_idx]; + MMA_16x8(A0_block, A1_block, B_block, acc); + } + if (kk == 0) { + save_acc(&acc[0], ii + i, jj + j); + save_acc(&acc[1], ii + i, jj + j + 4); + save_acc(&acc[2], ii + i + 4, jj + j); + save_acc(&acc[3], ii + i + 4, jj + j + 4); + save_acc(&acc[4], ii + i + 8, jj + j); + save_acc(&acc[5], ii + i + 8, jj + j + 4); + save_acc(&acc[6], ii + i + 12, jj + j); + save_acc(&acc[7], ii + i + 12, jj + j + 4); + } else { + add_save_acc(&acc[0], ii + i, jj + j); + add_save_acc(&acc[1], ii + i, jj + j + 4); + add_save_acc(&acc[2], ii + i + 4, jj + j); + add_save_acc(&acc[3], ii + i + 4, jj + j + 4); + add_save_acc(&acc[4], ii + i + 8, jj + j); + add_save_acc(&acc[5], ii + i + 8, jj + j + 4); + add_save_acc(&acc[6], ii + i + 12, jj + j); + add_save_acc(&acc[7], ii + i + 12, jj + j + 4); + } + } + } + } template void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n) { int64_t ytiles = (m - m0) / RM; @@ -2281,6 +2364,29 @@ class tinyBLAS_HP16_PPC { } } + void matmul_tiled(int64_t m, int64_t n, int64_t mc, int64_t nc, int64_t kc) { + int64_t ytiles = m / mc; + int64_t xtiles = n / nc; + int64_t tiles = xtiles * ytiles; + int64_t duty = (tiles + nth - 1) / nth; + int64_t start = duty * ith; + int64_t end = start + duty; + if (end > tiles) { + end = tiles; + } + for (int64_t job = start; job < end; ++job) { + int64_t ii = (job / xtiles) * mc; + int64_t jj = (job % xtiles) * nc; + for (int64_t kk = 0; kk < k; kk += kc) { + vec_t A_pack[kc * mc / 8]; + vec_t B_pack[kc * nc / 8]; + packNormal(A + (ii * lda) + kk, lda, kc, mc, (uint8_t *)A_pack); + packNormal(B + (jj * ldb) + kk, ldb, kc, nc, (uint8_t *)B_pack); + KERNEL(ii, jj, mc, nc, kc, A_pack, B_pack, kk); + } + } + } + template NOINLINE void gemm(int64_t m0, int64_t m, int64_t n0, int64_t n) { int64_t ytiles = (m - m0) / RM; diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp index f18758f16b..9689ca3ced 100644 --- a/ggml/src/ggml-cpu/repack.cpp +++ b/ggml/src/ggml-cpu/repack.cpp @@ -2739,7 +2739,7 @@ static block_q8_0x4 make_block_q8_0x4(block_q8_0 * in, unsigned int blck_size_in return out; } -static block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, unsigned int blck_size_interleave) { +static block_q4_0x4 make_block_q4_0x4(block_q4_0 * in, int blck_size_interleave) { block_q4_0x4 out; for (int i = 0; i < 4; i++) { diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index fa2dd26202..33be16dc5c 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -977,6 +977,13 @@ struct ggml_cuda_type_traits { static constexpr int qi = QI1_0; }; +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK2_0; + static constexpr int qr = QR2_0; + static constexpr int qi = QI2_0; +}; + template<> struct ggml_cuda_type_traits { static constexpr int qk = QK4_0; diff --git a/ggml/src/ggml-cuda/conv2d.cu b/ggml/src/ggml-cuda/conv2d.cu index 142dd66903..14774d4a5e 100644 --- a/ggml/src/ggml-cuda/conv2d.cu +++ b/ggml/src/ggml-cuda/conv2d.cu @@ -126,6 +126,7 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const float * X_D = (const float *) input->data; float * Y_D = (float *) dst->data; + GGML_ASSERT(ggml_is_contiguous(input)); GGML_ASSERT(ggml_is_contiguous(kernel)); GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32); diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index 946e02af50..360c614a44 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -459,6 +459,8 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return dequantize_block_cont_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cont_cuda; case GGML_TYPE_Q4_0: return dequantize_row_q4_0_cuda; case GGML_TYPE_Q4_1: @@ -514,6 +516,8 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return dequantize_block_cont_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cont_cuda; case GGML_TYPE_Q4_0: return dequantize_row_q4_0_cuda; case GGML_TYPE_Q4_1: @@ -572,6 +576,8 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return dequantize_block_cont_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cont_cuda; case GGML_TYPE_Q4_0: return dequantize_row_q4_0_cuda; case GGML_TYPE_Q4_1: @@ -629,6 +635,8 @@ to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) { return convert_unary_cuda; case GGML_TYPE_Q1_0: return dequantize_block_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cuda; case GGML_TYPE_Q4_0: return dequantize_block_cuda; case GGML_TYPE_Q4_1: @@ -652,6 +660,8 @@ to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type) { return convert_unary_cuda; case GGML_TYPE_Q1_0: return dequantize_block_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cuda; case GGML_TYPE_Q4_0: return dequantize_block_cuda; case GGML_TYPE_Q4_1: @@ -675,6 +685,8 @@ to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) { return convert_unary_cuda; case GGML_TYPE_Q1_0: return dequantize_block_cuda; + case GGML_TYPE_Q2_0: + return dequantize_block_cuda; case GGML_TYPE_Q4_0: return dequantize_block_cuda; case GGML_TYPE_Q4_1: diff --git a/ggml/src/ggml-cuda/dequantize.cuh b/ggml/src/ggml-cuda/dequantize.cuh index 8ab5ad8e74..651524be7b 100644 --- a/ggml/src/ggml-cuda/dequantize.cuh +++ b/ggml/src/ggml-cuda/dequantize.cuh @@ -23,6 +23,26 @@ static __device__ __forceinline__ void dequantize_q1_0(const void * vx, const in v.y = (2*bit_1 - 1) * d; } +static __device__ __forceinline__ void dequantize_q2_0(const void * vx, const int64_t ib, const int iqs, float2 & v){ + const block_q2_0 * x = (const block_q2_0 *) vx; + + const float d = x[ib].d; + + // Q2_0: 2 bits per element, 4 elements per byte. + // Stored code c in {0,1,2,3} maps to symbol s = c - 1 in {-1, 0, +1, +2}. + const int byte_index_0 = iqs / 4; + const int bit_offset_0 = (iqs % 4) * 2; + + const int byte_index_1 = (iqs + 1) / 4; + const int bit_offset_1 = ((iqs + 1) % 4) * 2; + + const int c0 = (x[ib].qs[byte_index_0] >> bit_offset_0) & 0x3; + const int c1 = (x[ib].qs[byte_index_1] >> bit_offset_1) & 0x3; + + v.x = (c0 - 1) * d; + v.y = (c1 - 1) * d; +} + static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, float2 & v){ const block_q4_0 * x = (const block_q4_0 *) vx; diff --git a/ggml/src/ggml-cuda/getrows.cu b/ggml/src/ggml-cuda/getrows.cu index a9ec4f697c..6b36b8fbc1 100644 --- a/ggml/src/ggml-cuda/getrows.cu +++ b/ggml/src/ggml-cuda/getrows.cu @@ -320,6 +320,10 @@ static void ggml_cuda_get_rows_switch_src0_type( get_rows_cuda_q(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); break; + case GGML_TYPE_Q2_0: + get_rows_cuda_q(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; case GGML_TYPE_Q4_0: get_rows_cuda_q(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index e73a7b8906..561ab7ac59 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -1836,6 +1836,20 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst); return; } + // A transposed vector can still use MMVQ (i.e. ne01 == 1) + if (ne01 == 1 && ne11 > MMVF_MAX_BATCH_SIZE && ne2 == 1 && ne3 == 1 + && src0->type == GGML_TYPE_F32 + && ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst) + && ggml_cuda_should_use_mmvf(src1->type, cc, src1->ne, src1->nb, /*ne11 =*/ 1)) { + ggml_tensor dst_vec = *dst; + dst_vec.ne[0] = ne11; + dst_vec.ne[1] = 1; + dst_vec.nb[1] = dst_vec.nb[0]*ne11; + dst_vec.nb[2] = dst_vec.nb[1]; + dst_vec.nb[3] = dst_vec.nb[1]; + ggml_cuda_mul_mat_vec_f(ctx, src1, src0, nullptr, &dst_vec); + return; + } if (ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11, /*mul_mat_id =*/ false)) { ggml_cuda_mul_mat_f(ctx, src0, src1, nullptr, dst); return; @@ -4802,6 +4816,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_F32: case GGML_TYPE_F16: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4840,6 +4855,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_BF16: case GGML_TYPE_I32: case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -5089,7 +5105,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_IM2COL: case GGML_OP_IM2COL_3D: case GGML_OP_CONV_2D: - return true; + return (ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1])); case GGML_OP_CONV_2D_DW: return op->src[0]->type == GGML_TYPE_F32; case GGML_OP_CONV_TRANSPOSE_2D: diff --git a/ggml/src/ggml-cuda/mmq-config-ampere.cuh b/ggml/src/ggml-cuda/mmq-config-ampere.cuh index 0037bac3d0..9f9fd19738 100644 --- a/ggml/src/ggml-cuda/mmq-config-ampere.cuh +++ b/ggml/src/ggml-cuda/mmq-config-ampere.cuh @@ -16,6 +16,23 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_Q1_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); CASE(GGML_TYPE_Q1_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); CASE(GGML_TYPE_Q4_0, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); CASE(GGML_TYPE_Q4_0, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); diff --git a/ggml/src/ggml-cuda/mmq-config-cdna.cuh b/ggml/src/ggml-cuda/mmq-config-cdna.cuh index 46ec6aa9d5..4a8d89f720 100644 --- a/ggml/src/ggml-cuda/mmq-config-cdna.cuh +++ b/ggml/src/ggml-cuda/mmq-config-cdna.cuh @@ -7,6 +7,14 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_Q1_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); CASE(GGML_TYPE_Q1_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q2_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_Q4_0, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); CASE(GGML_TYPE_Q4_0, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); CASE(GGML_TYPE_Q4_0, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); diff --git a/ggml/src/ggml-cuda/mmq-config-pascal.cuh b/ggml/src/ggml-cuda/mmq-config-pascal.cuh index 8f0faac889..e7d4a9a3fc 100644 --- a/ggml/src/ggml-cuda/mmq-config-pascal.cuh +++ b/ggml/src/ggml-cuda/mmq-config-pascal.cuh @@ -11,6 +11,18 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); diff --git a/ggml/src/ggml-cuda/mmq-config-rdna2.cuh b/ggml/src/ggml-cuda/mmq-config-rdna2.cuh index de4db0a3db..8324d9e1a8 100644 --- a/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +++ b/ggml/src/ggml-cuda/mmq-config-rdna2.cuh @@ -11,6 +11,18 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); diff --git a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh new file mode 100644 index 0000000000..180b2d9370 --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh @@ -0,0 +1,290 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna3_5(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq-config-rdna3.cuh b/ggml/src/ggml-cuda/mmq-config-rdna3.cuh new file mode 100644 index 0000000000..676f27fea4 --- /dev/null +++ b/ggml/src/ggml-cuda/mmq-config-rdna3.cuh @@ -0,0 +1,290 @@ +static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna3(ggml_type type, int J, bool fallback) { + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + +// --------------------------------------------------------------------------------------------- + + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); +} diff --git a/ggml/src/ggml-cuda/mmq-config-rdna4.cuh b/ggml/src/ggml-cuda/mmq-config-rdna4.cuh index 6280e80ee4..9293d9d558 100644 --- a/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +++ b/ggml/src/ggml-cuda/mmq-config-rdna4.cuh @@ -1,77 +1,89 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna4(ggml_type type, int J, bool fallback) { - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q8_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); @@ -79,66 +91,62 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf // --------------------------------------------------------------------------------------------- - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q6_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); @@ -146,105 +154,105 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf // --------------------------------------------------------------------------------------------- - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); @@ -252,27 +260,27 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf // --------------------------------------------------------------------------------------------- - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuh index 7fb242096e..8ed704c281 100644 --- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh +++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh @@ -95,6 +95,87 @@ template static __device__ __forceinline_ } } +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q2_0( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + 2*MMQ_TILE_NE_K); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q8_0, I); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int blocks_per_iter = MMQ_ITER_K / QK2_0; + constexpr int threads_per_row = blocks_per_iter * QI2_0; + constexpr int nrows = warp_size / threads_per_row; + constexpr int scale_entries_per_block = QK2_0 / QK8_1; + constexpr int scale_entries_per_row = blocks_per_iter * scale_entries_per_block; + + const int txi = threadIdx.x % threads_per_row; + const int kbx = txi / QI2_0; + const int kqsx = txi % QI2_0; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nrows*nwarps) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + kbx; + const int16_t * qxi = (const int16_t *) bxi->qs + kqsx * 4; + + const int dst_offset = kbx*(scale_entries_per_block*QI8_0) + kqsx*QI8_0; + +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int q = qxi[j]; + + // unpack even and odd crumbs into byte values + const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0); + const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2); + // unshuffle values + const int qx = __byte_perm(qe, qo, 0x5140); + const int qy = __byte_perm(qe, qo, 0x7362); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + dst_offset + j*2+0] = qx; + x_qs[i*sram_stride + dst_offset + j*2+1] = qy; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*2+0] = qx; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + dst_offset + j*2+1] = qy; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + } + + const int ksx = threadIdx.x % scale_entries_per_row; + const int scale_block = ksx / scale_entries_per_block; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps) { + int i = i0 + threadIdx.y; + + if (fallback) { + i = min(i, i_max); + } + + const block_q2_0 * bxi = (const block_q2_0 *) x + kbx0 + i*stride + scale_block; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*sram_stride + ksx] = bxi->d; +#else + x_df[i*(2*MMQ_TILE_NE_K/QI8_0) + i/(QI8_0/2) + ksx] = bxi->d; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_q4_0( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int warp_size = ggml_cuda_get_physical_warp_size(); diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 8a0f4d3b5c..707437ea3e 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -10,6 +10,9 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con case GGML_TYPE_Q1_0: mul_mat_q_case(ctx, args, stream); break; + case GGML_TYPE_Q2_0: + mul_mat_q_case(ctx, args, stream); + break; case GGML_TYPE_Q4_0: mul_mat_q_case(ctx, args, stream); break; @@ -262,6 +265,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t switch (type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -296,6 +300,15 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t return false; } + // MMQ tiles require at least 48 KiB per-block shared memory; fall back to BLAS otherwise. + { + const int id = ggml_cuda_get_device(); + const size_t smpbo = ggml_cuda_info().devices[id].smpbo; + if (smpbo < 48 * 1024) { + return false; + } + } + if (turing_mma_available(cc)) { return true; } diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 71e3b2647a..2eb15fdfad 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -60,6 +60,7 @@ static_assert(sizeof(block_fp4_mmq) == sizeof(block_q8_1_mmq), "Unexpected b static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) { switch (type_x) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: return MMQ_Q8_1_DS_LAYOUT_D4; case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -218,6 +219,8 @@ struct ggml_cuda_mmq_config { #include "mmq-config-cdna.cuh" #include "mmq-config-rdna2.cuh" +#include "mmq-config-rdna3.cuh" +#include "mmq-config-rdna3-5.cuh" #include "mmq-config-rdna4.cuh" #undef CASE @@ -227,9 +230,15 @@ static __host__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(const ggml_type ty if (GGML_CUDA_CC_IS_CDNA(cc)) { return ggml_cuda_mmq_get_config_cdna(type, J, fallback); } - if (amd_wmma_available(cc)) { + if (GGML_CUDA_CC_IS_RDNA4(cc)) { return ggml_cuda_mmq_get_config_rdna4(type, J, fallback); } + if (GGML_CUDA_CC_IS_RDNA3_5(cc)) { + return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback); + } + if (GGML_CUDA_CC_IS_RDNA3(cc)) { // covers RDNA 3.0 + return ggml_cuda_mmq_get_config_rdna3(type, J, fallback); + } return ggml_cuda_mmq_get_config_rdna2(type, J, fallback); } if (blackwell_mma_available(cc)) { @@ -245,8 +254,12 @@ static constexpr __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config(ggml_t #ifdef GGML_USE_HIP #ifdef CDNA return ggml_cuda_mmq_get_config_cdna(type, J, fallback); -#elif defined(AMD_WMMA_AVAILABLE) +#elif defined(RDNA4) return ggml_cuda_mmq_get_config_rdna4(type, J, fallback); +#elif defined(RDNA3_5) + return ggml_cuda_mmq_get_config_rdna3_5(type, J, fallback); +#elif defined(RDNA3) + return ggml_cuda_mmq_get_config_rdna3(type, J, fallback); #else return ggml_cuda_mmq_get_config_rdna2(type, J, fallback); #endif // CDNA @@ -373,6 +386,7 @@ static constexpr __device__ int ggml_cuda_mmq_get_rows_per_warp(ggml_type type, static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml_type type, int I) { switch (type) { case GGML_TYPE_Q1_0: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_Q2_0: return MMQ_DP4A_TXS_Q8_0; case GGML_TYPE_Q4_0: return MMQ_DP4A_TXS_Q4_0; case GGML_TYPE_Q4_1: return MMQ_DP4A_TXS_Q4_1; case GGML_TYPE_Q5_0: return MMQ_DP4A_TXS_Q8_0; @@ -530,6 +544,12 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func ggml_cuda_mmq_load_tiles_q1_0, ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_Q2_0: + return ggml_cuda_mmq_util_funcs( + VDR_Q2_0_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_q2_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_Q4_0: return ggml_cuda_mmq_util_funcs( VDR_Q4_0_Q8_1_MMQ, @@ -688,6 +708,12 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func ggml_cuda_mmq_load_tiles_q1_0, ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_Q2_0: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_q2_0, + ggml_cuda_mmq_vec_dot_q8_0_q8_1_mma, + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_Q4_0: return ggml_cuda_mmq_util_funcs( -1, @@ -1538,6 +1564,7 @@ void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cuda template void mul_mat_q_case(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) \ extern DECL_MMQ_CASE(GGML_TYPE_Q1_0); +extern DECL_MMQ_CASE(GGML_TYPE_Q2_0); extern DECL_MMQ_CASE(GGML_TYPE_Q4_0); extern DECL_MMQ_CASE(GGML_TYPE_Q4_1); extern DECL_MMQ_CASE(GGML_TYPE_Q5_0); diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index e18ada5377..0589e65bdf 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -10,6 +10,7 @@ typedef float (*vec_dot_q_cuda_t)(const void * __restrict__ vbq, const block_q8_ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return vec_dot_q1_0_q8_1; + case GGML_TYPE_Q2_0: return vec_dot_q2_0_q8_1; case GGML_TYPE_Q4_0: return vec_dot_q4_0_q8_1; case GGML_TYPE_Q4_1: return vec_dot_q4_1_q8_1; case GGML_TYPE_Q5_0: return vec_dot_q5_0_q8_1; @@ -38,6 +39,7 @@ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: return VDR_Q1_0_Q8_1_MMVQ; + case GGML_TYPE_Q2_0: return VDR_Q2_0_Q8_1_MMVQ; case GGML_TYPE_Q4_0: return VDR_Q4_0_Q8_1_MMVQ; case GGML_TYPE_Q4_1: return VDR_Q4_1_Q8_1_MMVQ; case GGML_TYPE_Q5_0: return VDR_Q5_0_Q8_1_MMVQ; @@ -1010,6 +1012,12 @@ static void mul_mat_vec_q_switch_type( nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); break; + case GGML_TYPE_Q2_0: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; case GGML_TYPE_Q4_0: mul_mat_vec_q_switch_ncols_dst (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, diff --git a/ggml/src/ggml-cuda/ssm-scan.cu b/ggml/src/ggml-cuda/ssm-scan.cu index 3022249c77..f3418c2af8 100644 --- a/ggml/src/ggml-cuda/ssm-scan.cu +++ b/ggml/src/ggml-cuda/ssm-scan.cu @@ -9,6 +9,21 @@ using namespace cub; #include "ssm-scan.cuh" + +// Minimum number of tokens to use SSD (State Space Duality) matmul path instead of scan path. +// For n_tok <= this threshold, the scan kernel is used (lower overhead for short sequences). +#define SSM_SSD_MIN_TOKENS 128 + +// prepare_dt kernel dimensions: one block per (head, seq), each block handles DT_MAX_ITEMS items. +#define SSM_SSD_DT_BLOCK 256 +#define SSM_SSD_DT_MAX_ITEMS 32 + +// Maximum tokens the SSD path supports, derived from the prepare_dt kernel block capacity. +#define SSM_SSD_MAX_TOKENS (SSM_SSD_DT_BLOCK * SSM_SSD_DT_MAX_ITEMS) + +// Chunk size for chunked SSD. Caps matmul cost at O(chunk^2) per chunk. +#define SSM_SSD_CHUNK_SIZE 256 + // We would like to keep pragma unroll for cases where L_template is not 0, // so we suppress the clang transformation warning. #ifdef __clang__ @@ -316,6 +331,429 @@ static void ssm_scan_f32_cuda(const float * src0, const float * src1, const floa } } +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +// ============================================================================ +// SSD (State Space Duality) kernels for Mamba-2 prefill (n_tok > SSM_SSD_MIN_TOKENS) +// +// Instead of a sequential scan, SSD reformulates the output as: +// Y = (L (.) (C @ B^T)) @ (X * dt) + decay * C @ s_init +// where L is a causal decay mask derived from A and dt. +// +// This converts the O(T*N) sequential scan into parallel matmuls. +// ============================================================================ +// Softplus(dt) and inclusive prefix sum per head using CUB BlockScan. +// Grid: (n_head, n_seqs) +template +__global__ void ssm_ssd_prepare_dt_kernel( + const float * __restrict__ dt_raw, + float * __restrict__ dt_sp_out, + float * __restrict__ cs_out, + const int n_head, const int n_tok, + const int dt_stride_tok, // elements between tokens in dt + const int dt_stride_seq) { // elements between sequences in dt + + const int h = blockIdx.x; + const int s = blockIdx.y; + + const float * dt_seq = dt_raw + s * dt_stride_seq; + + float * dt_sp_seq = dt_sp_out + s * n_tok * n_head; + float * cs_seq = cs_out + s * n_tok * n_head; + + const int items_per_thread = (n_tok + BLOCK_SIZE - 1) / BLOCK_SIZE; + + // Phase 1: softplus with interleaved distribution (t = i*BLOCK_SIZE + threadIdx.x). + // Each warp reads BLOCK_SIZE consecutive tokens, giving coalesced dt_raw loads + // (stride n_head between threads vs. items_per_thread*n_head in blocked layout). + float local_vals[MAX_ITEMS]; + for (int i = 0; i < items_per_thread; i++) { + const int t = i * BLOCK_SIZE + threadIdx.x; + if (t < n_tok) { + float val = dt_seq[h + t * dt_stride_tok]; + float sp = (val <= 20.0f) ? log1pf(expf(val)) : val; + local_vals[i] = sp; + dt_sp_seq[t * n_head + h] = sp; + } else { + local_vals[i] = 0.0f; + } + } + + // Phase 2+3: per-step inclusive scan to build cs[] in token order. + // With interleaved distribution the per-thread total scan would not give token-order + // prefix sums, so we scan one BLOCK_SIZE slab at a time and carry a running total. +#ifdef USE_CUB + using BlockScan = cub::BlockScan; + __shared__ typename BlockScan::TempStorage scan_temp; + __shared__ float step_total; + + float running = 0.0f; + for (int i = 0; i < items_per_thread; i++) { + float inclusive; + BlockScan(scan_temp).InclusiveSum(local_vals[i], inclusive); + const int t = i * BLOCK_SIZE + threadIdx.x; + if (t < n_tok) { + cs_seq[t * n_head + h] = running + inclusive; + } + if (threadIdx.x == BLOCK_SIZE - 1) { + step_total = inclusive; + } + __syncthreads(); + running += step_total; + } +#else + // Fallback: sequential prefix scan in shared memory, one slab at a time. + __shared__ float sdata[BLOCK_SIZE]; + float running = 0.0f; + for (int i = 0; i < items_per_thread; i++) { + const int t = i * BLOCK_SIZE + threadIdx.x; + sdata[threadIdx.x] = local_vals[i]; + __syncthreads(); + if (threadIdx.x == 0) { + for (int j = 1; j < BLOCK_SIZE; j++) { + sdata[j] += sdata[j - 1]; + } + } + __syncthreads(); + if (t < n_tok) { + cs_seq[t * n_head + h] = running + sdata[threadIdx.x]; + } + running += sdata[BLOCK_SIZE - 1]; + __syncthreads(); + } +#endif +} + +// Prepare SSD matmul inputs for one chunk: X_dt, B_weighted, C_scaled. +// T_matmul controls precision for X_dt, B_weighted (float or half). +// C_scaled is always float (pairs with float s_cur in step 3c). +// Computation is always FP32; only the final store converts to T_matmul. +// Also materializes the causal M matrix = exp(A*(cs_out - cs_in)) * CB (fused with prep to save a launch). +// Grid: (ceil(max(C*head_dim, d_state*C, chunk_len^2) / BLOCK), n_head, n_seqs) +template +__global__ void ssm_ssd_pre_matmul_kernel( + const float * __restrict__ cs, // {n_tok, n_head} cumulative dt sums + const float * __restrict__ dt_sp, // {n_tok, n_head} softplus(dt) + const float * __restrict__ A, // {1, n_head} + const float * __restrict__ x, // {head_dim, n_head, n_tok, n_seqs} + const float * __restrict__ B, // {d_state, n_group, n_tok, n_seqs} + const float * __restrict__ C_src, // {d_state, n_group, n_tok, n_seqs} + T_matmul * __restrict__ X_dt, // {head_dim, C, n_head} x * dt, d-fastest + T_matmul * __restrict__ B_weighted, // {d_state, C, n_head} B * decay_from_end + float * __restrict__ C_scaled, // {d_state, C, n_head} C * decay_to_pos (always float) + const float * __restrict__ CB, // {chunk_len, chunk_len, n_group, n_seqs} + half * __restrict__ M_out, // {chunk_len, chunk_len, n_head, n_seqs} + const int chunk_len, const int head_dim, const int n_head, const int n_group, + const int d_state, const int A_stride, + const int x_stride_tok, const int x_stride_seq, + const int B_stride_tok, const int B_stride_seq, + const int C_stride_tok, const int C_stride_seq, + const int chunk_offset, + const int n_tok_total) { + + const int h = blockIdx.y; + const int s = blockIdx.z; + const int g = h / (n_head / n_group); + + const float A_h = A[h * A_stride]; + const int idx = blockIdx.x * BLOCK_SIZE + threadIdx.x; + + const int cs_seq_off = s * n_tok_total * n_head; + const float cs_base = (chunk_offset > 0) ? cs[cs_seq_off + (chunk_offset - 1) * n_head + h] : 0.0f; + const float cs_last = cs[cs_seq_off + (chunk_offset + chunk_len - 1) * n_head + h] - cs_base; + + // Prepare X_dt = x * dt, stored d-fastest for coalesced reads and writes. + const int n_xdt = chunk_len * head_dim; + if (idx < n_xdt) { + const int d = idx % head_dim; + const int t = idx / head_dim; + + const float x_val = x[s * x_stride_seq + (chunk_offset + t) * x_stride_tok + d + h * head_dim]; + const float dt_val = dt_sp[cs_seq_off + (chunk_offset + t) * n_head + h]; + + X_dt[d + t * head_dim + h * n_xdt + s * n_xdt * n_head] = (T_matmul)(x_val * dt_val); + } + + // Prepare B_weighted and C_scaled together: both share the same index space (d_state * chunk_len) + // and the same cs_t load, so merging halves the cs[] global memory traffic. + const int n_bw = d_state * chunk_len; + if (idx < n_bw) { + const int n = idx % d_state; + const int t = idx / d_state; + + const float cs_t = cs[cs_seq_off + (chunk_offset + t) * n_head + h] - cs_base; + + const float B_val = B[s * B_stride_seq + (chunk_offset + t) * B_stride_tok + g * d_state + n]; + B_weighted[n + t * d_state + h * n_bw + s * n_bw * n_head] = (T_matmul)(B_val * __expf(A_h * (cs_last - cs_t))); + + const float C_val = C_src[s * C_stride_seq + (chunk_offset + t) * C_stride_tok + g * d_state + n]; + C_scaled[n + t * d_state + h * n_bw + s * n_bw * n_head] = C_val * __expf(A_h * cs_t); + } + + // Materialize M = exp(A*(cs_out - cs_in)) * CB with causal mask. + const int n_M = chunk_len * chunk_len; + if (idx < n_M) { + const int t_out = idx % chunk_len; + const int t_in = idx / chunk_len; + + half val; + if (t_in <= t_out) { + const float cs_out = cs[cs_seq_off + (chunk_offset + t_out) * n_head + h] - cs_base; + const float cs_in = cs[cs_seq_off + (chunk_offset + t_in) * n_head + h] - cs_base; + const float decay = __expf(A_h * (cs_out - cs_in)); + const float * CB_g = CB + (int64_t)s * chunk_len * chunk_len * n_group + + (int64_t)g * chunk_len * chunk_len; + const float cb_val = CB_g[t_out + t_in * chunk_len]; + val = __float2half(decay * cb_val); + } else { + val = __float2half(0.0f); + } + + M_out[(int64_t)s * n_M * n_head + (int64_t)h * n_M + t_in * chunk_len + t_out] = val; + } +} + +// Scale running state in-place: s_cur *= decay_total(chunk). +// Called BEFORE cuBLAS state update (beta=1) to fuse inter-chunk decay. +// Eliminates the s_old buffer and D2D memcpy vs the old approach of: +// memcpy(s_old, s_cur) -> cuBLAS(beta=0) -> s_cur += decay * s_old +// Grid: (ceil(d_state * head_dim / BLOCK), n_head, n_seqs) +template +__global__ void ssm_ssd_scale_state_kernel( + float * __restrict__ s_cur, // {d_state, head_dim, n_head, n_seqs} + const float * __restrict__ cs, // {n_tok, n_head} cumulative dt sums + const float * __restrict__ A, // {1, n_head} + const int d_state, const int head_dim, const int n_head, + const int chunk_offset, const int chunk_len, + const int n_tok_total, const int A_stride) { + + const int h = blockIdx.y; + const int s = blockIdx.z; + const int idx = blockIdx.x * BLOCK_SIZE + threadIdx.x; + const int state_per_head = d_state * head_dim; + if (idx >= state_per_head) return; + + const float A_h = A[h * A_stride]; + const int cs_seq_off = s * n_tok_total * n_head; + const float cs_base = (chunk_offset > 0) ? cs[cs_seq_off + (chunk_offset - 1) * n_head + h] : 0.0f; + const float cs_last = cs[cs_seq_off + (chunk_offset + chunk_len - 1) * n_head + h] - cs_base; + const float decay_total = __expf(A_h * cs_last); + + const int off = s * state_per_head * n_head + h * state_per_head + idx; + s_cur[off] *= decay_total; +} + +// Copy initial state from src0[ids[s]] into s_cur for each sequence. +// Grid: (ceil(d_state * head_dim * n_head / BLOCK), n_seqs) +template +__global__ void ssm_ssd_init_state_kernel( + const float * __restrict__ src0, // {d_state, head_dim, n_head, n_rs} + const int32_t * __restrict__ ids, // {n_seqs} + float * __restrict__ s_cur, // {d_state, head_dim, n_head, n_seqs} + const int state_size, // d_state * head_dim * n_head + const int64_t s0_stride_seq) { // elements between state rows + const int s = blockIdx.y; + const int idx = blockIdx.x * BLOCK_SIZE + threadIdx.x; + if (idx >= state_size) return; + + const float * s_src = src0 + (int64_t)ids[s] * s0_stride_seq; + s_cur[s * state_size + idx] = s_src[idx]; +} + +// SSD (State Space Duality) dispatch for Mamba-2 prefill. +// Chunked matmuls: CB, materialize M + cuBLAS Y, S@C, B@X_dt. +// All strides are in elements (floats), not bytes. +static void ssm_scan_ssd_f32_cuda( + ggml_backend_cuda_context & ctx, + const float * src0_d, const float * src1_d, const float * src2_d, const float * src3_d, + const float * src4_d, const float * src5_d, const int32_t * src6_d, float * dst_d, + const int64_t s0_stride_seq, // state (src0) stride between seqs + const int x_stride_tok, const int x_stride_seq, // x (src1) strides + const int dt_stride_tok, const int dt_stride_seq, // dt (src2) strides + const int A_stride, // A (src3) stride between heads + const int B_stride_tok, const int B_stride_seq, // B (src4) strides + const int C_stride_tok, const int C_stride_seq, // C (src5) strides + const int64_t s_off, const int64_t d_state, const int64_t head_dim, + const int64_t n_head, const int64_t n_group, const int64_t n_tok, const int64_t n_seq) { + + cudaStream_t stream = ctx.stream(); + const int64_t d_inner = head_dim * n_head; + + const int64_t chunk_size = SSM_SSD_CHUNK_SIZE; + const int64_t n_chunks = (n_tok + chunk_size - 1) / chunk_size; + + const int64_t state_per_head = d_state * head_dim; + + using matmul_t = half; + static constexpr cudaDataType_t matmul_dtype = CUDA_R_16F; + + ggml_cuda_pool_alloc dt_sp_buf(ctx.pool(), n_tok * n_head * n_seq); + ggml_cuda_pool_alloc cs_buf(ctx.pool(), n_tok * n_head * n_seq); + ggml_cuda_pool_alloc CB_buf(ctx.pool(), chunk_size * chunk_size * n_group * n_seq); + ggml_cuda_pool_alloc X_dt_buf(ctx.pool(), chunk_size * head_dim * n_head * n_seq); + ggml_cuda_pool_alloc B_w_buf(ctx.pool(), d_state * chunk_size * n_head * n_seq); + ggml_cuda_pool_alloc C_s_buf(ctx.pool(), d_state * chunk_size * n_head * n_seq); + float * dt_sp = dt_sp_buf.get(); + float * cs = cs_buf.get(); + float * CB = CB_buf.get(); + matmul_t * X_dt = X_dt_buf.get(); + matmul_t * B_weighted = B_w_buf.get(); + float * C_scaled = C_s_buf.get(); + float * s_cur = (float *)((char *)dst_d + s_off); // write state directly to dst + + // Step 1: softplus(dt) and parallel prefix sum over full sequence + { + dim3 grid(n_head, n_seq); + ssm_ssd_prepare_dt_kernel<<>>( + src2_d, dt_sp, cs, n_head, n_tok, dt_stride_tok, dt_stride_seq); + CUDA_CHECK(cudaGetLastError()); + } + + // Step 2: initialize running state from src0[ids[s]] + { + constexpr int BLOCK = 256; + const int64_t state_size = d_state * head_dim * n_head; + dim3 grid((state_size + BLOCK - 1) / BLOCK, n_seq); + ssm_ssd_init_state_kernel<<>>( + src0_d, src6_d, s_cur, state_size, s0_stride_seq); + CUDA_CHECK(cudaGetLastError()); + } + + // Step 3: chunked SSD loop + // Per chunk: pre_matmul (incl. M) + 4 cuBLAS (CB, Y, S@C, state update) + scale_state + cublasHandle_t handle = ctx.cublas_handle(); + CUBLAS_CHECK(cublasSetStream(handle, stream)); + const float alpha_one = 1.0f; + const float beta_zero = 0.0f; + const float beta_one = 1.0f; + const int lda_C_src = C_stride_tok; // leading dim for C in CB = C^T @ B + const int ldb_B_src = B_stride_tok; // leading dim for B in CB = C^T @ B + + // Scratch buffer for causal M matrix, reused across chunks (max size at chunk_size) + const int64_t n_M_max = chunk_size * chunk_size; + ggml_cuda_pool_alloc M_buf(ctx.pool(), n_M_max * n_head * n_seq); + half * M_mat = M_buf.get(); + + for (int64_t k = 0; k < n_chunks; k++) { + const int64_t chunk_offset = k * chunk_size; + const int64_t chunk_len = (chunk_offset + chunk_size <= n_tok) ? chunk_size : (n_tok - chunk_offset); + + // 3a: CB = C^T @ B per group + for (int64_t s = 0; s < n_seq; s++) { + const float * C_s = src5_d + s * C_stride_seq + chunk_offset * C_stride_tok; + const float * B_s = src4_d + s * B_stride_seq + chunk_offset * B_stride_tok; + float * CB_s = CB + s * chunk_len * chunk_len * n_group; + + if (n_group == 1) { + CUBLAS_CHECK(cublasSgemm(handle, CUBLAS_OP_T, CUBLAS_OP_N, + chunk_len, chunk_len, d_state, + &alpha_one, C_s, lda_C_src, B_s, ldb_B_src, + &beta_zero, CB_s, (int)chunk_len)); + } else { + CUBLAS_CHECK(cublasGemmStridedBatchedEx(handle, CUBLAS_OP_T, CUBLAS_OP_N, + chunk_len, chunk_len, d_state, + &alpha_one, + C_s, CUDA_R_32F, lda_C_src, d_state, + B_s, CUDA_R_32F, ldb_B_src, d_state, + &beta_zero, + CB_s, CUDA_R_32F, (int)chunk_len, (long long)(chunk_len * chunk_len), + n_group, + CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + } + } + + // 3b: prepare X_dt, B_weighted, C_scaled + materialize causal M matrix + const int64_t n_M = chunk_len * chunk_len; + { + constexpr int BLOCK = 256; + const int64_t n_xdt = chunk_len * head_dim; + const int64_t n_bw = d_state * chunk_len; + int64_t max_work = n_xdt; + if (n_bw > max_work) max_work = n_bw; + if (n_M > max_work) max_work = n_M; + dim3 grid((max_work + BLOCK - 1) / BLOCK, n_head, n_seq); + ssm_ssd_pre_matmul_kernel<<>>( + cs, dt_sp, src3_d, src1_d, src4_d, src5_d, + X_dt, B_weighted, C_scaled, + CB, M_mat, + chunk_len, head_dim, n_head, n_group, d_state, A_stride, + x_stride_tok, x_stride_seq, B_stride_tok, B_stride_seq, C_stride_tok, C_stride_seq, + chunk_offset, n_tok); + CUDA_CHECK(cudaGetLastError()); + } + + // 3c: dst = S_cur^T @ C_scaled (state contribution) + { + const int64_t stride_S = state_per_head; + const int64_t stride_Cs = d_state * chunk_len; + + for (int64_t s = 0; s < n_seq; s++) { + float * dst_chunk = dst_d + s * d_inner * n_tok + chunk_offset * d_inner; + + CUBLAS_CHECK(cublasGemmStridedBatchedEx(handle, CUBLAS_OP_T, CUBLAS_OP_N, + head_dim, chunk_len, d_state, + &alpha_one, + s_cur + s * stride_S * n_head, CUDA_R_32F, d_state, stride_S, + C_scaled + s * stride_Cs * n_head, CUDA_R_32F, d_state, stride_Cs, + &beta_zero, + dst_chunk, CUDA_R_32F, d_inner, head_dim, + n_head, + CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + } + } + + // 3d: dst += X_dt @ M^T (intra-chunk contribution, adds to 3c result) + // M is stored as M[t_out, t_in] (lower-triangular), transpose needed for Y = X @ M^T. + { + const int64_t stride_M = n_M; + const int64_t stride_X_h = (int64_t)chunk_len * head_dim; + + for (int64_t s = 0; s < n_seq; s++) { + float * dst_chunk = dst_d + s * d_inner * n_tok + chunk_offset * d_inner; + CUBLAS_CHECK(cublasGemmStridedBatchedEx(handle, CUBLAS_OP_N, CUBLAS_OP_T, + head_dim, chunk_len, chunk_len, + &alpha_one, + X_dt + s * stride_X_h * n_head, matmul_dtype, head_dim, stride_X_h, + M_mat + s * stride_M * n_head, matmul_dtype, chunk_len, stride_M, + &beta_one, + dst_chunk, CUDA_R_32F, d_inner, head_dim, + n_head, + CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + } + } + + // 3e: s_cur = B_weighted @ X_dt^T + decay_total * s_cur_old (state update) + { + // Scale s_cur in-place by per-head decay_total BEFORE cuBLAS overwrites it + constexpr int BLOCK = 256; + dim3 grid((state_per_head + BLOCK - 1) / BLOCK, n_head, n_seq); + ssm_ssd_scale_state_kernel<<>>( + s_cur, cs, src3_d, + d_state, head_dim, n_head, + chunk_offset, chunk_len, n_tok, A_stride); + CUDA_CHECK(cudaGetLastError()); + + // cuBLAS with beta=1: s_cur = B_weighted @ X_dt^T + 1.0 * s_cur (already scaled) + const int64_t stride_Bw = d_state * chunk_len; + const int64_t stride_X = chunk_len * head_dim; + const int64_t stride_S = state_per_head; + + for (int64_t s = 0; s < n_seq; s++) { + // X_dt is d-fastest {hd, C}, read as OP_T to get {C, hd} + CUBLAS_CHECK(cublasGemmStridedBatchedEx(handle, CUBLAS_OP_N, CUBLAS_OP_T, + d_state, head_dim, chunk_len, + &alpha_one, + B_weighted + s * stride_Bw * n_head, matmul_dtype, d_state, stride_Bw, + X_dt + s * stride_X * n_head, matmul_dtype, head_dim, stride_X, + &beta_one, + s_cur + s * stride_S * n_head, CUDA_R_32F, d_state, stride_S, + n_head, + CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT)); + } + } + } +} +#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const struct ggml_tensor * src0 = dst->src[0]; // s const struct ggml_tensor * src1 = dst->src[1]; // x @@ -357,6 +795,49 @@ void ggml_cuda_op_ssm_scan(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { GGML_ASSERT(src6->type == GGML_TYPE_I32); GGML_ASSERT(dst->type == GGML_TYPE_F32); + // Byte strides are narrowed to int for both scan and SSD paths. + GGML_ASSERT(src0->nb[2] <= (size_t)INT_MAX); + GGML_ASSERT(src0->nb[3] <= (size_t)INT_MAX); + GGML_ASSERT(src1->nb[2] <= (size_t)INT_MAX); + GGML_ASSERT(src1->nb[3] <= (size_t)INT_MAX); + GGML_ASSERT(src2->nb[1] <= (size_t)INT_MAX); + GGML_ASSERT(src2->nb[2] <= (size_t)INT_MAX); + GGML_ASSERT(src3->nb[1] <= (size_t)INT_MAX); + GGML_ASSERT(src4->nb[2] <= (size_t)INT_MAX); + GGML_ASSERT(src4->nb[3] <= (size_t)INT_MAX); + GGML_ASSERT(src5->nb[2] <= (size_t)INT_MAX); + GGML_ASSERT(src5->nb[3] <= (size_t)INT_MAX); + +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) + // Mamba-2 with scalar A per head: use SSD matmul path for long sequences. + // Requires NVIDIA Turing+ otherwise fallback to scan. + const bool is_mamba2 = (src3->nb[1] == sizeof(float)); + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const bool use_ssd = is_mamba2 && n_t > SSM_SSD_MIN_TOKENS + && n_t <= SSM_SSD_MAX_TOKENS + && GGML_CUDA_CC_IS_NVIDIA(cc) + && cc >= GGML_CUDA_CC_TURING + && nr % 8 == 0; // cuBLAS requires 8-element (16-byte) alignment + + if (use_ssd) { + // ssm_ssd_init_state_kernel uses flat linear indexing within each sequence, + // so src0 must be fully contiguous across all inner dimensions. + // The scan path handles non-contiguous nb[2] via src0_nb2 but does not handle nb[1]. + GGML_ASSERT(src0->nb[1] == nc * sizeof(float)); + GGML_ASSERT(src0->nb[2] == nc * nr * sizeof(float)); + + ssm_scan_ssd_f32_cuda(ctx, + src0_d, src1_d, src2_d, src3_d, src4_d, src5_d, src6_d, dst_d, + (int64_t)(src0->nb[3] / sizeof(float)), + (int)(src1->nb[2] / sizeof(float)), (int)(src1->nb[3] / sizeof(float)), + (int)(src2->nb[1] / sizeof(float)), (int)(src2->nb[2] / sizeof(float)), + (int)(src3->nb[1] / sizeof(float)), + (int)(src4->nb[2] / sizeof(float)), (int)(src4->nb[3] / sizeof(float)), + (int)(src5->nb[2] / sizeof(float)), (int)(src5->nb[3] / sizeof(float)), + s_off, nc, nr, nh, ng, n_t, n_s); + return; + } +#endif ssm_scan_f32_cuda(src0_d, src1_d, src2_d, src3_d, src4_d, src5_d, src6_d, dst_d, src0->nb[2], src0->nb[3], src1->nb[2], src1->nb[3], src2->nb[1], src2->nb[2], src3->nb[1], src4->nb[2], src4->nb[3], src5->nb[2], src5->nb[3], diff --git a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py index 614b1566c7..d7cd271675 100755 --- a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +++ b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py @@ -36,6 +36,7 @@ SOURCE_FATTN_MMA_CASE = "DECL_FATTN_MMA_F16_CASE({head_size_kq}, {head_size_v}, TYPES_MMQ = [ "GGML_TYPE_Q1_0", + "GGML_TYPE_Q2_0", "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_Q2_K", "GGML_TYPE_Q3_K", "GGML_TYPE_Q4_K", "GGML_TYPE_Q5_K", "GGML_TYPE_Q6_K", "GGML_TYPE_IQ2_XXS", "GGML_TYPE_IQ2_XS", "GGML_TYPE_IQ2_S", "GGML_TYPE_IQ3_XXS", "GGML_TYPE_IQ3_S", diff --git a/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu b/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu new file mode 100644 index 0000000000..750180e330 --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-q2_0.cu @@ -0,0 +1,5 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../mmq.cuh" + +DECL_MMQ_CASE(GGML_TYPE_Q2_0); diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuh index b9932bce9c..0f039c735b 100644 --- a/ggml/src/ggml-cuda/vecdotq.cuh +++ b/ggml/src/ggml-cuda/vecdotq.cuh @@ -109,6 +109,9 @@ static __device__ __forceinline__ uint32_t unpack_ksigns(const uint8_t v) { #define VDR_Q1_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism #define VDR_Q1_0_Q8_1_MMQ 4 // Q1_0 has 128 bits (4 ints) per block +#define VDR_Q2_0_Q8_1_MMVQ 1 // Process one 32-element chunk at a time for parallelism +#define VDR_Q2_0_Q8_1_MMQ 2 // Q2_0 group 64: 128 bits (4 ints) per block, 2 32-element chunks + #define VDR_Q4_0_Q8_1_MMVQ 2 #define VDR_Q4_0_Q8_1_MMQ 4 @@ -722,6 +725,44 @@ static __device__ __forceinline__ float vec_dot_q1_0_q8_1( return d1 * d8 * sumi; } +static __device__ __forceinline__ float vec_dot_q2_0_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q2_0 * bq2_0 = (const block_q2_0 *) vbq + kbx; + + // Q2_0 (group 64): 64 elements with ONE scale, 2 bits per element (4 elements per byte) + // Q8_1: 32 elements per block with individual scales + // iqs selects which of the 2 chunks of 32 elements to process (0-1) + + const float d2 = bq2_0->d; + const int16_t * qs = (const int16_t *) bq2_0->qs + iqs * 4; + + // Process only the chunk specified by iqs + const block_q8_1 * bq8_1_chunk = bq8_1 + iqs; + + int sumi = 0; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int q = qs[j]; + const int u = get_int_b4(bq8_1_chunk->qs, j*2+0); + const int v = get_int_b4(bq8_1_chunk->qs, j*2+1); + + // unpack even and odd crumbs into byte values + const int qe = __byte_perm(0x020100FF, 0x020100FF, q >> 0); + const int qo = __byte_perm(0x020100FF, 0x020100FF, q >> 2); + // unshuffle values + const int qx = __byte_perm(qe, qo, 0x5140); + const int qy = __byte_perm(qe, qo, 0x7362); + + sumi = ggml_cuda_dp4a(u, qx, sumi); + sumi = ggml_cuda_dp4a(v, qy, sumi); + } + + // Apply Q2_0's single scale and this chunk's Q8_1 scale + const float d8 = __low2float(bq8_1_chunk->ds); + return d2 * d8 * sumi; +} + static __device__ __forceinline__ float vec_dot_q4_0_q8_1( const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { diff --git a/ggml/src/ggml-hip/CMakeLists.txt b/ggml/src/ggml-hip/CMakeLists.txt index 5351dcae12..bbc51797c1 100644 --- a/ggml/src/ggml-hip/CMakeLists.txt +++ b/ggml/src/ggml-hip/CMakeLists.txt @@ -154,5 +154,3 @@ if (GGML_HIP_RCCL) endif() target_link_libraries(ggml-hip PRIVATE ggml-base hip::host roc::rocblas roc::hipblas) - -target_compile_options(ggml-hip PRIVATE "$<$:-ffast-math;-fno-finite-math-only>") diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 270c1411a0..16e98eb519 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -1252,6 +1252,21 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort_merge(gg return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht(ggml_metal_library_t lib, int n) { + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_fwht_f32_%d", n); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + return res; +} + // note: reuse the argsort kernel for top_k ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k(ggml_metal_library_t lib, const ggml_tensor * op) { assert(op->op == GGML_OP_TOP_K); diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index b36fa8110b..91b841b67b 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -139,6 +139,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_id struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argmax (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_argsort_merge (ggml_metal_library_t lib, const struct ggml_tensor * op); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_fwht (ggml_metal_library_t lib, int n); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_top_k_merge (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_bin (ggml_metal_library_t lib, const struct ggml_tensor * op, int32_t n_fuse ); @@ -212,7 +213,7 @@ typedef void * ggml_metal_rset_t; // a collection of residency sets (non-owning) typedef struct ggml_metal_rsets * ggml_metal_rsets_t; -ggml_metal_rsets_t ggml_metal_rsets_init(void); +ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev); void ggml_metal_rsets_free(ggml_metal_rsets_t rsets); // diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 4edd77c6f2..7d2a686850 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -557,7 +557,32 @@ struct ggml_metal_rsets { dispatch_group_t d_group; }; -ggml_metal_rsets_t ggml_metal_rsets_init(void) { +#if defined(GGML_METAL_HAS_RESIDENCY_SETS) +static void ggml_metal_dummy_work(ggml_metal_device_t dev) { + if (dev->mtl_queue == nil) { + return; + } + + @autoreleasepool { + // perform a minimal dummy operation on the GPU + id buf = [dev->mtl_device newBufferWithLength:1 options:MTLResourceStorageModePrivate]; + id cmd_buf = [dev->mtl_queue commandBuffer]; + + { + id encoder = [cmd_buf blitCommandEncoder]; + + [encoder fillBuffer:buf range:NSMakeRange(0, 1) value:0]; + + [encoder endEncoding]; + } + + [cmd_buf commit]; + [buf release]; + } +} +#endif + +ggml_metal_rsets_t ggml_metal_rsets_init(ggml_metal_device_t dev) { ggml_metal_rsets_t res = calloc(1, sizeof(struct ggml_metal_rsets)); res->lock = [[NSLock alloc] init]; @@ -610,6 +635,15 @@ ggml_metal_rsets_t ggml_metal_rsets_init(void) { #endif }); +#if defined(GGML_METAL_HAS_RESIDENCY_SETS) + if (@available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, *)) { + // workaround for residency set memory not being released if no GPU operation occurs + // https://developer.apple.com/forums/thread/839089 + // https://github.com/ggml-org/llama.cpp/issues/25937 + ggml_metal_dummy_work(dev); + } +#endif + return res; } @@ -864,7 +898,7 @@ ggml_metal_device_t ggml_metal_device_init(int device) { } if (dev->props.use_residency_sets) { - dev->rsets = ggml_metal_rsets_init(); + dev->rsets = ggml_metal_rsets_init(dev); } else { dev->rsets = nil; } @@ -1484,6 +1518,7 @@ static void ggml_metal_buffer_rset_free(ggml_metal_buffer_t buf) { if (buf->rset) { [buf->rset endResidency]; [buf->rset removeAllAllocations]; + [buf->rset commit]; [buf->rset release]; } } diff --git a/ggml/src/ggml-metal/ggml-metal-impl.h b/ggml/src/ggml-metal/ggml-metal-impl.h index 330278d003..9f350aad5b 100644 --- a/ggml/src/ggml-metal/ggml-metal-impl.h +++ b/ggml/src/ggml-metal/ggml-metal-impl.h @@ -1157,6 +1157,10 @@ typedef struct { int32_t len; } ggml_metal_kargs_argsort_merge; +typedef struct { + int32_t nrows; +} ggml_metal_kargs_fwht; + typedef struct { int64_t ne0; float start; diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index c716f118f6..76626a4518 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -1979,6 +1979,46 @@ int ggml_metal_op_pool_1d(ggml_metal_op_t ctx, int idx) { return 1; } +// supported FWHT sizes, must stay in sync with the +// kernel_fwht_f32_ templates in ggml-metal.metal +static bool ggml_metal_fwht_supported_size(int64_t n) { + return n == 64 || n == 128 || n == 256 || n == 512; +} + +int ggml_metal_op_fwht(ggml_metal_op_t ctx, int idx) { + ggml_tensor * op = ctx->node(idx); + + ggml_metal_library_t lib = ctx->lib; + ggml_metal_encoder_t enc = ctx->enc; + + ggml_tensor * src1 = op->src[1]; + + const int64_t n = src1->ne[0]; + const int64_t nrows = ggml_nrows(src1); + + ggml_metal_kargs_fwht args = { + /*.nrows = */ (int32_t) nrows, + }; + + auto pipeline = ggml_metal_library_get_pipeline_fwht(lib, n); + + ggml_metal_encoder_set_pipeline(enc, pipeline); + ggml_metal_encoder_set_bytes(enc, &args, sizeof(args), 0); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(src1), 1); + ggml_metal_encoder_set_buffer(enc, ggml_metal_get_buffer_id(op), 2); + + const int th_max = ggml_metal_pipeline_max_theads_per_threadgroup(pipeline); + const int simd_size = 32; + + int sg_per_tg = 2; + sg_per_tg = std::min(sg_per_tg, th_max/simd_size); + sg_per_tg = std::max(sg_per_tg, 1); + + const int64_t n_tg = (nrows + sg_per_tg - 1) / sg_per_tg; + ggml_metal_encoder_dispatch_threadgroups(enc, n_tg, 1, 1, 32*sg_per_tg, 1, 1); + + return 1; +} int ggml_metal_op_pool_2d(ggml_metal_op_t ctx, int idx) { ggml_tensor * op = ctx->node(idx); @@ -2046,6 +2086,18 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) { ggml_metal_library_t lib = ctx->lib; ggml_metal_encoder_t enc = ctx->enc; + const int32_t hint = ggml_get_op_params_i32(op, 1); + + if (hint == GGML_HINT_SRC0_IS_HADAMARD) { + if (op->src[1]->type == GGML_TYPE_F32 && + op->type == GGML_TYPE_F32 && + ggml_is_contiguous(op->src[1]) && + ggml_is_contiguous(op) && + ggml_are_same_shape(op->src[1], op) && + ggml_metal_fwht_supported_size(op->src[1]->ne[0])) { + return ggml_metal_op_fwht(ctx, idx); + } + } const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev); GGML_TENSOR_LOCALS( int32_t, ne0, op->src[0], ne); diff --git a/ggml/src/ggml-metal/ggml-metal-ops.h b/ggml/src/ggml-metal/ggml-metal-ops.h index 89a6ad82f1..2783ecb8b6 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.h +++ b/ggml/src/ggml-metal/ggml-metal-ops.h @@ -64,6 +64,7 @@ int ggml_metal_op_set (ggml_metal_op_t ctx, int idx); int ggml_metal_op_cpy (ggml_metal_op_t ctx, int idx); int ggml_metal_op_pool_1d (ggml_metal_op_t ctx, int idx); int ggml_metal_op_pool_2d (ggml_metal_op_t ctx, int idx); +int ggml_metal_op_fwht (ggml_metal_op_t ctx, int idx); int ggml_metal_op_mul_mat (ggml_metal_op_t ctx, int idx); int ggml_metal_op_mul_mat_id (ggml_metal_op_t ctx, int idx); int ggml_metal_op_add_id (ggml_metal_op_t ctx, int idx); diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 969fddfa5b..f14ee0792b 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -5762,7 +5762,7 @@ kernel void kernel_upscale_bicubic_f32( const float w_y2 = bicubic_weight1(1.0f - fd1); const float w_y3 = bicubic_weight2(2.0f - fd1); - const device const char * src_slice = src0 + i03 * args.nb03 + i02 * args.nb02; + const device char * src_slice = src0 + i03 * args.nb03 + i02 * args.nb02; device float * dst_ptr = (device float *)(dst + i3 * args.nb3 + i2 * args.nb2 + i1 * args.nb1); @@ -6172,6 +6172,68 @@ kernel void kernel_argsort_merge_f32_i32( template [[host_name("kernel_argsort_merge_f32_i32_asc")]] kernel argsort_merge_t kernel_argsort_merge_f32_i32; template [[host_name("kernel_argsort_merge_f32_i32_desc")]] kernel argsort_merge_t kernel_argsort_merge_f32_i32; +template +kernel void kernel_fwht_f32( + constant ggml_metal_kargs_fwht & args, + device const float * src, + device float * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort sgitg[[simdgroup_index_in_threadgroup]], + ushort tiisg[[thread_index_in_simdgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { + + constexpr int NW = N_SIMDWIDTH; + constexpr int NE = N / NW; + + const float scale = 1.0f / sqrt((float) N); + + const int sg_per_tg = ntg.x / NW; + const int64_t r = tgpig.x * sg_per_tg + sgitg; + if (r >= args.nrows) { + return; + } + + src += r * N; + dst += r * N; + + const int lane = tiisg; + + float reg[NE]; + for (int i = 0; i < NE; i++) { + reg[i] = src[i*NW + lane]*scale; + } + for (int i = 1; i < NW; i *= 2) { + for (int j = 0; j < NE; j++) { + const float val = reg[j]; + const float val2 = simd_shuffle_xor(val, i); + reg[j] = (lane & i) == 0 ? val2 + val : val2 - val; + } + } + + for (int i = NW; i < N; i *= 2) { + const int step = i / NW; + for (int j = 0; j < NE; j += (2 * step)) { + for (int k = 0; k < step; k++) { + const float x = reg[j + k ]; + const float y = reg[j + k + step]; + reg[j + k] = x + y; + reg[j + k + step] = x - y; + } + } + } + + for (int i = 0; i < NE; i++) { + dst[i*NW + lane] = reg[i]; + } +} + +typedef decltype(kernel_fwht_f32<64>) kernel_fwht_t; + +template [[host_name("kernel_fwht_f32_64")]] kernel kernel_fwht_t kernel_fwht_f32<64>; +template [[host_name("kernel_fwht_f32_128")]] kernel kernel_fwht_t kernel_fwht_f32<128>; +template [[host_name("kernel_fwht_f32_256")]] kernel kernel_fwht_t kernel_fwht_f32<256>; +template [[host_name("kernel_fwht_f32_512")]] kernel kernel_fwht_t kernel_fwht_f32<512>; + constant bool FC_flash_attn_ext_pad_has_mask [[function_constant(FC_FLASH_ATTN_EXT_PAD + 0)]]; constant int32_t FC_flash_attn_ext_pad_ncpsg [[function_constant(FC_FLASH_ATTN_EXT_PAD + 25)]]; diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index a05d18ee30..d07b8fe41a 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -15675,7 +15675,7 @@ static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_ten // <--------------------------------------------> // extra0 = src0->view_src ? (ggml_tensor_extra_cl *)src0->view_src->extra : (ggml_tensor_extra_cl *)src0->extra; - region.origin = (extra0->offset); + region.origin = (extra0->offset + src0->view_offs); if (nb01 > nb02) { // KQ region.size = nb01 * ne01; @@ -15691,7 +15691,7 @@ static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_ten // create sub-buffer for B // <--------------------------------------------> // - region.origin = (extra1->offset); + region.origin = (extra1->offset + src1->view_offs); region.size = nb10 * ne10 * ne11 * ne12; B_sub_buffer = clCreateSubBuffer((extra1->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); @@ -15712,7 +15712,7 @@ static void ggml_cl_mul_mat_kq_kqv_adreno(ggml_backend_t backend, const ggml_ten // create sub-buffer for output C // <--------------------------------------------> // - region.origin = (extrad->offset); + region.origin = (extrad->offset + dst->view_offs); region.size = ne0 * ne1 * dst->ne[2] * dst->nb[0]; // size of C in bytes D_sub_buffer = clCreateSubBuffer((extrad->data_device), 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &status); CL_CHECK(status); @@ -18591,6 +18591,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co #ifdef GGML_OPENCL_USE_ADRENO_KERNELS if(src0t == GGML_TYPE_F16 && src1t == GGML_TYPE_F32){ if (ne01 >= 64 && ne1 >= 32 && ne00 >= 16 && (ne12 % ne02) == 0 && + // the KQ/KQV image kernels do not handle dim 3 (multi-stream batches) + ne03 == 1 && ne13 == 1 && // dst is wrapped with image1d_buffer, the size limit applies, also src0 (ne0 * ne1 * dst->ne[2] * dst->nb[0] / 4 <= backend_ctx->image_max_buffer_size)) { // For KQ diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index d380577218..17c53a5f04 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -71,6 +71,7 @@ enum rpc_cmd { RPC_CMD_HELLO, RPC_CMD_DEVICE_COUNT, RPC_CMD_GRAPH_RECOMPUTE, + RPC_CMD_MEMSET_TENSOR, RPC_CMD_COUNT, }; @@ -152,6 +153,13 @@ struct rpc_msg_buffer_clear_req { uint8_t value; }; +struct rpc_msg_memset_tensor_req { + rpc_tensor tensor; + uint64_t offset; + uint64_t size; + uint8_t value; +}; + struct rpc_msg_set_tensor_hash_req { rpc_tensor tensor; uint64_t offset; @@ -462,6 +470,19 @@ static enum ggml_status ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_ return GGML_STATUS_SUCCESS; } +static void ggml_backend_rpc_buffer_memset_tensor( + ggml_backend_buffer_t buffer, ggml_tensor * tensor, uint8_t value, size_t offset, size_t size) { + ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; + rpc_msg_memset_tensor_req request = { + /* .tensor = */ serialize_tensor(tensor), + /* .offset = */ offset, + /* .size = */ size, + /* .value = */ value, + }; + bool status = send_rpc_cmd(ctx->sock, RPC_CMD_MEMSET_TENSOR, &request, sizeof(request), nullptr, 0); + RPC_STATUS_ASSERT(status); +} + static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) { ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context; rpc_tensor rpc_tensor = serialize_tensor(tensor); @@ -531,7 +552,7 @@ static ggml_backend_buffer_i ggml_backend_rpc_buffer_interface = { /* .free_buffer = */ ggml_backend_rpc_buffer_free_buffer, /* .get_base = */ ggml_backend_rpc_buffer_get_base, /* .init_tensor = */ ggml_backend_rpc_buffer_init_tensor, - /* .memset_tensor = */ NULL, + /* .memset_tensor = */ ggml_backend_rpc_buffer_memset_tensor, /* .set_tensor = */ ggml_backend_rpc_buffer_set_tensor, /* .get_tensor = */ ggml_backend_rpc_buffer_get_tensor, /* .set_tensor_2d = */ NULL, @@ -831,6 +852,7 @@ public: bool buffer_get_base(const rpc_msg_buffer_get_base_req & request, rpc_msg_buffer_get_base_rsp & response); bool free_buffer(const rpc_msg_free_buffer_req & request); bool buffer_clear(const rpc_msg_buffer_clear_req & request); + bool memset_tensor(const rpc_msg_memset_tensor_req & request); bool set_tensor(const std::vector & input); bool set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rpc_msg_set_tensor_hash_rsp & response); bool get_tensor(const rpc_msg_get_tensor_req & request, std::vector & response); @@ -989,6 +1011,52 @@ bool rpc_server::buffer_clear(const rpc_msg_buffer_clear_req & request) { return true; } +bool rpc_server::memset_tensor(const rpc_msg_memset_tensor_req & request) { + struct ggml_init_params params { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx_ptr { ggml_init(params) }; + GGML_ASSERT(ctx_ptr != nullptr); + ggml_context * ctx = ctx_ptr.get(); + ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor); + if (tensor == nullptr || tensor->buffer == nullptr) { + GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__); + return false; + } + + const uint64_t tensor_size = ggml_nbytes(tensor); + if (request.offset > tensor_size || request.size > tensor_size - request.offset) { + GGML_LOG_ERROR("[%s] tensor region (offset=%" PRIu64 ", size=%" PRIu64 ") out of tensor bounds [0, %" PRIu64 ")\n", + __func__, request.offset, request.size, tensor_size); + return false; + } + + const uint64_t buffer_start = (uint64_t) ggml_backend_buffer_get_base(tensor->buffer); + const uint64_t buffer_size = ggml_backend_buffer_get_size(tensor->buffer); + if (request.tensor.data < buffer_start) { + GGML_LOG_ERROR("[%s] tensor data before buffer start\n", __func__); + return false; + } + const uint64_t data_offset = request.tensor.data - buffer_start; + if (data_offset > buffer_size || + request.offset > buffer_size - data_offset || + request.size > buffer_size - data_offset - request.offset) { + GGML_LOG_ERROR("[%s] tensor region out of buffer bounds\n", __func__); + return false; + } + if (tensor->buffer->iface.memset_tensor == nullptr) { + GGML_LOG_ERROR("[%s] memset not implemented by backend buffer\n", __func__); + return false; + } + + LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 ", value: %u\n", + __func__, (void *) tensor->buffer, tensor->data, request.offset, request.size, request.value); + ggml_backend_tensor_memset(tensor, request.value, request.offset, request.size); + return true; +} + ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor) { // Validate tensor type before using it if (tensor->type >= GGML_TYPE_COUNT) { @@ -1585,6 +1653,19 @@ static void rpc_serve_client(const std::vector & backends, const } break; } + case RPC_CMD_MEMSET_TENSOR: { + rpc_msg_memset_tensor_req request; + if (!recv_msg(sock, &request, sizeof(request))) { + return; + } + if (!server.memset_tensor(request)) { + return; + } + if (!send_msg(sock, nullptr, 0)) { + return; + } + break; + } case RPC_CMD_SET_TENSOR: { std::vector input; if (!recv_msg(sock, input)) { diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index 1c17d20df1..a8d9c0d804 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -199,9 +199,20 @@ if (GGML_SYCL_DEVICE_ARCH) -fsycl-targets=spir64_gen "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH}\"" ) + + # Pass through parallel job (process) count for parallelising the + # `llvm-foreach -- ocloc` invocation for compiling AOT device images. + include(ProcessorCount) + ProcessorCount(_ggml_sycl_nproc) + if (_ggml_sycl_nproc LESS 1) + set(_ggml_sycl_nproc 1) + endif() + set(GGML_SYCL_MAX_PARALLEL_LINK_JOBS ${_ggml_sycl_nproc} CACHE STRING + "Parallel ocloc jobs for spir64_gen AOT device-image lowering") target_link_options( ggml-sycl PRIVATE -fsycl-targets=spir64_gen "SHELL:-Xsycl-target-backend=spir64_gen \"-device ${GGML_SYCL_DEVICE_ARCH}\"" + -fsycl-max-parallel-link-jobs=${GGML_SYCL_MAX_PARALLEL_LINK_JOBS} ) endif() diff --git a/ggml/src/ggml-sycl/backend.hpp b/ggml/src/ggml-sycl/backend.hpp index f299bcf62e..51ab6f930d 100644 --- a/ggml/src/ggml-sycl/backend.hpp +++ b/ggml/src/ggml-sycl/backend.hpp @@ -26,6 +26,7 @@ #include "dmmv.hpp" #include "element_wise.hpp" #include "fattn.hpp" +#include "fusion.hpp" #include "gated_delta_net.hpp" #include "gla.hpp" #include "im2col.hpp" diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index e5d9ee89dd..160331191d 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -65,6 +65,7 @@ extern int g_ggml_sycl_prioritize_dmmv; extern int g_ggml_sycl_enable_flash_attention; extern int g_ggml_sycl_dev2dev_memcpy; extern int g_ggml_sycl_fa_onednn; +extern int g_ggml_sycl_fa_onednn_max_kv; #if defined(__clang__) && __has_builtin(__builtin_expect) @@ -132,6 +133,7 @@ enum ggml_sycl_backend_gpu_mode { enum ggml_sycl_dev2dev_memcpy_mode { DEV2DEV_MEMCPY_SYCL = 0, DEV2DEV_MEMCPY_L0 = 1, + DEV2DEV_MEMCPY_FORWARD = 2 }; static_assert(sizeof(sycl::half) == sizeof(ggml_fp16_t), "wrong fp16 size"); diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp index 060d0aca2d..9ec9276952 100644 --- a/ggml/src/ggml-sycl/convert.cpp +++ b/ggml/src/ggml-sycl/convert.cpp @@ -644,6 +644,8 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) { switch (type) { case GGML_TYPE_Q1_0: return dequantize_block_sycl; + case GGML_TYPE_Q2_0: + return dequantize_block_sycl; case GGML_TYPE_Q4_0: if (dst->src[0]->extra && ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { @@ -728,6 +730,8 @@ to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) { switch (type) { case GGML_TYPE_Q1_0: return dequantize_block_sycl; + case GGML_TYPE_Q2_0: + return dequantize_block_sycl; case GGML_TYPE_Q4_0: if (dst->src[0]->extra && ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { diff --git a/ggml/src/ggml-sycl/cpy.cpp b/ggml/src/ggml-sycl/cpy.cpp index 5d0f9a89fd..55e0761722 100644 --- a/ggml/src/ggml-sycl/cpy.cpp +++ b/ggml/src/ggml-sycl/cpy.cpp @@ -8,7 +8,6 @@ #include "ggml-sycl/presets.hpp" #include "ggml.h" - static void cpy_1_f32_f32(const char * cxi, char * cdsti) { const float * xi = (const float *) cxi; float * dsti = (float *) cdsti; @@ -151,6 +150,20 @@ static void cpy_blck_q8_0_f32(const char * cxi, char * cdsti) { } } +static void cpy_blck_q2_0_f32(const char * cxi, char * cdsti) { + const block_q2_0 * xi = (const block_q2_0 *) cxi; + float * cdstf = (float *) cdsti; + + const float d = xi->d; + + for (int j = 0; j < QK2_0; ++j) { + const int byte_index = j / 4; + const int bit_offset = (j % 4) * 2; + const int q = (xi->qs[byte_index] >> bit_offset) & 0x3; + cdstf[j] = (float) (q - 1) * d; + } +} + template static void cpy_blck_q_f32(const char * cxi, char * cdsti) { @@ -256,7 +269,7 @@ static void ggml_cpy_f16_f32_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -274,7 +287,7 @@ static void ggml_cpy_f32_f32_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -292,7 +305,7 @@ static void ggml_cpy_f32_f16_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -308,7 +321,7 @@ static void ggml_cpy_f32_i32_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -324,7 +337,7 @@ static void ggml_cpy_i32_f32_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -338,7 +351,7 @@ static void ggml_cpy_f32_q8_0_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK8_0 == 0); const int num_blocks = ne / QK8_0; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -350,12 +363,25 @@ static void ggml_cpy_q8_0_f32_sycl(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ne; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } +static void ggml_cpy_q2_0_f32_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ne; + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + cpy_q_f32(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, + ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + static void ggml_cpy_f32_q4_0_sycl(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, @@ -363,7 +389,7 @@ static void ggml_cpy_f32_q4_0_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK4_0 == 0); const int num_blocks = ne / QK4_0; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -375,7 +401,8 @@ static void ggml_cpy_q4_0_f32_sycl(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ne; stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32, QK4_0>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -389,7 +416,7 @@ static void ggml_cpy_f32_q4_1_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK4_1 == 0); const int num_blocks = ne / QK4_1; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -401,7 +428,8 @@ static void ggml_cpy_q4_1_f32_sycl(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ne; stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32, QK4_1>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -415,7 +443,7 @@ static void ggml_cpy_f32_q5_0_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK5_0 == 0); const int num_blocks = ne / QK5_0; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -427,7 +455,8 @@ static void ggml_cpy_q5_0_f32_sycl(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ne; stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32, QK5_0>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -441,7 +470,7 @@ static void ggml_cpy_f32_q5_1_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK5_1 == 0); const int num_blocks = ne / QK5_1; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -453,7 +482,8 @@ static void ggml_cpy_q5_1_f32_sycl(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ne; stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_f32, QK5_1>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -466,7 +496,8 @@ static void ggml_cpy_mxfp4_f32_sycl(const char * cx, char * cdst, const int ne, const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ne; stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { cpy_q_f32, QK_MXFP4>(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -480,7 +511,8 @@ static void ggml_cpy_f32_iq4_nl_sycl(const char * cx, char * cdst, const int ne, GGML_ASSERT(ne % QK4_NL == 0); const int num_blocks = ne / QK4_NL; stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -526,7 +558,7 @@ static void ggml_cpy_f16_q4_0_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK4_0 == 0); const int num_blocks = ne / QK4_0; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -540,7 +572,7 @@ static void ggml_cpy_f16_q4_1_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK4_1 == 0); const int num_blocks = ne / QK4_1; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -554,7 +586,7 @@ static void ggml_cpy_f16_q5_0_sycl(const char * cx, char * cdst, const int ne, c GGML_ASSERT(ne % QK5_0 == 0); const int num_blocks = ne / QK5_0; stream->parallel_for(sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks), sycl::range<3>(1, 1, 1)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); @@ -564,6 +596,7 @@ static void ggml_cpy_f16_q5_0_sycl(const char * cx, char * cdst, const int ne, c static bool ggml_sycl_is_quantized_type(enum ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -594,6 +627,7 @@ static bool ggml_sycl_is_quantized_type(enum ggml_type type) { static bool ggml_sycl_can_quantize_rows_sycl(enum ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -651,7 +685,8 @@ static void ggml_sycl_quantize_rows_q(const char * cx, char * cdst, const int64_ constexpr int block_size = 256; const int64_t grid_size = ceil_div(total_blocks, (int64_t) block_size); - stream->parallel_for(sycl::nd_range<1>(grid_size * block_size, block_size), [=](sycl::nd_item<1> item_ct1) { + stream->parallel_for(sycl::nd_range<1>(grid_size * block_size, block_size), + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { const int64_t block_idx = item_ct1.get_global_linear_id(); if (block_idx >= total_blocks) { return; @@ -708,6 +743,11 @@ static void ggml_sycl_quantize_rows_sycl(const char * cx, char * cdst, const ggm nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, stream); break; + case GGML_TYPE_Q2_0: + ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, + nb02, nb03, ne10, ne11, ne12, nb10, nb11, + nb12, nb13, stream); + break; case GGML_TYPE_Q5_1: ggml_sycl_quantize_rows_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, @@ -760,7 +800,7 @@ static void ggml_cpy_f16_f16_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -779,7 +819,7 @@ static void ggml_cpy_i16_i16_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -798,7 +838,7 @@ static void ggml_cpy_i32_i32_sycl(const char * cx, char * cdst, const int ne, co stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -812,7 +852,8 @@ static void ggml_cpy_q8_0_q8_0(const char * cx, char * cdst, const int ne, const const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), - sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -825,7 +866,8 @@ static void ggml_cpy_q5_0_q5_0(const char * cx, char * cdst, const int ne, const const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), - sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -839,7 +881,8 @@ static void ggml_cpy_q5_1_q5_1(const char * cx, char * cdst, const int ne, const stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), - sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -851,7 +894,8 @@ static void ggml_cpy_q4_0_q4_0(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -864,7 +908,8 @@ static void ggml_cpy_q4_1_q4_1(const char * cx, char * cdst, const int ne, const const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -875,18 +920,32 @@ static void ggml_cpy_q1_0_q1_0(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } +static void ggml_cpy_q2_0_q2_0(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, + const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, + const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, + const int nb12, const int nb13, queue_ptr stream) { + const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); + stream->parallel_for( + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ + cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); + }); +} + static void ggml_cpy_mxfp4_mxfp4(const char * cx, char * cdst, const int ne, const int ne00, const int ne01, const int ne02, const int nb00, const int nb01, const int nb02, const int nb03, const int ne10, const int ne11, const int ne12, const int nb10, const int nb11, const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -897,7 +956,8 @@ static void ggml_cpy_nvfp4_nvfp4(const char * cx, char * cdst, const int ne, con const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -908,7 +968,8 @@ static void ggml_cpy_q2_K_q2_K(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -919,7 +980,8 @@ static void ggml_cpy_q3_K_q3_K(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -930,7 +992,8 @@ static void ggml_cpy_q4_K_q4_K(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -941,7 +1004,8 @@ static void ggml_cpy_q5_K_q5_K(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -952,7 +1016,8 @@ static void ggml_cpy_q6_K_q6_K(const char * cx, char * cdst, const int ne, const const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -963,7 +1028,8 @@ static void ggml_cpy_iq2_xxs_iq2_xxs(const char * cx, char * cdst, const int ne, const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -974,7 +1040,8 @@ static void ggml_cpy_iq2_xs_iq2_xs(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -985,7 +1052,8 @@ static void ggml_cpy_iq2_s_iq2_s(const char * cx, char * cdst, const int ne, con const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -996,7 +1064,8 @@ static void ggml_cpy_iq3_xxs_iq3_xxs(const char * cx, char * cdst, const int ne, const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -1007,7 +1076,8 @@ static void ggml_cpy_iq1_s_iq1_s(const char * cx, char * cdst, const int ne, con const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -1018,7 +1088,8 @@ static void ggml_cpy_iq1_m_iq1_m(const char * cx, char * cdst, const int ne, con const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -1029,7 +1100,8 @@ static void ggml_cpy_iq4_nl_iq4_nl(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -1040,7 +1112,8 @@ static void ggml_cpy_iq3_s_iq3_s(const char * cx, char * cdst, const int ne, con const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -1051,7 +1124,8 @@ static void ggml_cpy_iq4_xs_iq4_xs(const char * cx, char * cdst, const int ne, c const int nb12, const int nb13, queue_ptr stream) { const int num_blocks = ceil_div(ne, SYCL_CPY_BLOCK_SIZE); stream->parallel_for( - sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), [=](sycl::nd_item<3> item_ct1) { + sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_q_q(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); } @@ -1065,7 +1139,7 @@ static void ggml_cpy_f32_bf16_sycl(const char * cx, char * cdst, const int ne, c stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -1079,7 +1153,7 @@ static void ggml_cpy_bf16_f32_sycl(const char * cx, char * cdst, const int ne, c stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -1093,7 +1167,7 @@ static void ggml_cpy_bf16_bf16_sycl(const char * cx, char * cdst, const int ne, stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -1107,7 +1181,7 @@ static void ggml_cpy_f16_bf16_sycl(const char * cx, char * cdst, const int ne, c stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -1121,7 +1195,7 @@ static void ggml_cpy_bf16_f16_sycl(const char * cx, char * cdst, const int ne, c stream->parallel_for( sycl::nd_range<3>(sycl::range<3>(1, 1, num_blocks) * sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE), sycl::range<3>(1, 1, SYCL_CPY_BLOCK_SIZE)), - [=](sycl::nd_item<3> item_ct1) { + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]]{ cpy_f32_f16(cx, cdst, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, item_ct1); }); @@ -1213,6 +1287,9 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co } else if (src0->type == GGML_TYPE_Q8_0 && src1->type == GGML_TYPE_F32) { ggml_cpy_q8_0_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q2_0 && src1->type == GGML_TYPE_F32) { + ggml_cpy_q2_0_f32_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, + nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_Q5_0) { ggml_cpy_f32_q5_0_sycl(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); @@ -1243,6 +1320,8 @@ void ggml_sycl_cpy(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, co ggml_cpy_q4_1_q4_1(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_Q1_0 && src1->type == GGML_TYPE_Q1_0) { ggml_cpy_q1_0_q1_0(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); + } else if (src0->type == GGML_TYPE_Q2_0 && src1->type == GGML_TYPE_Q2_0) { + ggml_cpy_q2_0_q2_0(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_MXFP4 && src1->type == GGML_TYPE_MXFP4) { ggml_cpy_mxfp4_mxfp4(src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } else if (src0->type == GGML_TYPE_NVFP4 && src1->type == GGML_TYPE_NVFP4) { diff --git a/ggml/src/ggml-sycl/cpy.hpp b/ggml/src/ggml-sycl/cpy.hpp index c4cfd961da..34bae1b2dd 100644 --- a/ggml/src/ggml-sycl/cpy.hpp +++ b/ggml/src/ggml-sycl/cpy.hpp @@ -70,6 +70,39 @@ inline void cpy_blck_f32_q1_0(const char * cxi, char * cdsti) { } } +inline int round_nearest_int(float x) { + return (int)(x >= 0.0f ? x + 0.5f : x - 0.5f); +} + +inline void cpy_blck_f32_q2_0(const char * cxi, char * cdsti) { + const float * xi = (const float *) cxi; + block_q2_0 * dsti = (block_q2_0 *) cdsti; + + float amax = 0.0f; + + for (int j = 0; j < QK2_0; ++j) { + amax = sycl::fmax(amax, sycl::fabs((float) xi[j])); + } + + const float d = amax; + const float id = d > 0.0f ? 1.0f / d : 0.0f; + + dsti->d = d; + + for (int j = 0; j < QK2_0 / 4; ++j) { + dsti->qs[j] = 0; + } + + for (int j = 0; j < QK2_0; ++j) { + int q = round_nearest_int(xi[j] * id) + 1; + q = dpct::max(0, dpct::min(3, q)); + + const int byte_index = j / 4; + const int bit_offset = (j % 4) * 2; + dsti->qs[byte_index] |= (uint8_t) q << bit_offset; + } +} + inline int best_index_mxfp4(const float x, const float e) { int best_index = 0; float best_err = sycl::fabs((float) (kvalues_mxfp4[0] * e - x)); diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp index 3db55319fe..876ba1b444 100644 --- a/ggml/src/ggml-sycl/dequantize.hpp +++ b/ggml/src/ggml-sycl/dequantize.hpp @@ -25,6 +25,28 @@ typedef void (*dequantize_kernel_f32_t)(const void * vx, const int64_t ib, const static inline void get_scale_min_k4(int j, const uint8_t * q, uint8_t & d, uint8_t & m); #endif +static __dpct_inline__ void dequantize_q2_0(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { + const block_q2_0 * x = (const block_q2_0 *) vx; + + const dfloat d = x[ib].d; + + const int byte_idx = iqs / 4; + const int shift = (iqs % 4) * 2; + const uint8_t vui = x[ib].qs[byte_idx]; + + v.x() = (vui >> shift) & 3; + v.y() = (vui >> (shift + 2)) & 3; + +#ifdef GGML_SYCL_F16 + v.s0() = ((dfloat)v.s0() - 1.0f) * d; + v.s1() = ((dfloat)v.s1() - 1.0f) * d; +#else + v.x() = ((dfloat)v.x() - 1.0f) * d; + v.y() = ((dfloat)v.y() - 1.0f) * d; +#endif // GGML_SYCL_F16 +} + static __dpct_inline__ void dequantize_q4_0(const void *vx, const int64_t ib, const int iqs, dfloat2 &v) { const block_q4_0 * x = (const block_q4_0 *) vx; diff --git a/ggml/src/ggml-sycl/element_wise.cpp b/ggml/src/ggml-sycl/element_wise.cpp index b2406e11b5..3cd055494e 100644 --- a/ggml/src/ggml-sycl/element_wise.cpp +++ b/ggml/src/ggml-sycl/element_wise.cpp @@ -306,29 +306,43 @@ static __dpct_inline__ T op_trunc(T x) { } } +template +static void unary_op_flat_kernel(const T * x, T * dst, const int k, const sycl::nd_item<1> & item_ct1, F func) { + SYCL_GLOBAL_ID_LOOP(k, item_ct1) { + dst[i] = func(x[i]); + } +} + template static void unary_op_generic_kernel( const T * x, T * dst, const int k, - const int64_t ne0, const int64_t ne1, const int64_t ne2, const int64_t ne3, + const sycl::uint3 ne0_fd, const sycl::uint3 ne1_fd, const sycl::uint3 ne2_fd, const size_t nb0, const size_t nb1, const size_t nb2, const size_t nb3, const size_t nbd0, const size_t nbd1, const size_t nbd2, const size_t nbd3, const sycl::nd_item<1> & item_ct1, F func) { - (void) ne3; + // 32-bit index math: k is int, so every logical index fits u32. 64-bit integer div/mod is + // emulated on Xe and dominates this kernel otherwise, and even the 32-bit divide is worth + // avoiding -- the divisors are launch-invariant, so the magic numbers are precomputed + // host-side and each division becomes a multiply-high plus a shift. + // Byte offsets are widened back to size_t only for the final address math. SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - const int64_t i0 = i % ne0; - const int64_t i1 = (i / ne0) % ne1; - const int64_t i2 = (i / (ne0*ne1)) % ne2; - const int64_t i3 = i / (ne0*ne1*ne2); + sycl::uint2 dm = fast_div_modulo((uint32_t) i, ne0_fd); + const uint32_t i0 = dm.y(); + dm = fast_div_modulo(dm.x(), ne1_fd); + const uint32_t i1 = dm.y(); + dm = fast_div_modulo(dm.x(), ne2_fd); + const uint32_t i2 = dm.y(); + const uint32_t i3 = dm.x(); const char * src_base = (const char *) x; char * dst_base = (char *) dst; - const T * srcp = (const T *)(src_base + i0*nb0 + i1*nb1 + i2*nb2 + i3*nb3 ); - T * dstp = (T *)(dst_base + i0*nbd0 + i1*nbd1 + i2*nbd2 + i3*nbd3); + const T * srcp = (const T *)(src_base + (size_t) i0*nb0 + (size_t) i1*nb1 + (size_t) i2*nb2 + (size_t) i3*nb3 ); + T * dstp = (T *)(dst_base + (size_t) i0*nbd0 + (size_t) i1*nbd1 + (size_t) i2*nbd2 + (size_t) i3*nbd3); *dstp = func(*srcp); } @@ -407,46 +421,51 @@ static void clamp(const T * x, T * dst, const float min, const float max, const } template -static void gated_op_fused_geglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { +static void gated_op_fused_geglu(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - const int64_t j0 = (i / n) * o0 + (i % n); - const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); + const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd); + const int64_t j0 = rc.x() * o0 + rc.y(); + const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y(); dst[i] = op_gelu(x[j0]) * g[j1]; } } template -static void gated_op_fused_reglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { +static void gated_op_fused_reglu(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - const int64_t j0 = (i / n) * o0 + (i % n); - const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); + const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd); + const int64_t j0 = rc.x() * o0 + rc.y(); + const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y(); dst[i] = op_relu(x[j0]) * g[j1]; } } template -static void gated_op_fused_swiglu(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { +static void gated_op_fused_swiglu(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - const int64_t j0 = (i / n) * o0 + (i % n); - const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); + const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd); + const int64_t j0 = rc.x() * o0 + rc.y(); + const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y(); dst[i] = op_silu(x[j0]) * g[j1]; } } template -static void gated_op_fused_geglu_erf(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { +static void gated_op_fused_geglu_erf(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - const int64_t j0 = (i / n) * o0 + (i % n); - const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); + const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd); + const int64_t j0 = rc.x() * o0 + rc.y(); + const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y(); dst[i] = op_gelu_erf(x[j0]) * g[j1]; } } template -static void gated_op_fused_geglu_quick(const T * x, const T * g, T * dst, const uint64_t k, const uint64_t n, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { +static void gated_op_fused_geglu_quick(const T * x, const T * g, T * dst, const uint64_t k, const sycl::uint3 n_fd, const uint64_t o0, const uint64_t o1, const sycl::nd_item<1> &item_ct1) { SYCL_GLOBAL_ID_LOOP(k, item_ct1) { - const int64_t j0 = (i / n) * o0 + (i % n); - const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); + const sycl::uint2 rc = fast_div_modulo((uint32_t) i, n_fd); + const int64_t j0 = rc.x() * o0 + rc.y(); + const int64_t j1 = o0 == o1 ? j0 : rc.x() * o1 + rc.y(); dst[i] = op_gelu_quick(x[j0]) * g[j1]; } } @@ -529,6 +548,10 @@ static inline void dispatch_ggml_sycl_op_fused_glu(ggml_backend_sycl_context & c GGML_ASSERT(dst->ne[0] == nc); GGML_ASSERT(ggml_is_contiguous_1(dst->src[0])); GGML_ASSERT(ggml_is_contiguous(dst)); + // The fused GLU kernels index with 32-bit fastdiv, which is exact only for indices below + // 2^31. A dst that large is ~8 GB at f32, and the grid sizing already narrows to 32 bits, + // so assert the bound rather than carry a second code path for it. + GGML_ASSERT(ggml_nelements(dst) < ((int64_t) 1 << 31)); const int32_t swapped = ((const int32_t *) dst->op_params)[1]; void * src0_d = src0->data; void * src1_d = src1 ? src1->data : src0->data; @@ -597,7 +620,6 @@ static inline void ggml_sycl_op_unary( const int64_t ne0 = dst->ne[0]; const int64_t ne1 = dst->ne[1]; const int64_t ne2 = dst->ne[2]; - const int64_t ne3 = dst->ne[3]; const size_t nb0 = src0->nb[0]; const size_t nb1 = src0->nb[1]; @@ -609,24 +631,42 @@ static inline void ggml_sycl_op_unary( const size_t nbd2 = dst->nb[2]; const size_t nbd3 = dst->nb[3]; + // Hot unary ops (FFN/GDN silu, sigmoid, ...) run on contiguous tensors; + // skip the strided index math entirely for them. + const bool contiguous = ggml_is_contiguous(src0) && ggml_is_contiguous(dst); + ggml_sycl_detail::dispatch_ggml_sycl_op_unary(ctx, dst, [=](const auto* src, auto* dst_ptr, int k_elements, queue_ptr stream) { const int num_blocks = ceil_div(k_elements, 256); - stream->parallel_for( - sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256), - sycl::range<1>(256)), - [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - unary_op_generic_kernel( - src, dst_ptr, k_elements, - ne0, ne1, ne2, ne3, - nb0, nb1, nb2, nb3, - nbd0, nbd1, nbd2, nbd3, - item_ct1, - func - ); - }); + if (contiguous) { + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256), + sycl::range<1>(256)), + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + unary_op_flat_kernel(src, dst_ptr, k_elements, item_ct1, func); + }); + } else { + // Launch-invariant divisors: compute the magic numbers once on the host so the + // kernel never issues an integer divide. Only the strided path needs them. + const sycl::uint3 ne0_fd = init_fastdiv_values((uint32_t) ne0); + const sycl::uint3 ne1_fd = init_fastdiv_values((uint32_t) ne1); + const sycl::uint3 ne2_fd = init_fastdiv_values((uint32_t) ne2); + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>(num_blocks) * sycl::range<1>(256), + sycl::range<1>(256)), + [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + unary_op_generic_kernel( + src, dst_ptr, k_elements, + ne0_fd, ne1_fd, ne2_fd, + nb0, nb1, nb2, nb3, + nbd0, nbd1, nbd2, nbd3, + item_ct1, + func + ); + }); + } }); } @@ -930,10 +970,11 @@ static inline void ggml_sycl_op_geglu(ggml_backend_sycl_context & ctx, ggml_tens ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst, [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) { const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE); + const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n); main_stream->parallel_for( sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - gated_op_fused_geglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1); + gated_op_fused_geglu(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1); }); }); } @@ -942,10 +983,11 @@ static inline void ggml_sycl_op_reglu(ggml_backend_sycl_context & ctx, ggml_tens ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst, [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) { const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_RELU_BLOCK_SIZE); // Using RELU block size for reglu + const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n); main_stream->parallel_for( sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_RELU_BLOCK_SIZE)), sycl::range<1>(SYCL_RELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - gated_op_fused_reglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1); + gated_op_fused_reglu(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1); }); }); } @@ -954,10 +996,11 @@ static inline void ggml_sycl_op_swiglu(ggml_backend_sycl_context & ctx, ggml_ten ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst, [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) { const uint32_t num_blocks = ceil_div((uint32_t)k, SYCL_SILU_BLOCK_SIZE); // Using SILU block size for swiglu + const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n); main_stream->parallel_for( sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_SILU_BLOCK_SIZE)), sycl::range<1>(SYCL_SILU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - gated_op_fused_swiglu(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1); + gated_op_fused_swiglu(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1); }); }); } @@ -1057,10 +1100,11 @@ static inline void ggml_sycl_op_geglu_erf(ggml_backend_sycl_context & ctx, ggml_ ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst, [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) { const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE); + const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n); main_stream->parallel_for( sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - gated_op_fused_geglu_erf(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1); + gated_op_fused_geglu_erf(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1); }); }); } @@ -1069,10 +1113,11 @@ static inline void ggml_sycl_op_geglu_quick(ggml_backend_sycl_context & ctx, ggm ggml_sycl_detail::dispatch_ggml_sycl_op_fused_glu(ctx, dst, [](const auto* x_ptr, const auto* g_ptr, auto* dst_ptr, uint64_t k, uint64_t n, uint64_t o0, uint64_t o1, queue_ptr main_stream) { const uint32_t num_blocks = ceil_div(k, SYCL_GELU_BLOCK_SIZE); + const sycl::uint3 n_fd = init_fastdiv_values((uint32_t) n); main_stream->parallel_for( sycl::nd_range<1>((num_blocks * sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), sycl::range<1>(SYCL_GELU_BLOCK_SIZE)), [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - gated_op_fused_geglu_quick(x_ptr, g_ptr, dst_ptr, k, n, o0, o1, item_ct1); + gated_op_fused_geglu_quick(x_ptr, g_ptr, dst_ptr, k, n_fd, o0, o1, item_ct1); }); }); } diff --git a/ggml/src/ggml-sycl/fattn-mkl.cpp b/ggml/src/ggml-sycl/fattn-mkl.cpp new file mode 100644 index 0000000000..fc22b7bdb8 --- /dev/null +++ b/ggml/src/ggml-sycl/fattn-mkl.cpp @@ -0,0 +1,690 @@ +// Flash attention via oneMKL GEMM (XMX-accelerated). +// Uses column_major::gemm for Q*K^T and S*V matmuls +// with an online softmax SYCL kernel. +// +// All GQA query heads sharing a KV head are batched into single +// GEMM calls, amortizing MKL launch overhead across K and V reuse. +// + +#include "common.hpp" +#include "fattn-common.hpp" +#include "fattn-buffers.hpp" +#include "convert.hpp" +#include "fattn.hpp" + +#include +#include +#include + +#define MKL_FA_CHUNK_SIZE_KV 8192 + +// Number of query rows processed per tile. The score buffers (KQ_f32, S_f16) +// are sized q_tile_rows * chunk_size, so this bounds their footprint +// regardless of batch size (n_query_rows = n_queries * gqa_ratio). A typical +// single-ubatch prefill (e.g. ubatch 1024 * gqa 8 = 8192 rows) is exactly one +// tile, so it runs with no extra iterations. Larger batches tile and stay +// bounded. Override with GGML_SYCL_MKL_FA_Q_TILE. +#define MKL_FA_Q_TILE 8192 + +#define MKL_FA_WG_SIZE 256 + +using oneapi::mkl::transpose; +using oneapi::mkl::blas::column_major::gemm; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// Pack all GQA Q heads for one KV head into fp16, applying q_scale. +// Launches one kernel per GQA group — each kernel copies exactly +// n_queries * DKQ elements using the per-group dst offset and +// per-head source stride. +static void mkl_fa_pack_q_fp16( + dpct::queue_ptr stream, + sycl::half * __restrict dst, + const float * __restrict q_src, + int n_queries, int n_query_rows, int DKQ, + int gqa_ratio, int kvh_base_head, + float q_scale, int64_t q_row_stride, int64_t q_head_stride, + int64_t wg_size) { + + for (int iqg = 0; iqg < gqa_ratio; iqg++) { + int iqh = kvh_base_head + iqg; + sycl::half * dst_g = dst + (int64_t)iqg * n_queries * DKQ; + + const int64_t n_elem = (int64_t)n_queries * DKQ; + const int64_t wg = ((n_elem + wg_size - 1) / wg_size) * wg_size; + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int64_t e = item.get_global_id(0); + if (e >= n_elem) return; + + int64_t q = e / DKQ; + int64_t d = e - q * DKQ; + + // Stride-aware source offset: handles permuted, + // sliced, or contiguous Q tensor layouts. + int64_t src_off = d + + q * q_row_stride + + (int64_t)iqh * q_head_stride; + + dst_g[e] = sycl::half( + q_src[src_off] * q_scale); + }); + }); + } +} + +// Zero-initialize the online softmax state arrays. +// KQ_max → -inf, KQ_sum → 0, VKQ_accum → 0. +// Merged into one kernel to avoid per-array launch overhead. +static void mkl_fa_init_softmax_state( + dpct::queue_ptr stream, + float * kmax, float * ksum, float * vacc, + int n_query_rows, int DV, int64_t wg_size) { + + const float neg_inf = -1e30f; + const int64_t n_maxsum = n_query_rows; + const int64_t n_vacc = (int64_t)n_query_rows * DV; + const int64_t total = (n_vacc > n_maxsum) ? n_vacc : n_maxsum; + const int64_t wg = ((total + wg_size - 1) / wg_size) * wg_size; + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int64_t i = item.get_global_id(0); + if (i < n_maxsum) { + kmax[i] = neg_inf; + ksum[i] = 0.0f; + } + if (i < n_vacc) { + vacc[i] = 0.0f; + } + }); + }); +} + +// Online softmax over one KV chunk for a tile of GQA query rows. +// The tile spans absolute rows [q0, q0 + q_rows). Score buffers +// (KQ_f32/S_f16) are indexed RELATIVE to the tile; the persistent state +// (VKQ_accum/KQ_max/KQ_sum) and mask are indexed by ABSOLUTE row. +// For each row: find local max → rescale previous VKQ_accum → +// compute exp(s - max) → write S_f16 → update running max/sum. +static void mkl_fa_online_softmax_chunk( + dpct::queue_ptr stream, + float * __restrict KQ_f32, + sycl::half * __restrict S_f16, + float * __restrict KQ_max, + float * __restrict KQ_sum, + float * __restrict VKQ_accum, + int q0, int q_rows, int n_queries, int DV, + int chunk_size, int chunk_start, + int kvh_head, int gqa_ratio, + const sycl::half * mask_data, int64_t mask_head_stride, + int64_t mask_row_stride, int mask_n_heads, + float logit_softcap, int64_t wg_size) { + + const int64_t wg = ((q_rows + wg_size - 1) / wg_size) * wg_size; + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int jc_rel = item.get_global_id(0); + if (jc_rel >= q_rows) return; + int jc_abs = q0 + jc_rel; + + const int gqa_group = jc_abs / n_queries; + const int q_row = jc_abs % n_queries; + + // Score buffers are tile-local (relative index). + const float * __restrict KQ_row = KQ_f32 + + jc_rel * (int64_t)chunk_size; + // Persistent accumulator is full-sized (absolute index). + float * __restrict vkq = VKQ_accum + + jc_abs * (int64_t)DV; + + const sycl::half * mask_h = nullptr; + int64_t m_stride = 0; + if (mask_data) { + int m_head = (mask_n_heads > 1) + ? (kvh_head + gqa_group) : 0; + mask_h = mask_data + (int64_t)m_head * mask_head_stride; + m_stride = mask_row_stride; + } + + // Row-wise local maximum (softcap before mask) + float local_max = -1e30f; + for (int i = 0; i < chunk_size; i++) { + float s = KQ_row[i]; + if (logit_softcap != 0.0f) { + s = logit_softcap * sycl::tanh(s); + } + if (mask_h) { + s += (float)mask_h[q_row * m_stride + + (chunk_start + i)]; + } + if (s > local_max) local_max = s; + } + + // Rescale previous accumulator by exp(old_max - new_max) + float old_max = KQ_max[jc_abs]; + float new_max = (old_max > local_max) ? old_max : local_max; + float rescale = (old_max < -1e29f) ? 1.0f + : sycl::native::exp(old_max - new_max); + + for (int v = 0; v < DV; v++) { + vkq[v] *= rescale; + } + + // Softmax and write S_f16 (tile-local index) + float local_sum = 0.0f; + sycl::half * __restrict S_row = S_f16 + + jc_rel * (int64_t)chunk_size; + + for (int i = 0; i < chunk_size; i++) { + float s = KQ_row[i]; + if (logit_softcap != 0.0f) { + s = logit_softcap * sycl::tanh(s); + } + if (mask_h) { + s += (float)mask_h[q_row * m_stride + + (chunk_start + i)]; + } + float val = sycl::native::exp(s - new_max); + S_row[i] = sycl::half(val); + local_sum += val; + } + + KQ_sum[jc_abs] = KQ_sum[jc_abs] * rescale + local_sum; + KQ_max[jc_abs] = new_max; + }); + }); +} + +// Write one GQA group's normalized output to its destination head. +static void mkl_fa_normalize_head( + dpct::queue_ptr stream, + float * __restrict dst_batch, + const float * __restrict VKQ_accum, + const float * __restrict KQ_sum, + int iqh, int n_queries, int DV, int n_q_heads, + int64_t src_offset, int64_t wg_size) { + + const int64_t wg = ((n_queries + wg_size - 1) / wg_size) * wg_size; + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int jc = item.get_global_id(0); + if (jc >= n_queries) return; + + int ksum_idx = (int)(src_offset / DV) + jc; + float inv_sum = 1.0f / KQ_sum[ksum_idx]; + const float * __restrict src = VKQ_accum + + src_offset + jc * (int64_t)DV; + // Interleaved dst layout (matching TILE): + // rows alternate between heads, then increment query. + // offset = (query * n_q_heads + head) * DV + float * __restrict dst_row = dst_batch + + ((int64_t)jc * n_q_heads + iqh) * (int64_t)DV; + + for (int v = 0; v < DV; v++) { + dst_row[v] = src[v] * inv_sum; + } + }); + }); +} + +// --------------------------------------------------------------------------- +// Per-chunk dequant +// +// Rather than dequantizing all of K/V up front (footprint scales with +// context), we dequant one KV-head chunk at a time into a dense +// [this_chunk x D] fp16 buffer (row-major, lda = D). The source address of +// element (head=ikvh, row=chunk_start+r, col=c) decomposes into independent +// linear terms head_off(ikvh) + row_off(chunk_start) + (r,c), so slicing a +// chunk is a clean pointer offset in every layout case. The true-Gemma- +// interleave vs padded-seq-view distinction is resolved once when the +// descriptor is built; slicing does not reintroduce it. +// --------------------------------------------------------------------------- +enum mkl_fa_kv_desc_mode { + MKL_FA_KV_MODE_F16_DENSE = 0, + MKL_FA_KV_MODE_F16_INTERLEAVED = 1, + MKL_FA_KV_MODE_QUANT_CONTIG = 2, + MKL_FA_KV_MODE_QUANT_NC = 3, +}; + +struct mkl_fa_kv_desc { + const char * data = nullptr; + ggml_type type = GGML_TYPE_F16; + int64_t D = 0; // ne[0] + int64_t nb1 = 0; // byte stride, seq dim + int64_t nb2 = 0; // byte stride, head dim + mkl_fa_kv_desc_mode mode = MKL_FA_KV_MODE_F16_DENSE; + int64_t ts = 0; // type size (mode 3 base offset) + int64_t s01 = 0; // nc row stride in blocks (mode 3) + int64_t s02 = 0; // nc head stride in blocks (mode 3) +}; + +static mkl_fa_kv_desc mkl_fa_make_desc(const ggml_tensor * T, bool interleaved, int n_kv_heads) { + mkl_fa_kv_desc d; + d.data = (const char *)T->data; + d.type = T->type; + d.D = T->ne[0]; + d.nb1 = (int64_t)T->nb[1]; + d.nb2 = (int64_t)T->nb[2]; + d.ts = (int64_t)ggml_type_size(T->type); + + if (T->type == GGML_TYPE_F16) { + d.mode = interleaved ? MKL_FA_KV_MODE_F16_INTERLEAVED + : MKL_FA_KV_MODE_F16_DENSE; + } else if (ggml_is_contiguously_allocated(T) && !interleaved) { + d.mode = MKL_FA_KV_MODE_QUANT_CONTIG; + } else { + d.mode = MKL_FA_KV_MODE_QUANT_NC; + const int64_t bs = (int64_t)ggml_blck_size(T->type); + const int64_t blk_per_row = T->ne[0] / bs; + // True Gemma interleave packs heads within a row (nb[2] < ne[1]*nb[1]) + // → reconstruct physical strides. Padded seq-views (nb[2] > ne[1]*nb[1]) + // already have correct physical strides. + const bool gemma = interleaved && + ((int64_t)T->nb[2] < (int64_t)T->ne[1] * (int64_t)T->nb[1]); + if (gemma) { + d.s01 = (int64_t)n_kv_heads * blk_per_row; + d.s02 = blk_per_row; + } else { + d.s01 = d.nb1 / d.ts; + d.s02 = d.nb2 / d.ts; + } + } + return d; +} + +// Dequant one KV-head chunk into a dense [this_chunk x D] fp16 buffer. +static void mkl_fa_dequant_chunk( + dpct::queue_ptr stream, const mkl_fa_kv_desc & d, ggml_tensor * dst_ctx, + sycl::half * out, int ikvh, int chunk_start, int this_chunk) { + + const int64_t D = d.D; + switch (d.mode) { + case MKL_FA_KV_MODE_F16_DENSE: { + const char * base = d.data + (int64_t)ikvh * d.nb2 + + (int64_t)chunk_start * d.nb1; + stream->memcpy(out, base, (size_t)this_chunk * D * sizeof(sycl::half)); + break; + } + case MKL_FA_KV_MODE_F16_INTERLEAVED: { + const char * base = d.data + (int64_t)ikvh * d.nb2 + + (int64_t)chunk_start * d.nb1; + const int64_t row_halfs = d.nb1 / (int64_t)sizeof(sycl::half); + const sycl::half * src = (const sycl::half *)base; + stream->parallel_for( + sycl::range<2>((size_t)this_chunk, (size_t)D), + [=](sycl::item<2> it) { + int64_t r = it.get_id(0); + int64_t c = it.get_id(1); + out[r * D + c] = src[r * row_halfs + c]; + }); + break; + } + case MKL_FA_KV_MODE_QUANT_CONTIG: { + const char * base = d.data + (int64_t)ikvh * d.nb2 + + (int64_t)chunk_start * d.nb1; + to_fp16_sycl_t to_fp16 = ggml_get_to_fp16_sycl(d.type, dst_ctx); + to_fp16(base, out, (int64_t)this_chunk * D, stream); + break; + } + default: { // MKL_FA_KV_MODE_QUANT_NC + to_fp16_nc_sycl_t to_fp16 = ggml_get_to_fp16_nc_sycl(d.type); + const int64_t base_blocks = (int64_t)ikvh * d.s02 + + (int64_t)chunk_start * d.s01; + const char * base = d.data + base_blocks * d.ts; + // ne02 = ne03 = 1 → s02/s03 inert; head+chunk offset carried by base. + to_fp16(base, out, D, this_chunk, 1, 1, d.s01, d.s02, d.s02, stream); + break; + } + } +} + +// --------------------------------------------------------------------------- +// MKL Flash Attention orchestrator +// +// Pipeline: dequantize K/V → for each KV head: +// pack GQA Q heads → MKL GEMM KQ → online softmax → +// MKL GEMM VKQ → accumulate → normalize → scatter to dst +// --------------------------------------------------------------------------- +void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + ggml_tensor * KQV = dst; + + GGML_ASSERT(Q->type == GGML_TYPE_F32); + GGML_ASSERT(KQV->type == GGML_TYPE_F32); + + // --- Op params --- + float scale = 1.0f, max_bias = 0.0f, logit_softcap = 0.0f; + memcpy(&scale, (const float *)KQV->op_params + 0, sizeof(float)); + memcpy(&max_bias, (const float *)KQV->op_params + 1, sizeof(float)); + memcpy(&logit_softcap, (const float *)KQV->op_params + 2, sizeof(float)); + + const float q_scale = scale; + + // --- Dimensions --- + const int DKQ = (int)K->ne[0]; + const int DV = (int)V->ne[0]; + const int n_queries = (int)Q->ne[1]; + const int n_q_heads = (int)Q->ne[2]; + const int n_kv_heads = (int)K->ne[2]; + const int n_batch = (int)Q->ne[3]; + const int n_kv = (int)K->ne[1]; + const int gqa_ratio = n_q_heads / n_kv_heads; + const int n_query_rows = n_queries * gqa_ratio; + + GGML_ASSERT(n_q_heads % n_kv_heads == 0); + GGML_ASSERT(max_bias == 0.0f); // ALiBi not supported + GGML_ASSERT(Q->ne[3] == K->ne[3] || K->ne[3] == 1); + + const int chunk_size = std::min(MKL_FA_CHUNK_SIZE_KV, n_kv); + + // Query rows are processed in tiles of q_tile_rows so the score buffers + // (KQ_f32/S_f16 = q_tile_rows * chunk_size) stay bounded regardless of + // batch size. n_query_rows <= Q_TILE is a single tile (no extra work). + static int q_tile_env = ggml_sycl_get_env("GGML_SYCL_MKL_FA_Q_TILE", MKL_FA_Q_TILE); + const int q_tile_rows = std::max(1, std::min(q_tile_env, n_query_rows)); + + const int64_t wg_size = MKL_FA_WG_SIZE; + + // --- Debug output (gated by GGML_SYCL_MKL_FA_DEBUG=1) --- + static int mkl_call_count = 0; + mkl_call_count++; + static int mkl_debug = ggml_sycl_get_env("GGML_SYCL_MKL_FA_DEBUG", 0); + const bool do_print = (mkl_debug == 1); + + const int64_t q_row_stride = Q->nb[1] / sizeof(float); + const int64_t q_head_stride = Q->nb[2] / sizeof(float); + + const bool V_is_K_view = V->view_src + && (V->view_src == K || (V->view_src == K->view_src + && V->view_offs == K->view_offs)); + + // Early interleaved detection for debug output. + // True interleaved detection happens after dequant (nb12_fp16 == nb11_fp16), + // but we can pre-detect on the original tensor strides. + const bool k_early_interleaved = + ((int64_t)K->ne[1] * K->nb[1] != K->nb[2]); + const bool v_early_interleaved = + !V_is_K_view && ((int64_t)V->ne[1] * V->nb[1] != V->nb[2]); + + if (do_print) { + GGML_LOG_INFO("[MKL-FA] #%d D=%d DV=%d n_q=%d n_kv=%d " + "n_qh=%d n_kvh=%d gqa=%d batch=%d K=%s V=%s " + "chunk=%d buf=%.1fMB%s%s\n", + mkl_call_count, DKQ, DV, n_queries, n_kv, + n_q_heads, n_kv_heads, gqa_ratio, n_batch, + ggml_type_name(K->type), ggml_type_name(V->type), + chunk_size, + (double)((int64_t)n_query_rows * chunk_size * sizeof(float)) + / (1024.0 * 1024.0), + k_early_interleaved ? " K_ILV" : "", + v_early_interleaved ? " V_ILV" : ""); + GGML_LOG_INFO("[MKL-FA] #%d Q-nb1=%lld Q-nb2=%lld " + "q_rs=%lld q_hs=%lld dst_rs=%lld dst_hs=%lld\n", + mkl_call_count, + (long long)Q->nb[1], (long long)Q->nb[2], + (long long)q_row_stride, (long long)q_head_stride, + (long long)(KQV->nb[1] / sizeof(float)), + (long long)(KQV->nb[2] / sizeof(float))); + } + + // --- Stream and allocators --- + dpct::queue_ptr stream = ctx.stream(); + +#define MKL_TAKE_TIME(t0) auto t0 = std::chrono::steady_clock::now() +#define MKL_ACCUM(acc, t0) do { if (do_print) { \ + acc += (int64_t)std::chrono::duration_cast \ + (std::chrono::steady_clock::now() - (t0)).count(); \ +} } while(0) + + int64_t gemm_kq_time_us = 0; + int64_t gemm_vkq_time_us = 0; + int64_t softmax_time_us = 0; + int64_t dequant_time_us = 0; + + MKL_TAKE_TIME(t_deq); + + // --- K/V dequant descriptors --- + // Dequant is done per-chunk inside the KV loop (footprint independent of + // context). Output is always dense row-major fp16 [this_chunk x D], lda=D. + // Interleaved detection: ne[1]*nb[1] != nb[2] means heads are interleaved. + const bool k_interleaved = + ((int64_t)K->ne[1] * K->nb[1] != K->nb[2]) && K->ne[2] > 1; + const bool v_interleaved = + ((int64_t)V->ne[1] * V->nb[1] != V->nb[2]) && V->ne[2] > 1; + + const mkl_fa_kv_desc K_desc = mkl_fa_make_desc(K, k_interleaved, n_kv_heads); + const mkl_fa_kv_desc V_desc = V_is_K_view + ? K_desc : mkl_fa_make_desc(V, v_interleaved, n_kv_heads); + + MKL_ACCUM(dequant_time_us, t_deq); + + // --- Resolve mask pointers --- + const sycl::half * mask_data = nullptr; + int64_t mask_head_stride = 0; + int64_t mask_row_stride = 0; + int mask_n_heads = 0; + + if (mask) { + // Use actual fp16 device size (2 bytes), NOT sizeof(sycl::half) + // which may be 4 on the host in oneAPI. + mask_head_stride = mask->nb[2] / 2; + mask_row_stride = mask->nb[1] / 2; + mask_n_heads = (int)mask->ne[2]; + } + + // --- Allocate intermediates from pool --- + ggml_sycl_pool & pool = ctx.pool(); + + ggml_sycl_pool_alloc KQ_f32(pool); // [q_tile_rows x chunk] + ggml_sycl_pool_alloc S_f16(pool); // [q_tile_rows x chunk] + ggml_sycl_pool_alloc VKQ_chunk(pool); // [q_tile_rows x DV] + ggml_sycl_pool_alloc VKQ_accum(pool); // [n_query_rows x DV] (full) + ggml_sycl_pool_alloc KQ_max(pool); // [n_query_rows] (full) + ggml_sycl_pool_alloc KQ_sum(pool); // [n_query_rows] (full) + ggml_sycl_pool_alloc Q_head_f16(pool); // [n_query_rows x DKQ] (full) + ggml_sycl_pool_alloc K_chunk_f16(pool); // [chunk x DKQ] (per-chunk dequant) + ggml_sycl_pool_alloc V_chunk_f16(pool); // [chunk x DV] (per-chunk dequant) + + KQ_f32.alloc((size_t)q_tile_rows * chunk_size); + S_f16.alloc((size_t)q_tile_rows * chunk_size); + VKQ_chunk.alloc((size_t)q_tile_rows * DV); + VKQ_accum.alloc((size_t)n_query_rows * DV); + KQ_max.alloc(n_query_rows); + KQ_sum.alloc(n_query_rows); + Q_head_f16.alloc((size_t)n_query_rows * DKQ); + K_chunk_f16.alloc((size_t)chunk_size * DKQ); + + sycl::half * V_chunk_f16_ptr; + if (V_is_K_view) { + V_chunk_f16_ptr = K_chunk_f16.ptr; // V aliases K (DV == DKQ) + } else { + V_chunk_f16.alloc((size_t)chunk_size * DV); + V_chunk_f16_ptr = V_chunk_f16.ptr; + } + + sycl::half * Q_head_f16_ptr = Q_head_f16.ptr; + float * KQ_f32_ptr = KQ_f32.ptr; + sycl::half * S_f16_ptr = S_f16.ptr; + float * VKQ_chunk_ptr = VKQ_chunk.ptr; + float * VKQ_accum_ptr = VKQ_accum.ptr; + float * KQ_max_ptr = KQ_max.ptr; + float * KQ_sum_ptr = KQ_sum.ptr; + sycl::half * K_chunk_f16_ptr = K_chunk_f16.ptr; + + const float alpha = 1.0f; + const float beta = 0.0f; + + for (int ib = 0; ib < n_batch; ib++) { + const float * Q_batch = (const float *)Q->data + + ib * (Q->nb[3] / sizeof(float)); + float * dst_batch = (float *)KQV->data + + ib * (KQV->nb[3] / sizeof(float)); + + const sycl::half * mask_batch = nullptr; + if (mask) { + int m_batch = (mask->ne[3] > 1) ? ib : 0; + mask_batch = (const sycl::half *)mask->data + + m_batch * (mask->nb[3] / 2); // 2 = actual fp16 device size + } + + for (int ikvh = 0; ikvh < n_kv_heads; ikvh++) { + int kvh_base_head = ikvh * gqa_ratio; + + // 1. Pack all GQA Q heads into fp16 (full n_query_rows) + mkl_fa_pack_q_fp16(stream, + Q_head_f16_ptr, Q_batch, + n_queries, n_query_rows, DKQ, + gqa_ratio, kvh_base_head, + q_scale, q_row_stride, q_head_stride, wg_size); + + // 2. Initialize softmax state (full n_query_rows) + mkl_fa_init_softmax_state(stream, + KQ_max_ptr, KQ_sum_ptr, VKQ_accum_ptr, + n_query_rows, DV, wg_size); + + // Sync before MKL GEMM (MKL may use an internal queue) + stream->wait(); + + // 3. KV chunk loop (OUTER): dequant each chunk once, then tile queries. + for (int chunk_start = 0; chunk_start < n_kv; chunk_start += chunk_size) { + int this_chunk = std::min(chunk_size, n_kv - chunk_start); + + // 3a. Dequant this KV chunk to dense fp16 (once per chunk) + { + MKL_TAKE_TIME(t0); + mkl_fa_dequant_chunk(stream, K_desc, KQV, + K_chunk_f16_ptr, ikvh, chunk_start, this_chunk); + if (!V_is_K_view) { + mkl_fa_dequant_chunk(stream, V_desc, KQV, + V_chunk_f16_ptr, ikvh, chunk_start, this_chunk); + } + stream->wait(); // dequant must be ready before MKL GEMM + MKL_ACCUM(dequant_time_us, t0); + } + + // 3b. Query tile loop (INNER) — bounds KQ_f32/S_f16 footprint. + for (int q0 = 0; q0 < n_query_rows; q0 += q_tile_rows) { + int q_rows = std::min(q_tile_rows, n_query_rows - q0); + + // GEMM: KQ = Q_tile × K_chunk^T + { + MKL_TAKE_TIME(t0); + sycl::event ev = gemm(*stream, + transpose::trans, transpose::nontrans, + this_chunk, q_rows, DKQ, + alpha, + K_chunk_f16_ptr, DKQ, + Q_head_f16_ptr + (int64_t)q0 * DKQ, DKQ, + beta, + KQ_f32_ptr, this_chunk); + try { ev.wait_and_throw(); } catch (sycl::exception & e) { + GGML_LOG_INFO("[MKL-FA] GEMM KQ: %s\n", e.what()); + GGML_ABORT("MKL GEMM KQ failed"); + } + MKL_ACCUM(gemm_kq_time_us, t0); + } + // Online softmax over this chunk for this query tile + { + MKL_TAKE_TIME(t0); + mkl_fa_online_softmax_chunk(stream, + KQ_f32_ptr, S_f16_ptr, + KQ_max_ptr, KQ_sum_ptr, VKQ_accum_ptr, + q0, q_rows, n_queries, DV, + this_chunk, chunk_start, + kvh_base_head, gqa_ratio, + mask_batch, mask_head_stride, + mask_row_stride, mask_n_heads, + logit_softcap, wg_size); + stream->wait(); // S_f16 must be ready for GEMM + MKL_ACCUM(softmax_time_us, t0); + } + + // GEMM: VKQ_chunk = S × V_chunk + { + MKL_TAKE_TIME(t0); + sycl::event ev = gemm(*stream, + transpose::nontrans, transpose::nontrans, + DV, q_rows, this_chunk, + alpha, + V_chunk_f16_ptr, DV, + S_f16_ptr, this_chunk, + beta, + VKQ_chunk_ptr, DV); + try { ev.wait_and_throw(); } catch (sycl::exception & e) { + GGML_LOG_INFO("[MKL-FA] GEMM VKQ: %s\n", e.what()); + GGML_ABORT("MKL GEMM VKQ failed"); + } + MKL_ACCUM(gemm_vkq_time_us, t0); + } + // VKQ_accum[q0..] += VKQ_chunk + { + const int64_t n_total = (int64_t)q_rows * DV; + const int64_t wg = ((n_total + wg_size - 1) / wg_size) + * wg_size; + float * accum = VKQ_accum_ptr + (int64_t)q0 * DV; + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + [=](sycl::nd_item<1> item) { + int64_t i = item.get_global_id(0); + if (i < n_total) { + accum[i] += VKQ_chunk_ptr[i]; + } + }); + }); + } + } + } + + // 4. Normalize and scatter each GQA head to dst + for (int iqg = 0; iqg < gqa_ratio; iqg++) { + int iqh = kvh_base_head + iqg; + int64_t src_offset = (int64_t)iqg * n_queries * DV; + mkl_fa_normalize_head(stream, + dst_batch, VKQ_accum_ptr, KQ_sum_ptr, + iqh, n_queries, DV, n_q_heads, + src_offset, wg_size); + } + } + } + +#undef MKL_TAKE_TIME +#undef MKL_ACCUM + + if (do_print) { + const int64_t v_chunk_elems = V_is_K_view ? 0 : (int64_t)chunk_size * DV; + double total_mb = (double)( + (int64_t)q_tile_rows * chunk_size * sizeof(float) // KQ_f32 + + (int64_t)q_tile_rows * chunk_size * sizeof(sycl::half) // S_f16 + + (int64_t)q_tile_rows * DV * sizeof(float) // VKQ_chunk + + (int64_t)n_query_rows * DV * sizeof(float) // VKQ_accum + + (int64_t)n_query_rows * sizeof(float) // KQ_max + + (int64_t)n_query_rows * sizeof(float) // KQ_sum + + (int64_t)n_query_rows * DKQ * sizeof(sycl::half) // Q_head_f16 + + (int64_t)chunk_size * DKQ * sizeof(sycl::half) // K_chunk_f16 + + v_chunk_elems * (int64_t)sizeof(sycl::half) // V_chunk_f16 + ) / (1024.0 * 1024.0); + GGML_LOG_INFO("[MKL-FA] #%d n_kv=%d n_q=%d q_tile=%d time_us: " + "dequant=%lld GEMM_KQ=%lld softmax=%lld GEMM_VKQ=%lld " + "buf_mb=%.1f\n", + mkl_call_count, n_kv, n_queries, q_tile_rows, + (long long)dequant_time_us, + (long long)gemm_kq_time_us, + (long long)softmax_time_us, + (long long)gemm_vkq_time_us, + total_mb); + } +} diff --git a/ggml/src/ggml-sycl/fattn-onednn.cpp b/ggml/src/ggml-sycl/fattn-onednn.cpp index f2e12ef1ae..8465e12248 100644 --- a/ggml/src/ggml-sycl/fattn-onednn.cpp +++ b/ggml/src/ggml-sycl/fattn-onednn.cpp @@ -38,6 +38,12 @@ bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst) { if (K->type != GGML_TYPE_F16 || V->type != GGML_TYPE_F16) { return false; } + // Optional KV-length ceiling (GGML_SYCL_FA_ONEDNN_MAX_KV, 0 = unlimited). Escape hatch: + // very long sequences make the fused SDPA slow enough to risk the xe driver watchdog on + // some stacks; past the cap we fall back to the native FA kernel instead. + if (g_ggml_sycl_fa_onednn_max_kv > 0 && K->ne[1] > g_ggml_sycl_fa_onednn_max_kv) { + return false; + } // gate for the following cases // 1. if the oneDNN graph Add node has no input --> skip // 2. types other than f16 need different logical_tensor declaration @@ -208,9 +214,17 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso cont_to_f16_sycl((const char *) V->data, Vf.get(), d, seq, Hkv, mb, V->nb[1], V->nb[2], V->nb[3], stream); // divide-by-(1/scale) reproduces ggml's score *= kq_scale on the proven probe graph. + // + // The scale must not be uploaded with an async memcpy from a stack local: on the in-order + // queue that copy waits behind the K/V staging kernels, and once those take long enough + // (n_kv >= ~26k on B70) the host frame is recycled before the copy runs, feeding the SDPA a + // garbage scale (output collapses to a repeated token). Write the scalar from a kernel + // instead -- the value is captured into the command, so no host memory has to outlive the + // call, and the enqueue stays async. const sycl::half scale_h = (sycl::half) (1.0f / kq_scale); ggml_sycl_pool_alloc scbuf(ctx.pool(), 1); - stream->memcpy(scbuf.get(), &scale_h, sizeof(sycl::half)); + sycl::half * const scale_dev = scbuf.get(); + stream->single_task([=]() { *scale_dev = scale_h; }); ggml_sycl_pool_alloc outf(ctx.pool(), (size_t) H * q * d); // f16 contiguous SDPA out [mb,H,q,d] @@ -232,7 +246,7 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso if (r == E.id_q) return Qf.get(); if (r == E.id_k) return Kf.get(); if (r == E.id_v) return Vf.get(); - if (r == E.id_scale) return scbuf.get(); + if (r == E.id_scale) return scale_dev; if (r == E.id_mask) return (void *) mask->data; return nullptr; }; @@ -245,14 +259,12 @@ void ggml_sycl_flash_attn_ext_onednn(ggml_backend_sycl_context & ctx, ggml_tenso E.cp.execute(strm, ti, {to}); permute_sdpa_out_sycl(outf.get(), (float *) dst->data, mb, H, q, d, stream); - // Single device: no sync is required, and actually PP perf is ~6% > wait_and_throw() (tested on llama-3.1-8b & qwen3.6-27b, both Q8_0, with Arc B70). - // Any future multi-GPU refactor MUST re-measure this single-device path and keep the best - // single-device PP speed. Otherwise (multiple devices/streams can race the reuse): + // Single device needs no sync: the dnnl stream wraps this same in-order queue, so the SDPA + // serializes with the staging kernels before it and the permute/pool reuse after it. The + // garbage output formerly blamed on the missing sync here was the scale use-after-return + // fixed above. Keep the conservative wait for multi-GPU, where other devices' streams can + // race the pool: if (ggml_sycl_info().device_count > 1) { - // cont_to_f16 -> oneDNN execute -> permute is async on this stream, but the - // pool_alloc*s above free their device buffers at host return. Without this wait the next - // scheduler op re-acquires those bytes while the GPU is still computing the SDPA, turning - // it into garbage and collapsing multi-turn output to a single repeated token ("GGGGG..."). stream->wait_and_throw(); } } diff --git a/ggml/src/ggml-sycl/fattn.cpp b/ggml/src/ggml-sycl/fattn.cpp index 1772b9c858..f8efa53acd 100644 --- a/ggml/src/ggml-sycl/fattn.cpp +++ b/ggml/src/ggml-sycl/fattn.cpp @@ -99,8 +99,10 @@ enum best_fattn_kernel { BEST_FATTN_KERNEL_VEC = 100, BEST_FATTN_KERNEL_ONEDNN = 150, // added enum for onednn==150 BEST_FATTN_KERNEL_TILE = 200, + BEST_FATTN_KERNEL_MKL = 300, }; + static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const ggml_tensor * dst) { GGML_UNUSED(device); #ifndef SYCL_FLASH_ATTN @@ -115,6 +117,7 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const const ggml_tensor * K = dst->src[1]; const ggml_tensor * V = dst->src[2]; const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; const int gqa_ratio = Q->ne[2] / K->ne[2]; GGML_ASSERT(Q->ne[2] % K->ne[2] == 0); @@ -122,7 +125,49 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const float max_bias = 0.0f; memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float)); + float logit_softcap = 0.0f; + memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); + bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && K->ne[1] % FATTN_KQ_STRIDE == 0; + + // MKL path: XMX-accelerated GEMM for prompt processing (all KV cache types). + // The MKL kernel converts non-F16 K/V to F16 via to_fp16_sycl before GEMM, + // so quantized, F16, BF16, and F32 caches all benefit from XMX acceleration. + // Activates automatically when flash-attn is enabled (--flash-attn on or -fa) + // and n_kv >= 1024. Falls through to TILE/VEC for ALiBi, logit softcap, + // and mismatched batch dimensions (unsupported by the MKL kernel). + // Set GGML_SYCL_ENABLE_MKL_FA=0 to force TILE/VEC path for A/B testing. + // Example: GGML_SYCL_ENABLE_MKL_FA=0 llama-cli -m model.gguf -fa -ngl 99 ... + // Note: MKL GEMM calls are incompatible with SYCL graph capture replay. + static int mkl_enable = ggml_sycl_get_env("GGML_SYCL_ENABLE_MKL_FA", 1); + // MKL is validated for the mainstream GQA envelope: grouped-query + // (gqa_ratio >= 2), head_dim a multiple of 64 in [64,512] with matching + // K/V head size, mask, no sinks/ALiBi/softcap. Gemma's global layers use + // head_dim 512, so the cap must include it. Head sizes not a multiple of + // 64 (72/80/96), MHA (gqa_ratio == 1), and MLA (DKQ != DV, e.g. 576/512) + // fall through to TILE/VEC; see follow-up work. + if (mkl_enable == 1 && mask && !sinks && gqa_ratio >= 2 && + Q->ne[0] >= 64 && Q->ne[0] <= 512 && Q->ne[0] % 64 == 0 && + Q->ne[0] == V->ne[0] && + Q->ne[1] >= 32 && K->ne[1] >= 1024 && + max_bias == 0.0f && logit_softcap == 0.0f && + (Q->ne[3] == K->ne[3] || K->ne[3] == 1)) { + // F16 K/V strides must be a multiple of ne[0]*2 (the natural row size + // in bytes). This passes both dense (nb1 == ne0*2) and interleaved + // (nb1 == H * ne0*2). Only pathological test strides like nb1=32 or + // nb1=75 for ne0=40 fall through to TILE. + bool kv_strides_ok = true; + for (const ggml_tensor * t : {K, V}) { + if (t->type == GGML_TYPE_F16 && t->nb[1] % (t->ne[0] * 2) != 0) { + kv_strides_ok = false; + break; + } + } + if (kv_strides_ok) { + return BEST_FATTN_KERNEL_MKL; + } + } + for (const ggml_tensor * t : {Q, K, V, mask}) { if (t == nullptr || ggml_is_quantized(t->type)) { continue; @@ -216,6 +261,37 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { ggml_sycl_set_device(ctx.device); + + // n_kv watchdog: log when n_kv differs from the last FA call with + // the same D — helps detect cache-truncation issues. + static int nkv_debug = ggml_sycl_get_env("GGML_SYCL_MKL_FA_DEBUG", 0); + if (nkv_debug == 1) { + const ggml_tensor * K_dbg = dst->src[1]; + const ggml_tensor * V_dbg = dst->src[2]; + static int64_t last_nkv_d256 = 0, last_nkv_d512 = 0; + static int fa_call_seq = 0; + fa_call_seq++; + int64_t cur_nkv = K_dbg->ne[1]; + int Dk = (int)K_dbg->ne[0]; + const char * kname = "TILE"; + best_fattn_kernel k = ggml_sycl_get_best_fattn_kernel(ctx.device, dst); + if (k == BEST_FATTN_KERNEL_MKL) kname = "MKL"; + if (k == BEST_FATTN_KERNEL_VEC) kname = "VEC"; + int64_t delta = 0; + if (Dk == 256) { + delta = cur_nkv - last_nkv_d256; + last_nkv_d256 = cur_nkv; + } else if (Dk == 512) { + delta = cur_nkv - last_nkv_d512; + last_nkv_d512 = cur_nkv; + } + GGML_LOG_INFO("[FA-DISP] #%d %s D=%d n_kv=%lld delta=%lld " + "V_ne1=%lld\n", + fa_call_seq, kname, Dk, + (long long)cur_nkv, (long long)delta, + (long long)V_dbg->ne[1]); + } + switch (ggml_sycl_get_best_fattn_kernel(ggml_sycl_get_device(), dst)) { case BEST_FATTN_KERNEL_NONE: GGML_ABORT("Not support Flash-Attention"); @@ -232,6 +308,51 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst case BEST_FATTN_KERNEL_VEC: ggml_sycl_flash_attn_ext_vec(ctx, dst); break; + case BEST_FATTN_KERNEL_MKL: + ggml_sycl_flash_attn_ext_mkl(ctx, dst); + break; + } + + // --- Output fingerprint (GGML_SYCL_MKL_FA_DIAG=1) --- + // Copy first 64 float output values to host for fingerprinting. + // Compare MKL vs TILE (GGML_SYCL_ENABLE_MKL_FA=0) to detect divergence. + // Only fingerprints the first 6 FA calls with n_kv >= 1024. + static int fa_diag = ggml_sycl_get_env("GGML_SYCL_MKL_FA_DIAG", 0); + static int fa_diag_count = 0; + if (fa_diag == 1 && fa_diag_count < 6) { + const ggml_tensor * K_diag = dst->src[1]; + const ggml_tensor * V_diag = dst->src[2]; + const ggml_tensor * Q_diag = dst->src[0]; + if (K_diag->ne[1] >= 1024) { + fa_diag_count++; + float diag_buf[64]; + dpct::queue_ptr q = ctx.stream(); + q->memcpy(diag_buf, dst->data, 64 * sizeof(float)); + q->wait(); + const char * kname = "???"; + best_fattn_kernel kb = ggml_sycl_get_best_fattn_kernel(ctx.device, dst); + if (kb == BEST_FATTN_KERNEL_MKL) kname = "MKL"; + if (kb == BEST_FATTN_KERNEL_TILE) kname = "TILE"; + if (kb == BEST_FATTN_KERNEL_VEC) kname = "VEC"; + GGML_LOG_INFO("[FA-DIAG] #%d %s D=%d n_kv=%lld n_q=%lld " + "n_qh=%lld n_kvh=%lld K=%s V=%s " + "nb1=%zu nb2=%zu first 64 floats:\n", + fa_diag_count, kname, + (int)K_diag->ne[0], (long long)K_diag->ne[1], + (long long)Q_diag->ne[1], + (long long)Q_diag->ne[2], (long long)K_diag->ne[2], + ggml_type_name(K_diag->type), + ggml_type_name(V_diag->type), + K_diag->nb[1], K_diag->nb[2]); + for (int i = 0; i < 64; i += 8) { + GGML_LOG_INFO(" [%2d] %08x %08x %08x %08x %08x %08x %08x %08x\n", + i, + *(unsigned *)&diag_buf[i+0], *(unsigned *)&diag_buf[i+1], + *(unsigned *)&diag_buf[i+2], *(unsigned *)&diag_buf[i+3], + *(unsigned *)&diag_buf[i+4], *(unsigned *)&diag_buf[i+5], + *(unsigned *)&diag_buf[i+6], *(unsigned *)&diag_buf[i+7]); + } + } } } diff --git a/ggml/src/ggml-sycl/fattn.hpp b/ggml/src/ggml-sycl/fattn.hpp index f2a8ffc97d..c093970a3f 100644 --- a/ggml/src/ggml-sycl/fattn.hpp +++ b/ggml/src/ggml-sycl/fattn.hpp @@ -19,4 +19,6 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst bool ggml_sycl_flash_attn_ext_supported(int device, const ggml_tensor * dst); +void ggml_sycl_flash_attn_ext_mkl(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + #endif // GGML_SYCL_FATTN_HPP diff --git a/ggml/src/ggml-sycl/fusion.cpp b/ggml/src/ggml-sycl/fusion.cpp new file mode 100644 index 0000000000..4a6027f39b --- /dev/null +++ b/ggml/src/ggml-sycl/fusion.cpp @@ -0,0 +1,44 @@ +#include "fusion.hpp" + +bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializer_list ops) { + if (!g_ggml_sycl_enable_fusion) { + return false; + } + + if (!ggml_can_fuse(cgraph, node_idx, ops)) { + return false; + } + + if (ops.size() == 2 && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) { + const ggml_tensor * rms_norm = cgraph->nodes[node_idx]; + const ggml_tensor * mul = cgraph->nodes[node_idx + 1]; + + GGML_ASSERT(rms_norm->src[0]->type == GGML_TYPE_F32); + GGML_ASSERT(rms_norm->type == GGML_TYPE_F32); + + if (mul->src[0]->type != GGML_TYPE_F32 || + mul->src[1]->type != GGML_TYPE_F32 || + mul->type != GGML_TYPE_F32) { + return false; + } + + // if rms norm is the B operand, then we don't handle broadcast + if (rms_norm == mul->src[1] && !ggml_are_same_shape(mul->src[0], rms_norm)) { + return false; + } + + const ggml_tensor * mul_w = (mul->src[0] == rms_norm) ? mul->src[1] : mul->src[0]; + // the fused kernel indexes the weight as mul[col], so it must span ncols contiguously + if (mul_w->ne[0] != rms_norm->ne[0] || mul_w->nb[0] != ggml_type_size(mul_w->type)) { + return false; + } + + if (!ggml_is_contiguous_rows(mul->src[0]) || !ggml_is_contiguous_rows(mul->src[1])) { + return false; + } + + return true; + } + + return false; +} diff --git a/ggml/src/ggml-sycl/fusion.hpp b/ggml/src/ggml-sycl/fusion.hpp new file mode 100644 index 0000000000..7d7c79e028 --- /dev/null +++ b/ggml/src/ggml-sycl/fusion.hpp @@ -0,0 +1,15 @@ +#ifndef GGML_SYCL_FUSION_HPP +#define GGML_SYCL_FUSION_HPP + +#include + +#include "common.hpp" + +// Backend-side fusability test. `ops` names a candidate op sequence starting at cgraph node +// `node_idx`; the result is true only if ggml considers that subgraph fusable *and* the SYCL +// kernel which would service it accepts the tensors involved (types, shapes, contiguity). +// +// Lives in its own translation unit because it grows a branch per supported op sequence. +bool ggml_sycl_can_fuse(const ggml_cgraph * cgraph, int node_idx, std::initializer_list ops); + +#endif // GGML_SYCL_FUSION_HPP diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index cb8974eedb..07ddfda887 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -85,6 +85,7 @@ int g_ggml_sycl_enable_optimize = 1; int g_ggml_sycl_enable_graph = 0; int g_ggml_sycl_enable_dnn = 1; int g_ggml_sycl_fa_onednn = 1; +int g_ggml_sycl_fa_onednn_max_kv = 0; int g_ggml_sycl_enable_vmm = 1; int g_ggml_sycl_enable_fusion = 1; int g_ggml_sycl_prioritize_dmmv = 0; @@ -273,6 +274,8 @@ static const char* dev2dev_int2str(int dev2dev) { return "SYCL API"; } else if (dev2dev == DEV2DEV_MEMCPY_L0) { return "Level Zero API"; + } else if (dev2dev == DEV2DEV_MEMCPY_FORWARD) { + return "Host Forward"; } else { return "Unknown"; } @@ -287,6 +290,7 @@ static void ggml_check_sycl() try { g_ggml_sycl_enable_graph = ggml_sycl_get_env("GGML_SYCL_ENABLE_GRAPH", 0); g_ggml_sycl_enable_dnn = ggml_sycl_get_env("GGML_SYCL_ENABLE_DNN", 1); g_ggml_sycl_fa_onednn = ggml_sycl_get_env("GGML_SYCL_FA_ONEDNN", 1); + g_ggml_sycl_fa_onednn_max_kv = ggml_sycl_get_env("GGML_SYCL_FA_ONEDNN_MAX_KV", 0); g_ggml_sycl_enable_vmm = ggml_sycl_get_env("GGML_SYCL_ENABLE_VMM", 1); g_ggml_sycl_enable_fusion = ggml_sycl_get_env("GGML_SYCL_ENABLE_FUSION", 1); g_ggml_sycl_prioritize_dmmv = ggml_sycl_get_env("GGML_SYCL_PRIORITIZE_DMMV", 0); @@ -359,6 +363,7 @@ static void ggml_check_sycl() try { GGML_LOG_INFO(" GGML_SYCL_ENABLE_DNN: DNN disabled by compile flag\n"); GGML_LOG_INFO(" GGML_SYCL_FA_ONEDNN: %d\n", g_ggml_sycl_fa_onednn); #endif + GGML_LOG_INFO(" GGML_SYCL_FA_ONEDNN_MAX_KV: %d\n", g_ggml_sycl_fa_onednn_max_kv); #ifdef SYCL_FLASH_ATTN GGML_LOG_INFO(" GGML_SYCL_ENABLE_FLASH_ATTN: %d\n", g_ggml_sycl_enable_flash_attention); #else @@ -681,7 +686,11 @@ static void dev2dev_memcpy(int device_dst, sycl::queue &q_dst, int device_src, s } // Host-staged copy - GGML_SYCL_DEBUG("[SYCL] dev2dev memcpy by host forward\n"); + if(g_ggml_sycl_dev2dev_memcpy == DEV2DEV_MEMCPY_FORWARD) { + GGML_SYCL_DEBUG("[SYCL] dev2dev memcpy by host forward for setting GGML_SYCL_DEV2DEV_MEMCPY=2\n"); + } else { + GGML_SYCL_DEBUG("[SYCL] dev2dev memcpy by host forward for SYCL/L0 fallback\n"); + } char *host_buf = (char *)malloc(size); q_src.memcpy(host_buf, (const char *)ptr_src, size).wait(); q_dst.memcpy((char *)ptr_dst, host_buf, size).wait(); @@ -5395,6 +5404,13 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc } } #endif + if (node->op == GGML_OP_RMS_NORM && + ggml_sycl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL })) { + ggml_sycl_op_rms_norm_fused(*sycl_ctx, node, cgraph->nodes[i + 1]); + i++; + continue; + } + bool ok = ggml_sycl_compute_forward(*sycl_ctx, node); if (!ok) { GGML_LOG_ERROR("%s: error: op not supported %s (%s)\n", __func__, node->name, ggml_op_name(node->op)); diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index 7b1b3d467f..863d34eabb 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -1254,6 +1254,66 @@ static void mul_mat_vec_q1_0_q8_1_sycl_switch_ncols( } } +static void mul_mat_vec_q2_0_q8_1_sycl(const void * vx, const void * vy, + float * dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK2_0 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1); + }); + }); +} + +template +static void mul_mat_vec_q2_0_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK2_0 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_ncols( + vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); + }); + }); +} + +static void mul_mat_vec_q2_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: mul_mat_vec_q2_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: mul_mat_vec_q2_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 3: mul_mat_vec_q2_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 4: mul_mat_vec_q2_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 5: mul_mat_vec_q2_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 6: mul_mat_vec_q2_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 7: mul_mat_vec_q2_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 8: mul_mat_vec_q2_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for Q2_0 multi-col MMVQ", ncols_dst); + } +} + static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, @@ -2194,6 +2254,20 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens mul_mat_vec_q1_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); } break; + case GGML_TYPE_Q2_0: + if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + const int stride_col_y = src1_padded_col_size / QK8_1; + const int stride_col_dst = dst->ne[0]; + GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); + mul_mat_vec_q2_0_q8_1_sycl_switch_ncols( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, + src1_ncols, stride_col_y, stride_col_dst, stream); + return; + } else if (i == 0 || src1_ncols == 1) { + GGML_SYCL_DEBUG("Calling mul_mat_vec_q2_0_q8_1_sycl\n"); + mul_mat_vec_q2_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } + break; case GGML_TYPE_Q2_K: if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { const int stride_col_y = src1_padded_col_size / QK8_1; @@ -2503,6 +2577,11 @@ bool ggml_sycl_mul_mat_vec_q_id( vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, expert_weight_stride, dst_row_stride, src1_row_stride, stream); return true; + case GGML_TYPE_Q2_0: + launch_mul_mat_vec_q_moe( + vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, + expert_weight_stride, dst_row_stride, src1_row_stride, stream); + return true; case GGML_TYPE_Q2_K: launch_mul_mat_vec_q_moe( vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used, diff --git a/ggml/src/ggml-sycl/norm.cpp b/ggml/src/ggml-sycl/norm.cpp index c4472e4bd6..682a9f51ee 100644 --- a/ggml/src/ggml-sycl/norm.cpp +++ b/ggml/src/ggml-sycl/norm.cpp @@ -147,10 +147,13 @@ static void group_norm_f32(const float* x, float* dst, const int group_size, con } } +template static void rms_norm_f32(const float* x, float* dst, const int ncols, const int64_t src_stride_col, const int64_t src_stride_row, const int64_t src_stride_channel, const int64_t src_stride_sample, const int64_t dst_stride_col, const int64_t dst_stride_row, const int64_t dst_stride_channel, const int64_t dst_stride_sample, - const float eps, const sycl::nd_item<3>& item_ct1, float* s_sum, int block_size) { + const float eps, const sycl::nd_item<3>& item_ct1, float* s_sum, int block_size, + const float* mul = nullptr, const int64_t mul_stride_row = 0, const int64_t mul_stride_channel = 0, + const int64_t mul_stride_sample = 0, const int mul_nrows = 0, const int mul_nchannels = 0, const int mul_nsamples = 0) { const int nrows = item_ct1.get_group_range(2); const int nchannels = item_ct1.get_group_range(1); @@ -170,6 +173,12 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols, x += src_offset; dst += dst_offset; + if constexpr (do_multiply) { + const int mul_row = row % mul_nrows; + const int mul_channel = channel % mul_nchannels; + const int mul_sample = sample % mul_nsamples; + mul += mul_sample * mul_stride_sample + mul_channel * mul_stride_channel + mul_row * mul_stride_row; + } float tmp = 0.0f; // partial sum for thread in warp @@ -202,7 +211,11 @@ static void rms_norm_f32(const float* x, float* dst, const int ncols, const float scale = sycl::rsqrt(mean + eps); for (int col = tid; col < ncols; col += block_size) { - dst[col * dst_stride_col] = scale * x[col * src_stride_col]; + if constexpr (do_multiply) { + dst[col * dst_stride_col] = scale * x[col * src_stride_col] * mul[col]; + } else { + dst[col * dst_stride_col] = scale * x[col * src_stride_col]; + } } } @@ -376,6 +389,49 @@ static void rms_norm_f32_sycl(const float* x, float* dst, const int ncols, const } } +static void rms_norm_mul_f32_sycl(const float* x, const float* mul, float* dst, const int ncols, const int nrows, + const int nchannels, const int nsamples, + const int64_t src_stride_col, const int64_t src_stride_row, const int64_t src_stride_channel, const int64_t src_stride_sample, + const int64_t dst_stride_col, const int64_t dst_stride_row, const int64_t dst_stride_channel, const int64_t dst_stride_sample, + const int64_t mul_stride_row, const int64_t mul_stride_channel, const int64_t mul_stride_sample, + const int mul_nrows, const int mul_nchannels, const int mul_nsamples, + const float eps, queue_ptr stream, int device) { + const sycl::range<3> global_dims(nsamples, nchannels, nrows); + if (ncols < 1024) { + const sycl::range<3> block_dims(1, 1, WARP_SIZE); + stream->submit([&](sycl::handler& cgh) { + cgh.parallel_for( + sycl::nd_range<3>(global_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + rms_norm_f32(x, dst, ncols, + src_stride_col, src_stride_row, src_stride_channel, src_stride_sample, + dst_stride_col, dst_stride_row, dst_stride_channel, dst_stride_sample, + eps, item_ct1, nullptr, WARP_SIZE, + mul, mul_stride_row, mul_stride_channel, mul_stride_sample, mul_nrows, mul_nchannels, mul_nsamples); + }); + }); + } + else { + const int work_group_size = ggml_sycl_info().max_work_group_sizes[device]; + assert(work_group_size % (WARP_SIZE * WARP_SIZE) == 0); + const sycl::range<3> block_dims(1, 1, work_group_size); + stream->submit([&](sycl::handler& cgh) { + sycl::local_accessor s_sum_acc_ct1(sycl::range<1>(work_group_size / WARP_SIZE), cgh); + cgh.parallel_for( + sycl::nd_range<3>(global_dims * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) + [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + rms_norm_f32(x, dst, ncols, + src_stride_col, src_stride_row, src_stride_channel, src_stride_sample, + dst_stride_col, dst_stride_row, dst_stride_channel, dst_stride_sample, + eps, item_ct1, get_pointer(s_sum_acc_ct1), work_group_size, + mul, mul_stride_row, mul_stride_channel, mul_stride_sample, mul_nrows, mul_nchannels, mul_nsamples); + }); + }); + } +} + template static void l2_norm_f32_sycl(const float * x, float * dst, @@ -518,6 +574,66 @@ void ggml_sycl_op_rms_norm(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { ss0, ss1, ss2, ss3, ds0, ds1, ds2, ds3, eps, main_stream, ctx.device); } +void ggml_sycl_op_rms_norm_fused(ggml_backend_sycl_context & ctx, ggml_tensor * dst, ggml_tensor * mul_tensor) { + const ggml_tensor * rms_norm_src = dst->src[0]; + float eps = 0.0f; + memcpy(&eps, dst->op_params, sizeof(float)); + + const float * src0_dd = static_cast(rms_norm_src->data); + const float * mul_dd = nullptr; + const ggml_tensor * mul_src = nullptr; + if (mul_tensor->src[0] == dst) { + mul_dd = static_cast(mul_tensor->src[1]->data); + mul_src = mul_tensor->src[1]; + } else if (mul_tensor->src[1] == dst) { + mul_dd = static_cast(mul_tensor->src[0]->data); + mul_src = mul_tensor->src[0]; + } else { + GGML_ASSERT(false); + } + float * dst_dd = static_cast(mul_tensor->data); + + dpct::queue_ptr main_stream = ctx.stream(); + SYCL_CHECK(ggml_sycl_set_device(ctx.device)); + + GGML_ASSERT(rms_norm_src->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT(mul_tensor->type == GGML_TYPE_F32); + GGML_ASSERT(eps >= 0.0f); + + const int64_t ne00 = rms_norm_src->ne[0]; + const int64_t ne01 = rms_norm_src->ne[1]; + const int64_t ne02 = rms_norm_src->ne[2]; + const int64_t ne03 = rms_norm_src->ne[3]; + + const size_t ts0 = ggml_type_size(rms_norm_src->type); + GGML_ASSERT(rms_norm_src->nb[0] == ts0); + const int64_t s00 = rms_norm_src->nb[0] / ts0; + const int64_t s01 = rms_norm_src->nb[1] / ts0; + const int64_t s02 = rms_norm_src->nb[2] / ts0; + const int64_t s03 = rms_norm_src->nb[3] / ts0; + + const size_t tdst = ggml_type_size(mul_tensor->type); + GGML_ASSERT(mul_tensor->nb[0] == tdst); + const int64_t d00 = mul_tensor->nb[0] / tdst; + const int64_t d01 = mul_tensor->nb[1] / tdst; + const int64_t d02 = mul_tensor->nb[2] / tdst; + const int64_t d03 = mul_tensor->nb[3] / tdst; + + const size_t ts_mul = ggml_type_size(mul_src->type); + GGML_ASSERT(mul_src->nb[0] == ts_mul); + const int64_t mul_s01 = mul_src->nb[1] / ts_mul; + const int64_t mul_s02 = mul_src->nb[2] / ts_mul; + const int64_t mul_s03 = mul_src->nb[3] / ts_mul; + const int mul_nrows = mul_src->ne[1]; + const int mul_nchannels = mul_src->ne[2]; + const int mul_nsamples = mul_src->ne[3]; + + rms_norm_mul_f32_sycl(src0_dd, mul_dd, dst_dd, ne00, ne01, ne02, ne03, + s00, s01, s02, s03, d00, d01, d02, d03, + mul_s01, mul_s02, mul_s03, mul_nrows, mul_nchannels, mul_nsamples, eps, main_stream, ctx.device); +} + void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); diff --git a/ggml/src/ggml-sycl/norm.hpp b/ggml/src/ggml-sycl/norm.hpp index 8cb885eb2e..51217c4219 100644 --- a/ggml/src/ggml-sycl/norm.hpp +++ b/ggml/src/ggml-sycl/norm.hpp @@ -19,6 +19,8 @@ void ggml_sycl_op_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst); void ggml_sycl_op_rms_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst); +void ggml_sycl_op_rms_norm_fused(ggml_backend_sycl_context& ctx, ggml_tensor* dst, ggml_tensor* mul); + void ggml_sycl_op_rms_norm_back(ggml_backend_sycl_context& ctx, ggml_tensor* dst); void ggml_sycl_op_group_norm(ggml_backend_sycl_context& ctx, ggml_tensor* dst); diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp index 765fb7f159..c11a6e8f9c 100644 --- a/ggml/src/ggml-sycl/vecdotq.hpp +++ b/ggml/src/ggml-sycl/vecdotq.hpp @@ -658,6 +658,40 @@ template <> struct reorder_vec_dot_q_sycl { #define VDR_Q4_0_Q8_1_MMVQ 2 #define VDR_Q4_0_Q8_1_MMQ 4 +#define VDR_Q2_0_Q8_1_MMVQ 1 + +template +static __dpct_inline__ float vec_dot_q2_0_q8_1_impl( + const int * v, + const int * u, + const float & d2, + const sycl::half2 & ds8) { + int sumi = 0; + +#pragma unroll + for (int i = 0; i < vdr; ++i) { +#pragma unroll + for (int j = 0; j < 4; ++j) { + const uint8_t q = (uint8_t) ((uint32_t) v[i] >> (8 * j)); + + // unpack 2-bit values to byte lanes (0..3), then apply zero-point + // correction with ds8f.y() below, mirroring the q4_0 style. + int vi = 0; + vi |= (((q >> 0) & 0x3) & 0xFF) << 0; + vi |= (((q >> 2) & 0x3) & 0xFF) << 8; + vi |= (((q >> 4) & 0x3) & 0xFF) << 16; + vi |= (((q >> 6) & 0x3) & 0xFF) << 24; + + sumi = dpct::dp4a(vi, u[4 * i + j], sumi); + } + } + + const sycl::float2 ds8f = ds8.convert(); + // q2_0 has zero-point 1. Scale ds8f.y() by processed-lane ratio, + // consistent with q4_0's explicit zero-point subtraction style. + return d2 * (sumi * ds8f.x() - ((float) vdr / (float) QI2_0) * ds8f.y()); +} + template static __dpct_inline__ float vec_dot_q4_0_q8_1_impl(const int * v, const int * u, const float & d4, const sycl::half2 & ds8) { @@ -882,6 +916,41 @@ vec_dot_q4_0_q8_1(const void *__restrict__ vbq, return vec_dot_q4_0_q8_1_impl(v, u, bq4_0->d, bq8_1->ds); } +static __dpct_inline__ float +vec_dot_q2_0_q8_1(const void *__restrict__ vbq, + const block_q8_1 *__restrict__ bq8_1, const int &iqs) { + + const block_q2_0 * bq2_0 = (const block_q2_0 *) vbq; + + int v[2 * VDR_Q2_0_Q8_1_MMVQ]; + int u[8 * VDR_Q2_0_Q8_1_MMVQ]; + +#pragma unroll + for (int i = 0; i < VDR_Q2_0_Q8_1_MMVQ; ++i) { + const int base = 4 * (iqs + i); + + // Q2_0 has QK2_0 = 64 and uses 2 x QK8_1 blocks on the RHS. + v[2 * i + 0] = get_int_from_uint8(bq2_0->qs, iqs + i); + v[2 * i + 1] = get_int_from_uint8(bq2_0->qs, iqs + i + QI2_0); + + u[8 * i + 0] = get_int_from_int8_aligned(bq8_1[0].qs, base + 0); + u[8 * i + 1] = get_int_from_int8_aligned(bq8_1[0].qs, base + 1); + u[8 * i + 2] = get_int_from_int8_aligned(bq8_1[0].qs, base + 2); + u[8 * i + 3] = get_int_from_int8_aligned(bq8_1[0].qs, base + 3); + + u[8 * i + 4] = get_int_from_int8_aligned(bq8_1[1].qs, base + 0); + u[8 * i + 5] = get_int_from_int8_aligned(bq8_1[1].qs, base + 1); + u[8 * i + 6] = get_int_from_int8_aligned(bq8_1[1].qs, base + 2); + u[8 * i + 7] = get_int_from_int8_aligned(bq8_1[1].qs, base + 3); + } + + const float sum0 = vec_dot_q2_0_q8_1_impl( + v + 0, u + 0, bq2_0->d, bq8_1[0].ds); + const float sum1 = vec_dot_q2_0_q8_1_impl( + v + VDR_Q2_0_Q8_1_MMVQ, u + 4 * VDR_Q2_0_Q8_1_MMVQ, bq2_0->d, bq8_1[1].ds); + return sum0 + sum1; +} + static __dpct_inline__ float vec_dot_q4_1_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs) { diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 5dcf4503bb..ff72fa60a8 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -998,6 +998,7 @@ struct vk_device_struct { vk_pipeline pipeline_snake_f32; vk_pipeline pipeline_snake_f16; vk_pipeline pipeline_snake_bf16; + vk_pipeline pipeline_pool1d_f32; vk_pipeline pipeline_pool2d_f32; vk_pipeline pipeline_rwkv_wkv6_f32; vk_pipeline pipeline_rwkv_wkv7_f32; @@ -1481,6 +1482,11 @@ struct vk_op_binary_push_constants { float param1; float param2; int32_t param3; }; +// Distinct type with the same layout so concat can overload tensor offset initialization. +struct vk_op_concat_push_constants : vk_op_binary_push_constants {}; +static_assert(sizeof(vk_op_concat_push_constants) == sizeof(vk_op_binary_push_constants)); +static_assert(std::is_standard_layout_v); + struct vk_op_multi_add_push_constants { // shape for dst uint32_t ne20; uint32_t ne21; uint32_t ne22; uint32_t ne23; @@ -1682,6 +1688,17 @@ struct vk_op_snake_push_constants { uint32_t ne1; }; +struct vk_op_pool1d_push_constants { + uint32_t IL; + uint32_t OL; + uint32_t OC; + uint32_t pelements; + uint32_t op; + int32_t k0; + int32_t s0; + int32_t p0; +}; + struct vk_op_pool2d_push_constants { uint32_t IW; uint32_t IH; uint32_t OW; uint32_t OH; @@ -1929,6 +1946,7 @@ struct ggml_vk_garbage_collector { static void ggml_vk_preallocate_buffers(ggml_backend_vk_context * ctx, vk_context subctx); static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested = nullptr); static void ggml_pipeline_allocate_descriptor_sets(ggml_backend_vk_context * ctx); +static bool ggml_vk_intel_windows_driver_equals_or_newer_than(uint32_t driver_version, uint32_t threshold_major, uint32_t threshold_minor); static bool vk_memory_logger_enabled = false; @@ -2246,6 +2264,40 @@ static uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const gg return ((vk_tensor_offset(t) + t->view_offs) & (ctx->device->properties.limits.minStorageBufferOffsetAlignment - 1));; } +static uint32_t ggml_vk_concat_unit_size(ggml_type type) { + const uint32_t type_size = ggml_type_size(type); + + if (!ggml_is_quantized(type)) { + return type_size; + } + + // Use the widest existing concat shader that evenly divides a quant block. + if (type_size % 8 == 0) { + return 8; + } + if (type_size % 4 == 0) { + return 4; + } + if (type_size % 2 == 0) { + return 2; + } + return 1; +} + +static bool ggml_vk_concat_supported(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) { + if (src0->type != src1->type || src0->type != dst->type) { + return false; + } + + if (!ggml_is_quantized(src0->type)) { + const size_t type_size = ggml_type_size(src0->type); + return type_size == 1 || type_size == 2 || type_size == 4 || type_size == 8; + } + + // Quantized tensor rows are block-aligned when created. + return ggml_is_contiguous_rows(src0) && ggml_is_contiguous_rows(src1) && ggml_is_contiguous_rows(dst); +} + template void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, T &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { GGML_UNUSED(p); GGML_UNUSED(src0); @@ -3490,7 +3542,7 @@ struct vk_fa_tuning_params { }; static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type); -static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type = GGML_TYPE_F16); +static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type = GGML_TYPE_F16, ggml_type v_type = GGML_TYPE_F16); static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device, uint32_t hsk, uint32_t hsv, uint32_t n_rows, uint32_t n_kv, ggml_type k_type, ggml_type v_type, bool f32acc) { @@ -3646,7 +3698,7 @@ static vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_ bool shape_ok = (f32acc && device->coopmat_support_16x16x16_f32acc) || (!f32acc && device->coopmat_support_16x16x16_f16acc); const vk_fa_tuning_params params = get_fa_tuning_params_coopmat1(device, hsk, hsv, n_rows, n_kv, k_type, v_type, f32acc); - bool shmem_ok = ggml_vk_flash_attn_coopmat_shmem_support(device, params, hsk, hsv, f32acc, k_type); + bool shmem_ok = ggml_vk_flash_attn_coopmat_shmem_support(device, params, hsk, hsv, f32acc, k_type, v_type); if (!shape_ok || !shmem_ok) { path = FA_SCALAR; @@ -3658,11 +3710,6 @@ static vk_fa_tuning_params get_fa_tuning_params(const vk_device& device, uint32_ path = FA_SCALAR; } - // Q1_0 K/V is only implemented on coopmat2 (flash_attn_cm2); there is no scalar FA shader for it. - if ((k_type == GGML_TYPE_Q1_0 || v_type == GGML_TYPE_Q1_0) && device->coopmat2) { - path = FA_COOPMAT2; - } - switch (path) { case FA_SCALAR: return get_fa_tuning_params_scalar(device, hsk, hsv, n_rows, n_kv, k_type, v_type, f32acc); @@ -3904,16 +3951,27 @@ static uint32_t get_subgroup_size(const std::string &pipeline_name, const vk_dev return 0; // If no matching configuration is found } -// Whether scalar flash attention will use the MMQ path for the given k_type. -static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type) { +// Whether scalar flash attention will use the MMQ path for the given K/V types. +static bool ggml_vk_fa_type_needs_shmem(ggml_type type) { + switch (type) { + case GGML_TYPE_IQ4_NL: + return true; + default: + return false; + } +} + +static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type, ggml_type v_type) { #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) return device->integer_dot_product && device->subgroup_clustered && + !ggml_vk_fa_type_needs_shmem(v_type) && (k_type == GGML_TYPE_Q4_0 || k_type == GGML_TYPE_Q4_1 || k_type == GGML_TYPE_Q5_0 || k_type == GGML_TYPE_Q5_1 || k_type == GGML_TYPE_Q8_0); #else GGML_UNUSED(device); GGML_UNUSED(k_type); + GGML_UNUSED(v_type); return false; #endif } @@ -4246,7 +4304,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { const bool fa_ds = fa.first.subgroup_size == 0; const bool bf16_kv = fa.first.k_type == GGML_TYPE_BF16; - const bool use_mmq = ggml_vk_fa_scalar_uses_mmq(device, fa.first.k_type); + const bool use_mmq = ggml_vk_fa_scalar_uses_mmq(device, fa.first.k_type, fa.first.v_type); const void * spv_data = nullptr; size_t spv_size = 0; const char *name = nullptr; @@ -5507,8 +5565,11 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_argmax_f32, "argmax_f32", argmax_f32_len, argmax_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); ggml_vk_create_pipeline(device, device->pipeline_sum_rows_f32, "sum_rows_f32", sum_rows_f32_len, sum_rows_f32_data, "main", 2, sizeof(vk_op_sum_rows_push_constants), {1, 1, 1}, { device->subgroup_size }, 1); - // Intel Arc B390 was observed segfaulting with this shader. - if (device->subgroup_basic && device->subgroup_shuffle && device->vendor_id != VK_VENDOR_ID_INTEL) { + // Intel Windows driver older than 32.0.101.8860 will crash when using fwht kernels on Xe2+ GPUS so we gate that here + const bool can_use_fwht = device->driver_id != vk::DriverId::eIntelProprietaryWindows || + device->architecture != vk_device_architecture::INTEL_XE2 || + (device->architecture == vk_device_architecture::INTEL_XE2 && ggml_vk_intel_windows_driver_equals_or_newer_than(device->properties.driverVersion, 101, 8860)); + if (can_use_fwht && device->subgroup_basic && device->subgroup_shuffle) { int idx = 0; for (uint32_t n : {64, 128, 256, 512}) { if (device->subgroup_size <= n) { @@ -5516,8 +5577,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { } ++idx; } - } else if (device->driver_id != vk::DriverId::eIntelProprietaryWindows) { - // Disabled on Intel Windows due to a driver bug: https://github.com/ggml-org/llama.cpp/pull/23964#issuecomment-4598226147 + } else if (can_use_fwht) { int idx = 0; for (uint32_t n : {64, 128, 256, 512}) { const uint32_t block_size = std::min(device->subgroup_size, n); @@ -5574,6 +5634,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_snake_f16, "snake_f16", snake_f16_len, snake_f16_data, "main", 4, sizeof(vk_op_snake_push_constants), {256, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_snake_bf16, "snake_bf16", snake_bf16_len, snake_bf16_data, "main", 4, sizeof(vk_op_snake_push_constants), {256, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_pool1d_f32, "pool1d_f32", pool1d_f32_len, pool1d_f32_data, "main", 2, sizeof(vk_op_pool1d_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_pool2d_f32, "pool2d_f32", pool2d_f32_len, pool2d_f32_data, "main", 2, sizeof(vk_op_pool2d_push_constants), {512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_rwkv_wkv6_f32, "rwkv_wkv6_f32", rwkv_wkv6_f32_len, rwkv_wkv6_f32_data, "main", 7, sizeof(vk_op_rwkv_wkv6_push_constants), {1, 1, 1}, {device->subgroup_size}, 1); @@ -10380,7 +10441,6 @@ static void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) { GGML_UNUSED(f32acc); - GGML_UNUSED(v_type); // Needs to be kept up to date on shader changes const uint32_t wg_size = params.workgroup_size; const uint32_t Br = params.block_rows; @@ -10389,13 +10449,15 @@ static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, con // BF16 uses the fp32 shader (FLOAT_TYPE=float) const uint32_t float_type_size = (device->fp16 && k_type != GGML_TYPE_BF16) ? sizeof(ggml_fp16_t) : sizeof(float); - const bool mmq = ggml_vk_fa_scalar_uses_mmq(device, k_type); + const bool mmq = ggml_vk_fa_scalar_uses_mmq(device, k_type, v_type); // tmpsh is overestimated slightly const uint32_t tmpsh = wg_size * sizeof(float); const uint32_t tmpshv4 = wg_size * 4 * float_type_size; const uint32_t masksh = Bc * (Br + 1) * float_type_size; + // DATA_A_IQ4_NL is compiled into the FA shaders unconditionally, so its shared table is always allocated. + const uint32_t iq_shmem = 16 * float_type_size; uint32_t Qf, kvsh, kblocksh_size; if (mmq) { @@ -10420,7 +10482,7 @@ static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, con kblocksh_size = 0; } - const uint32_t total_size = tmpsh + tmpshv4 + masksh + Qf + kvsh + kblocksh_size; + const uint32_t total_size = tmpsh + tmpshv4 + masksh + iq_shmem + Qf + kvsh + kblocksh_size; const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; VK_LOG_DEBUG("ggml_vk_flash_attn_scalar_shmem_support(HSK=" << hsk << ", HSV=" << hsv << ", mmq=" << mmq << ", total_size=" << total_size << ", supported=" << supported); @@ -10428,7 +10490,8 @@ static bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, con return supported; } -static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type) { +static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, const vk_fa_tuning_params& params, uint32_t hsk, uint32_t hsv, bool f32acc, ggml_type k_type, ggml_type v_type) { + GGML_UNUSED(v_type); // Needs to be kept up to date on shader changes const uint32_t Br = params.block_rows; const uint32_t Bc = params.block_cols; @@ -10444,6 +10507,8 @@ static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, co const uint32_t f16vec4 = 8; const uint32_t tmpsh = (Bc / MatBc) * sizeof(float); + // DATA_A_IQ4_NL is compiled into the FA shaders unconditionally, so its shared table is always allocated. + const uint32_t iq_shmem = 16 * sizeof(ggml_fp16_t); const uint32_t qstride = hsk_pad / 4 + 2; const uint32_t Qf = Br * qstride * f16vec4; @@ -10465,7 +10530,7 @@ static bool ggml_vk_flash_attn_coopmat_shmem_support(const vk_device& device, co const uint32_t slope = Br * acctype; - const uint32_t total_size = tmpsh + Qf + Psh + sfsh + ksh + pvsh + slope; + const uint32_t total_size = tmpsh + iq_shmem + Qf + Psh + sfsh + ksh + pvsh + slope; const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize; VK_LOG_DEBUG("ggml_vk_flash_attn_coopmat_shmem_support(HSK=" << hsk << ", HSV=" << hsv << ", f32acc=" << f32acc << ", total_size=" << total_size << ", supported=" << supported); @@ -10886,14 +10951,10 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const } return nullptr; case GGML_OP_CONCAT: { - if (src0->type != src1->type || src0->type != dst->type) { + if (!ggml_vk_concat_supported(src0, src1, dst)) { return nullptr; } - if (ggml_blck_size(src0->type) != 1) { - return nullptr; - } - const size_t type_size = ggml_type_size(src0->type); - switch (type_size) { + switch (ggml_vk_concat_unit_size(src0->type)) { case 1: return ctx->device->pipeline_concat_i8; case 2: @@ -11287,6 +11348,11 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const case GGML_TYPE_BF16: return ctx->device->pipeline_col2im_1d_bf16; default: return nullptr; } + case GGML_OP_POOL_1D: + if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { + return ctx->device->pipeline_pool1d_f32; + } + return nullptr; case GGML_OP_POOL_2D: if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { return ctx->device->pipeline_pool2d_f32; @@ -11585,6 +11651,18 @@ template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk GGML_UNUSED(src3); } +template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_concat_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { + const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type); + const uint32_t a_offset = get_misalign_bytes(ctx, src0) / unit_size; + const uint32_t b_offset = get_misalign_bytes(ctx, src1) / unit_size; + const uint32_t d_offset = get_misalign_bytes(ctx, dst) / unit_size; + + p.misalign_offsets = (a_offset << 16) | (b_offset << 8) | d_offset; + + GGML_UNUSED(src2); + GGML_UNUSED(src3); +} + template <> void init_pushconst_tensor_offsets(ggml_backend_vk_context * ctx, vk_op_upscale_push_constants &p, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * src2, const ggml_tensor * src3, ggml_tensor * dst) { const uint32_t a_offset = get_misalign_bytes(ctx, src0) / ggml_type_size(src0->type); const uint32_t d_offset = get_misalign_bytes(ctx, dst) / ggml_type_size(dst->type); @@ -11620,7 +11698,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co } std::cerr << "), (" << dst << ", name=" << dst->name << ", type=" << dst->type << ", ne0=" << dst->ne[0] << ", ne1=" << dst->ne[1] << ", ne2=" << dst->ne[2] << ", ne3=" << dst->ne[3] << ", nb0=" << dst->nb[0] << ", nb1=" << dst->nb[1] << ", nb2=" << dst->nb[2] << ", nb3=" << dst->nb[3]; std::cerr << "), " << ggml_op_name(op) << ")"); - GGML_ASSERT(op == GGML_OP_GET_ROWS || op == GGML_OP_CPY || (!ggml_is_quantized(src0->type) && (src1 == nullptr || !ggml_is_quantized(src1->type)))); // NOLINT + GGML_ASSERT(op == GGML_OP_GET_ROWS || op == GGML_OP_CPY || op == GGML_OP_CONCAT || (!ggml_is_quantized(src0->type) && (src1 == nullptr || !ggml_is_quantized(src1->type)))); // NOLINT GGML_ASSERT(dst->buffer != nullptr); const uint64_t ne00 = src0->ne[0]; const uint64_t ne01 = src0->ne[1]; @@ -11797,6 +11875,13 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co { elements = { uint32_t(dst->ne[0]), uint32_t(dst->ne[1]), 1 }; } break; + case GGML_OP_POOL_1D: + { + const uint32_t N = dst->ne[3] * dst->ne[2]; + const uint32_t OC = dst->ne[1]; + const uint32_t OL = dst->ne[0]; + elements = { N * OC * OL, 1, 1}; + } break; case GGML_OP_POOL_2D: { const uint32_t N = dst->ne[3]; @@ -11875,6 +11960,9 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co ne *= ggml_type_size(src0->type) / 2; } } + if (op == GGML_OP_CONCAT && ggml_is_quantized(dst->type)) { + ne = ne / ggml_blck_size(dst->type) * ggml_type_size(dst->type) / ggml_vk_concat_unit_size(dst->type); + } // copy_to_quant has block size of 32, and each thread does QUANT_K elements. // Splitting into 512x512xZ wouldn't work well since each workgroup does 1024 elements. // So divide by block size here before splitting into 512x512 groups. @@ -12515,18 +12603,28 @@ static void ggml_vk_opt_step_sgd(ggml_backend_vk_context * ctx, vk_context& subc static void ggml_vk_concat(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { int * op_params = (int *)dst->op_params; - const uint32_t src0_type_size = ggml_type_size(src0->type); - const uint32_t src1_type_size = ggml_type_size(src1->type); - const uint32_t dst_type_size = ggml_type_size(dst->type); + const uint32_t unit_size = ggml_vk_concat_unit_size(dst->type); + const uint32_t units_per_block = ggml_type_size(dst->type) / unit_size; + const uint32_t block_size = ggml_blck_size(dst->type); + const bool quantized = ggml_is_quantized(dst->type); - ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONCAT, { - (uint32_t)ggml_nelements(dst), - (uint32_t)src0->ne[0], (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], (uint32_t)src0->nb[0] / src0_type_size, (uint32_t)src0->nb[1] / src0_type_size, (uint32_t)src0->nb[2] / src0_type_size, (uint32_t)src0->nb[3] / src0_type_size, - (uint32_t)src1->ne[0], (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], (uint32_t)src1->nb[0] / src1_type_size, (uint32_t)src1->nb[1] / src1_type_size, (uint32_t)src1->nb[2] / src1_type_size, (uint32_t)src1->nb[3] / src1_type_size, - (uint32_t) dst->ne[0], (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], (uint32_t) dst->nb[0] / dst_type_size, (uint32_t) dst->nb[1] / dst_type_size, (uint32_t) dst->nb[2] / dst_type_size, (uint32_t) dst->nb[3] / dst_type_size, + // Address dimension 0 in packed storage units; higher strides may be noncontiguous. + const uint32_t ne00 = src0->ne[0] / block_size * units_per_block; + const uint32_t ne10 = src1->ne[0] / block_size * units_per_block; + const uint32_t ne20 = dst->ne[0] / block_size * units_per_block; + const uint32_t nb00 = quantized ? 1 : src0->nb[0] / unit_size; + const uint32_t nb10 = quantized ? 1 : src1->nb[0] / unit_size; + const uint32_t nb20 = quantized ? 1 : dst->nb[0] / unit_size; + + vk_op_concat_push_constants pc {{ + ne20 * (uint32_t)dst->ne[1] * (uint32_t)dst->ne[2] * (uint32_t)dst->ne[3], + ne00, (uint32_t)src0->ne[1], (uint32_t)src0->ne[2],(uint32_t)src0->ne[3], nb00, (uint32_t)src0->nb[1] / unit_size, (uint32_t)src0->nb[2] / unit_size, (uint32_t)src0->nb[3] / unit_size, + ne10, (uint32_t)src1->ne[1], (uint32_t)src1->ne[2],(uint32_t)src1->ne[3], nb10, (uint32_t)src1->nb[1] / unit_size, (uint32_t)src1->nb[2] / unit_size, (uint32_t)src1->nb[3] / unit_size, + ne20, (uint32_t) dst->ne[1], (uint32_t) dst->ne[2],(uint32_t) dst->ne[3], nb20, (uint32_t) dst->nb[1] / unit_size, (uint32_t) dst->nb[2] / unit_size, (uint32_t) dst->nb[3] / unit_size, 0, 0.0f, 0.0f, op_params[0], - }); + }}; + ggml_vk_op_f32(ctx, subctx, src0, src1, nullptr, nullptr, dst, GGML_OP_CONCAT, std::move(pc)); } static void ggml_vk_upscale(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { @@ -13703,6 +13801,29 @@ static void ggml_vk_snake_dispatch_fused(ggml_backend_vk_context * ctx, vk_conte ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { x_buf, a_buf, inv_b_buf, dst_buf }, pc, elements); } +static void ggml_vk_pool_1d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { + uint32_t op = static_cast(dst->op_params[0]); + const int32_t k0 = dst->op_params[1]; + const int32_t s0 = dst->op_params[2]; + const int32_t p0 = dst->op_params[3]; + + const uint32_t IL = src0->ne[0]; + + const uint32_t N = dst->ne[3] * dst->ne[2]; + + const uint32_t OC = dst->ne[1]; + const uint32_t OL = dst->ne[0]; + + const uint32_t parallel_elements = N * OC * OL; + + ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_POOL_1D, { + IL, OL, OC, + parallel_elements, + op, + k0, s0, p0, + }); +} + static void ggml_vk_pool_2d(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) { uint32_t op = static_cast(dst->op_params[0]); const int32_t k1 = dst->op_params[1]; @@ -15214,6 +15335,10 @@ static bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgr case GGML_OP_CONV_TRANSPOSE_1D: ggml_vk_conv_transpose_1d(ctx, compute_ctx, src0, src1, node); + break; + case GGML_OP_POOL_1D: + ggml_vk_pool_1d(ctx, compute_ctx, src0, node); + break; case GGML_OP_POOL_2D: ggml_vk_pool_2d(ctx, compute_ctx, src0, node); @@ -17617,7 +17742,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm if (op->src[3] && op->src[3]->type != GGML_TYPE_F16) { return false; } - auto fa_kv_ok = [coopmat2](ggml_type t) { + auto fa_kv_ok = [](ggml_type t) { switch (t) { case GGML_TYPE_F32: case GGML_TYPE_F16: @@ -17627,9 +17752,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_TYPE_Q5_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q4_0: + case GGML_TYPE_IQ4_NL: return true; - case GGML_TYPE_Q1_0: - return coopmat2; default: return false; } @@ -17863,12 +17987,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm return op->src[0]->type == op->src[1]->type && op->src[0]->type == op->type && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_I32); case GGML_OP_CONCAT: { - if (op->src[0]->type != op->src[1]->type || op->src[0]->type != op->type) { - return false; - } - const size_t type_size = ggml_type_size(op->type); - return ggml_blck_size(op->type) == 1 && - (type_size == 1 || type_size == 2 || type_size == 4 || type_size == 8); + return ggml_vk_concat_supported(op->src[0], op->src[1], op); } case GGML_OP_ADD1: return (op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32) @@ -17937,6 +18056,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_CONV_2D_DW: return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && op->src[1]->type == GGML_TYPE_F32; + case GGML_OP_POOL_1D: + return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_POOL_2D: return ggml_is_contiguous(op->src[0]) && op->src[0]->type == GGML_TYPE_F32; case GGML_OP_RWKV_WKV6: @@ -18007,10 +18128,17 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_OP_CONV_2D: case GGML_OP_CONV_TRANSPOSE_2D: { + const bool transpose = op->op == GGML_OP_CONV_TRANSPOSE_2D; + const int64_t cout = !transpose ? op->src[0]->ne[3] : op->src[0]->ne[2]; + const int64_t cin = !transpose ? op->src[0]->ne[2] : op->src[0]->ne[3]; + // Channel-contiguous format is not supported yet. return ((op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + (op->src[0]->nb[0] == sizeof(float) || op->src[0]->nb[0] == sizeof(ggml_fp16_t) ) && op->src[1]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 && + cout == op->ne[2] && + cin == op->src[1]->ne[2] && ggml_is_contiguous(op->src[0]) && ggml_is_contiguous(op->src[1]) && ggml_is_contiguous(op)); @@ -18395,6 +18523,22 @@ static uint32_t ggml_vk_intel_shader_core_count(const vk::PhysicalDevice& vkdev) } } +static bool ggml_vk_intel_windows_driver_equals_or_newer_than(uint32_t driver_version, uint32_t threshold_major, uint32_t threshold_minor) { +#if defined(_WIN32) + // Intel Windows encodes xxx.yyyy as [31:14].[13:0]. + const uint32_t major = driver_version >> 14; + const uint32_t minor = driver_version & 0x3fff; + + return major > threshold_major || (major == threshold_major && minor >= threshold_minor); +#else + GGML_UNUSED(driver_version); + GGML_UNUSED(threshold_major); + GGML_UNUSED(threshold_minor); + return true; +#endif +} + + // checks #ifdef GGML_VULKAN_CHECK_RESULTS @@ -18849,6 +18993,13 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * const int32_t oc = tensor->op_params[1]; const int32_t p0 = tensor->op_params[2]; tensor_clone = ggml_col2im_1d(ggml_ctx, src_clone[0], stride, oc, p0); + } else if (tensor->op == GGML_OP_POOL_1D) { + enum ggml_op_pool op = static_cast(tensor->op_params[0]); + const int32_t k0 = tensor->op_params[1]; + const int32_t s0 = tensor->op_params[2]; + const int32_t p0 = tensor->op_params[3]; + + tensor_clone = ggml_pool_1d(ggml_ctx, src_clone[0], op, k0, s0, p0); } else if (tensor->op == GGML_OP_POOL_2D) { enum ggml_op_pool op = static_cast(tensor->op_params[0]); const int32_t k0 = tensor->op_params[1]; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp index 3192130ccf..6c264c7861 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn.comp @@ -80,7 +80,9 @@ shared vec4 occupancy_limiter[LIMIT_OCCUPANCY_SHMEM > 0 ? LIMIT_OCCUPANCY_SHMEM void main() { #ifdef NEEDS_INIT_IQ_SHMEM - init_iq_shmem(gl_WorkGroupSize); + if (fa_type_needs_shmem(FaTypeK) || fa_type_needs_shmem(FaTypeV)) { + init_iq_shmem(gl_WorkGroupSize); + } #endif init_indices(); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl index 66dcf61021..3c64f91dad 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_base.glsl @@ -97,8 +97,8 @@ layout (binding = 6) readonly buffer MO {uint32_t data_mask_opt[];}; #define FA_TYPE_Q5_0 6u #define FA_TYPE_Q5_1 7u #define FA_TYPE_Q8_0 8u +#define FA_TYPE_IQ4_NL 20u #define FA_TYPE_BF16 30u -#define FA_TYPE_Q1_0 41u #if defined(BFLOAT16) #define O_TYPE float @@ -120,8 +120,8 @@ uint fa_block_elems(uint ty) { case FA_TYPE_Q5_0: return uint(QUANT_K_Q5_0); case FA_TYPE_Q5_1: return uint(QUANT_K_Q5_1); case FA_TYPE_Q8_0: return uint(QUANT_K_Q8_0); + case FA_TYPE_IQ4_NL: return uint(QUANT_K_IQ4_NL); case FA_TYPE_BF16: return 1u; - case FA_TYPE_Q1_0: return uint(QUANT_K_Q1_0); // cm2-only, harmless elsewhere default: return 1u; } } @@ -140,6 +140,13 @@ uint fa_quant_r_mmq(uint ty) { } } +bool fa_type_needs_shmem(uint ty) { + switch (ty) { + case FA_TYPE_IQ4_NL: return true; + default: return false; + } +} + // These can't be `const` globals because GLSL forbids function calls in global // const initializers, even when the spec constants would let the driver fold // them. Macros expand at the use site and fold after specialization. diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp index 16178e5770..057ed739aa 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm1.comp @@ -64,7 +64,9 @@ shared ACC_TYPE slope[Br]; void main() { #ifdef NEEDS_INIT_IQ_SHMEM - init_iq_shmem(gl_WorkGroupSize); + if (fa_type_needs_shmem(FaTypeK) || fa_type_needs_shmem(FaTypeV)) { + init_iq_shmem(gl_WorkGroupSize); + } #endif init_indices(); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp index b9c03fe499..3174111530 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_cm2.comp @@ -46,7 +46,7 @@ float16_t faDecodeK(const decodeBufFA_K bl_in, const uint blockCoords[2], const case FA_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); case FA_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); case FA_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q1_0: return dequantFuncQ1_0(decodeBufQ1_0(bl_in), blockCoords, coordInBlock); + case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); default: return float16_t(0); } } @@ -59,7 +59,7 @@ float16_t faDecodeV(const decodeBufFA_V bl_in, const uint blockCoords[2], const case FA_TYPE_Q5_0: return dequantFuncQ5_0(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); case FA_TYPE_Q5_1: return dequantFuncQ5_1(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); case FA_TYPE_Q8_0: return dequantFuncQ8_0(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case FA_TYPE_Q1_0: return dequantFuncQ1_0(decodeBufQ1_0(bl_in), blockCoords, coordInBlock); + case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); default: return float16_t(0); } } @@ -67,26 +67,26 @@ float16_t faDecodeV(const decodeBufFA_V bl_in, const uint blockCoords[2], const // V=4 vector decode for K/V; dispatches to per-format _v decoders. f16vec4 faDecodeKVector(const decodeBufFA_K bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeK) { - case 0u: return f16vec4(decodeBufF32(bl_in).block); - case 2u: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case 3u: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case 6u: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case 7u: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case 8u: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case 41u: return dequantFuncQ1_0_v(decodeBufQ1_0(bl_in), blockCoords, coordInBlock); + case FA_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); + case FA_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case FA_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case FA_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case FA_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case FA_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); default: return f16vec4(0); } } f16vec4 faDecodeVVector(const decodeBufFA_V bl_in, const uint blockCoords[2], const uint coordInBlock[2]) { switch (FaTypeV) { - case 0u: return f16vec4(decodeBufF32(bl_in).block); - case 2u: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); - case 3u: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); - case 6u: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); - case 7u: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); - case 8u: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); - case 41u: return dequantFuncQ1_0_v(decodeBufQ1_0(bl_in), blockCoords, coordInBlock); + case FA_TYPE_F32: return f16vec4(decodeBufF32(bl_in).block); + case FA_TYPE_Q4_0: return dequantFuncQ4_0_v(decodeBufQ4_0(bl_in), blockCoords, coordInBlock); + case FA_TYPE_Q4_1: return dequantFuncQ4_1_v(decodeBufQ4_1(bl_in), blockCoords, coordInBlock); + case FA_TYPE_Q5_0: return dequantFuncQ5_0_v(decodeBufQ5_0(bl_in), blockCoords, coordInBlock); + case FA_TYPE_Q5_1: return dequantFuncQ5_1_v(decodeBufQ5_1(bl_in), blockCoords, coordInBlock); + case FA_TYPE_Q8_0: return dequantFuncQ8_0_v(decodeBufQ8_0(bl_in), blockCoords, coordInBlock); + case FA_TYPE_IQ4_NL: return dequantFuncIQ4_NL_v(decodeBufIQ4_NL(bl_in), blockCoords, coordInBlock); default: return f16vec4(0); } } @@ -169,6 +169,12 @@ ACC_TYPE perElemOpNonGqaSplitKStoreCol0(const in uint32_t r, const in uint32_t c } void main() { +#ifdef NEEDS_INIT_IQ_SHMEM + if (fa_type_needs_shmem(FaTypeK) || fa_type_needs_shmem(FaTypeV)) { + init_iq_shmem(gl_WorkGroupSize); + } +#endif + init_indices(); tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutQ = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV); @@ -302,7 +308,7 @@ void main() { coopmat K_T; uint32_t k_offset = ik2*p.nb12 + ik3*p.nb13; - // F16: bs_k==1 (direct load). F32: bs_k==4 (vec4 / dequantFuncF32). Q4/Q8 family: bs_k==32. Q1_0: bs_k==128. + // F16: bs_k==1 (direct load). F32: bs_k==4 (vec4 / dequantFuncF32). Quantized types: bs_k==32. #if defined(BFLOAT16) coopMatLoadTensorNV(K_T, data_k, k_offset, sliceTensorLayoutNV(tensorLayoutK, j * Bc, Bc, 0, HSK_pad), tensorViewTranspose); #else diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl index 8704479d96..8ba4725f33 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/flash_attn_dequant.glsl @@ -27,6 +27,8 @@ layout (binding = 1) readonly buffer K_PACKED_Q5_1 { block_q5_1_packed16 data[]; layout (binding = 2) readonly buffer V_PACKED_Q5_1 { block_q5_1_packed16 data[]; } v_packed_q5_1; layout (binding = 1) readonly buffer K_PACKED_Q8_0 { block_q8_0_packed16 data[]; } k_packed_q8_0; layout (binding = 2) readonly buffer V_PACKED_Q8_0 { block_q8_0_packed16 data[]; } v_packed_q8_0; +layout (binding = 1) readonly buffer K_PACKED_IQ4_NL { block_iq4_nl_packed16 data[]; } k_packed_iq4_nl; +layout (binding = 2) readonly buffer V_PACKED_IQ4_NL { block_iq4_nl_packed16 data[]; } v_packed_iq4_nl; layout (binding = 1) readonly buffer K_PACKED_BF16 { u16vec4 data[]; } k_packed_bf16; layout (binding = 2) readonly buffer V_PACKED_BF16 { u16vec4 data[]; } v_packed_bf16; @@ -102,6 +104,17 @@ layout (binding = 1) readonly buffer K_PACKED_Q5_1_P32 { block_q5_1_packed32 dat return FLOAT_TYPE(BUF.data[a_offset + ib].d) * FLOAT_TYPEV4(v0.x, v0.y, v1.x, v1.y); \ } +#define FA_DEQUANT4_IQ4_NL(BUF) { \ + const uint shift = (iqs & 0x10) >> 2; \ + const uint qs_i = (iqs & 0xC) >> 1; \ + const uint qsw = uint(BUF.data[a_offset + ib].qs[qs_i]) \ + | (uint(BUF.data[a_offset + ib].qs[qs_i + 1u]) << 16); \ + const FLOAT_TYPE d = FLOAT_TYPE(BUF.data[a_offset + ib].d); \ + const u8vec4 q = unpack8((qsw >> shift) & 0x0F0F0F0Fu); \ + return d * FLOAT_TYPEV4(kvalues_iq4nl[q.x], kvalues_iq4nl[q.y], \ + kvalues_iq4nl[q.z], kvalues_iq4nl[q.w]); \ +} + #define FA_DEQUANT4_BF16(BUF) \ return FLOAT_TYPEV4(bf16_to_fp32(uvec4(BUF.data[(a_offset + ib) / 4]))); @@ -114,6 +127,7 @@ FLOAT_TYPEV4 dequantize4(uint ib, uint iqs, uint a_offset, uint binding_idx) { case FA_TYPE_Q5_0: FA_DEQUANT4_Q5_0(k_packed_q5_0) case FA_TYPE_Q5_1: FA_DEQUANT4_Q5_1(k_packed_q5_1) case FA_TYPE_Q8_0: FA_DEQUANT4_Q8_0(k_packed_q8_0) + case FA_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(k_packed_iq4_nl) case FA_TYPE_BF16: FA_DEQUANT4_BF16(k_packed_bf16) } } else { @@ -124,6 +138,7 @@ FLOAT_TYPEV4 dequantize4(uint ib, uint iqs, uint a_offset, uint binding_idx) { case FA_TYPE_Q5_0: FA_DEQUANT4_Q5_0(v_packed_q5_0) case FA_TYPE_Q5_1: FA_DEQUANT4_Q5_1(v_packed_q5_1) case FA_TYPE_Q8_0: FA_DEQUANT4_Q8_0(v_packed_q8_0) + case FA_TYPE_IQ4_NL: FA_DEQUANT4_IQ4_NL(v_packed_iq4_nl) case FA_TYPE_BF16: FA_DEQUANT4_BF16(v_packed_bf16) } } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp b/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp new file mode 100644 index 0000000000..bb87631ce3 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/pool1d.comp @@ -0,0 +1,65 @@ +#version 450 + +#include "types.glsl" + +#extension GL_EXT_shader_16bit_storage : require + +layout(push_constant) uniform parameter { + uint IL; + uint OL; + uint OC; + uint pelements; + uint op; + int k0; + int s0; + int p0; +} p; + +#define BLOCK_SIZE 512 +#define FLT_MAX 3.402823466e+38F +#define OP_POOL_MAX 0u +#define OP_POOL_AVG 1u + +layout (local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in; + +layout(binding = 0) readonly buffer X {A_TYPE data_a[];}; +layout(binding = 1) writeonly buffer D {D_TYPE data_d[];}; + +void main() { + const uint idx = gl_GlobalInvocationID.x; + if (idx >= p.pelements) { + return; + } + + const uint nc = idx / p.OL; + const uint cur_ol = idx % p.OL; + + const int start = int(cur_ol) * p.s0 - p.p0; + const int bl = max(start, 0); + const int el = min(max(start + p.k0, 0), int(p.IL)); + + const int window_size = el - bl; + const float scale = window_size > 0 ? 1.0 / float(window_size) : 0.0; + float res; + + if (p.op == OP_POOL_AVG) { + res = 0.0; + } else if (p.op == OP_POOL_MAX) { + res = -FLT_MAX; + } else { + return; + } + + #pragma unroll + for (uint i = bl; i < el; i++) { + const float cur = D_TYPE(data_a[nc * p.IL + i]); + + if (p.op == OP_POOL_AVG) { + res += cur * scale; + } else if (p.op == OP_POOL_MAX) { + res = max(res, cur); + } + } + + data_d[nc * p.OL + cur_ol] = res; +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 58d347bc54..0223d2e01e 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -673,6 +673,8 @@ void process_shaders() { fa_base_dict["ACC_TYPE"] = fp16 && f16acc ? "float16_t" : "float"; fa_base_dict["ACC_TYPEV2"] = fp16 && f16acc ? "f16vec2" : "vec2"; fa_base_dict["ACC_TYPEV4"] = fp16 && f16acc ? "f16vec4" : "vec4"; + // Compile IQ4_NL support into all FA variants so its shared LUT is available when K or V uses it. + fa_base_dict["DATA_A_IQ4_NL"] = "1"; if (fp16 && f16acc) { fa_base_dict["ACC_TYPE_MAX"] = "float16_t(65504.0)"; } @@ -1050,6 +1052,7 @@ void process_shaders() { string_to_spv("snake_f16", "snake.comp", {{"DATA_A_F16", "1"}, {"A_TYPE", "float16_t"}, {"D_TYPE", "float16_t"}}); string_to_spv("snake_bf16", "snake.comp", {{"DATA_A_BF16", "1"}, {"DATA_D_BF16", "1"}, {"A_TYPE", "uint16_t"}, {"D_TYPE", "uint16_t"}}); + string_to_spv("pool1d_f32", "pool1d.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("pool2d_f32", "pool2d.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}})); string_to_spv("rwkv_wkv6_f32", "wkv6.comp", merge_maps(base_dict, {{"A_TYPE", "float"}})); diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp index bed9265b8a..babaddb654 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp @@ -73,11 +73,6 @@ inline bool ggml_webgpu_tensor_equal(const ggml_tensor * a, const ggml_tensor * return a->buffer == b->buffer && ggml_webgpu_tensor_addr(a) == ggml_webgpu_tensor_addr(b); } -inline bool ggml_webgpu_tensor_overlap(const ggml_tensor * a, const ggml_tensor * b) { - return a->buffer == b->buffer && ggml_webgpu_tensor_addr(a) < ggml_webgpu_tensor_addr(b) + ggml_nbytes(b) && - ggml_webgpu_tensor_addr(b) < ggml_webgpu_tensor_addr(a) + ggml_nbytes(a); -} - struct ggml_webgpu_shader_lib_context { ggml_tensor * src0; ggml_tensor * src1; @@ -118,6 +113,11 @@ struct ggml_webgpu_binary_shader_decisions { bool src_overlap = false; }; +struct ggml_webgpu_glu_shader_decisions { + uint32_t wg_size = 0; + bool src_overlap = false; +}; + struct ggml_webgpu_processed_shader { std::string wgsl; std::string variant; @@ -133,9 +133,12 @@ struct ggml_webgpu_ssm_scan_pipeline_key { int type; int d_state; bool xbc_overlap; + bool a_overlap; + bool ids_overlap; bool operator==(const ggml_webgpu_ssm_scan_pipeline_key & other) const { - return type == other.type && d_state == other.d_state && xbc_overlap == other.xbc_overlap; + return type == other.type && d_state == other.d_state && xbc_overlap == other.xbc_overlap && + a_overlap == other.a_overlap && ids_overlap == other.ids_overlap; } }; @@ -145,6 +148,8 @@ struct ggml_webgpu_ssm_scan_pipeline_key_hash { ggml_webgpu_hash_combine(seed, key.type); ggml_webgpu_hash_combine(seed, key.d_state); ggml_webgpu_hash_combine(seed, key.xbc_overlap); + ggml_webgpu_hash_combine(seed, key.a_overlap); + ggml_webgpu_hash_combine(seed, key.ids_overlap); return seed; } }; @@ -153,6 +158,8 @@ struct ggml_webgpu_ssm_scan_shader_decisions { uint32_t wg_size; uint32_t tokens_per_tile; bool xbc_overlap = false; + bool a_overlap = false; + bool ids_overlap = false; }; /** Argsort **/ @@ -264,7 +271,7 @@ struct ggml_webgpu_row_norm_pipeline_key_hash { struct ggml_webgpu_rms_norm_mul_pipeline_key { bool inplace; // rn_src == dst bool overlap; // mul_src == dst - bool src_overlap; // rn_src == mul_src + bool src_overlap; // rn_src binding overlaps mul_src binding bool operator==(const ggml_webgpu_rms_norm_mul_pipeline_key & other) const { return inplace == other.inplace && overlap == other.overlap && src_overlap == other.src_overlap; @@ -584,7 +591,8 @@ struct ggml_webgpu_flash_attn_common_pipeline_key { ggml_type dst_type; uint32_t head_dim_qk; uint32_t head_dim_v; - bool kv_direct; + bool k_direct; + bool v_direct; bool kv_overlap; bool has_mask; bool has_sinks; @@ -593,8 +601,9 @@ struct ggml_webgpu_flash_attn_common_pipeline_key { bool operator==(const ggml_webgpu_flash_attn_common_pipeline_key & other) const { return q_type == other.q_type && k_type == other.k_type && v_type == other.v_type && dst_type == other.dst_type && head_dim_qk == other.head_dim_qk && head_dim_v == other.head_dim_v && - kv_direct == other.kv_direct && kv_overlap == other.kv_overlap && has_mask == other.has_mask && - has_sinks == other.has_sinks && uses_logit_softcap == other.uses_logit_softcap; + k_direct == other.k_direct && v_direct == other.v_direct && kv_overlap == other.kv_overlap && + has_mask == other.has_mask && has_sinks == other.has_sinks && + uses_logit_softcap == other.uses_logit_softcap; } }; @@ -606,7 +615,8 @@ inline void ggml_webgpu_flash_attn_hash_common_pipeline_key(size_t & ggml_webgpu_hash_combine(seed, key.dst_type); ggml_webgpu_hash_combine(seed, key.head_dim_qk); ggml_webgpu_hash_combine(seed, key.head_dim_v); - ggml_webgpu_hash_combine(seed, key.kv_direct); + ggml_webgpu_hash_combine(seed, key.k_direct); + ggml_webgpu_hash_combine(seed, key.v_direct); ggml_webgpu_hash_combine(seed, key.kv_overlap); ggml_webgpu_hash_combine(seed, key.has_mask); ggml_webgpu_hash_combine(seed, key.has_sinks); @@ -680,17 +690,19 @@ inline bool ggml_webgpu_flash_attn_float_vec4_aligned(const ggml_tensor * K, ggml_webgpu_flash_attn_float_vec4_aligned(V, storage_offset_alignment); } -inline bool ggml_webgpu_flash_attn_kv_direct(const ggml_tensor * Q, - const ggml_tensor * K, - const ggml_tensor * V, - uint32_t kv_direct_align) { - return K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16 && (Q->ne[0] % kv_direct_align == 0) && - (K->ne[1] % GGML_WEBGPU_KV_SEQ_PAD == 0); +inline bool ggml_webgpu_flash_attn_k_direct(const ggml_tensor * Q, const ggml_tensor * K, uint32_t kv_direct_align) { + return (K->type == GGML_TYPE_F16 || K->type == GGML_TYPE_Q8_0 || K->type == GGML_TYPE_Q4_0) && + (Q->ne[0] % kv_direct_align == 0) && (K->ne[1] % GGML_WEBGPU_KV_SEQ_PAD == 0); +} + +inline bool ggml_webgpu_flash_attn_v_direct(const ggml_tensor * Q, const ggml_tensor * V, uint32_t kv_direct_align) { + return ggml_webgpu_flash_attn_k_direct(Q, V, kv_direct_align); } inline ggml_webgpu_flash_attn_common_pipeline_key ggml_webgpu_flash_attn_make_common_pipeline_key( const ggml_webgpu_shader_lib_context & context, - uint32_t kv_direct_align) { + uint32_t kv_direct_align, + bool kv_overlap) { ggml_webgpu_flash_attn_common_pipeline_key key = {}; key.q_type = context.src0->type; key.k_type = context.src1->type; @@ -698,10 +710,11 @@ inline ggml_webgpu_flash_attn_common_pipeline_key ggml_webgpu_flash_attn_make_co key.dst_type = context.dst->type; key.head_dim_qk = (uint32_t) context.src0->ne[0]; key.head_dim_v = (uint32_t) context.src2->ne[0]; - key.kv_direct = ggml_webgpu_flash_attn_kv_direct(context.src0, context.src1, context.src2, kv_direct_align); - key.kv_overlap = ggml_webgpu_tensor_overlap(context.src1, context.src2); - key.has_mask = context.src3 != nullptr; - key.has_sinks = context.src4 != nullptr; + key.k_direct = ggml_webgpu_flash_attn_k_direct(context.src0, context.src1, kv_direct_align); + key.v_direct = ggml_webgpu_flash_attn_v_direct(context.src0, context.src2, kv_direct_align); + key.kv_overlap = kv_overlap; + key.has_mask = context.src3 != nullptr; + key.has_sinks = context.src4 != nullptr; key.uses_logit_softcap = ggml_get_op_params_f32(context.dst, 2) != 0.0f; return key; } @@ -786,9 +799,13 @@ inline std::vector ggml_webgpu_flash_attn_common_defines( defines.push_back("LOGIT_SOFTCAP"); variant += "_lgsc"; } - if (key.kv_direct) { - defines.push_back("KV_DIRECT"); - variant += "_kvdirect"; + if (key.k_direct) { + defines.push_back("K_DIRECT"); + variant += "_k_direct"; + } + if (key.v_direct) { + defines.push_back("V_DIRECT"); + variant += "_v_direct"; } if (key.kv_overlap) { defines.push_back("KV_OVERLAP"); @@ -807,6 +824,12 @@ inline std::vector ggml_webgpu_flash_attn_common_defines( if (ggml_is_quantized(key.k_type) || ggml_is_quantized(key.v_type)) { defines.push_back("U32_DEQUANT_HELPERS"); + if (ggml_is_quantized(key.k_type)) { + defines.push_back("LOADERS_QUANTIZED_K"); + } + if (ggml_is_quantized(key.v_type)) { + defines.push_back("LOADERS_QUANTIZED_V"); + } } return defines; @@ -1066,9 +1089,10 @@ struct ggml_webgpu_glu_pipeline_key { ggml_glu_op glu_op; ggml_type type; bool split; + bool src_overlap; bool operator==(const ggml_webgpu_glu_pipeline_key & other) const { - return glu_op == other.glu_op && type == other.type && split == other.split; + return glu_op == other.glu_op && type == other.type && split == other.split && src_overlap == other.src_overlap; } }; @@ -1078,6 +1102,7 @@ struct ggml_webgpu_glu_pipeline_key_hash { ggml_webgpu_hash_combine(seed, key.glu_op); ggml_webgpu_hash_combine(seed, key.type); ggml_webgpu_hash_combine(seed, key.split); + ggml_webgpu_hash_combine(seed, key.src_overlap); return seed; } }; @@ -1758,12 +1783,16 @@ class ggml_webgpu_shader_lib { return ssm_conv_pipelines[key]; } - webgpu_pipeline get_ssm_scan_pipeline(const ggml_webgpu_shader_lib_context & context) { + webgpu_pipeline get_ssm_scan_pipeline(const ggml_webgpu_shader_lib_context & context, + bool xbc_overlap, + bool a_overlap, + bool ids_overlap) { ggml_webgpu_ssm_scan_pipeline_key key = {}; key.type = context.dst->type; key.d_state = (int) context.src0->ne[0]; - key.xbc_overlap = ggml_webgpu_tensor_overlap(context.src1, context.src4) && - ggml_webgpu_tensor_overlap(context.src1, context.src5); + key.xbc_overlap = xbc_overlap; + key.a_overlap = a_overlap; + key.ids_overlap = ids_overlap; auto it = ssm_scan_pipelines.find(key); if (it != ssm_scan_pipelines.end()) { @@ -1798,7 +1827,12 @@ class ggml_webgpu_shader_lib { if (key.xbc_overlap) { defines.push_back("XBC_OVERLAP"); } - + if (key.a_overlap) { + defines.push_back("A_OVERLAP"); + } + if (key.ids_overlap) { + defines.push_back("IDS_OVERLAP"); + } variant += "_d" + std::to_string(key.d_state); auto processed = preprocessor.preprocess(wgsl_ssm_scan, defines); @@ -1806,6 +1840,8 @@ class ggml_webgpu_shader_lib { decisions->wg_size = wg_size; decisions->tokens_per_tile = tokens_per_tile; decisions->xbc_overlap = key.xbc_overlap; + decisions->a_overlap = key.a_overlap; + decisions->ids_overlap = key.ids_overlap; webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, variant); pipeline.context = decisions; ssm_scan_pipelines[key] = pipeline; @@ -2549,11 +2585,11 @@ class ggml_webgpu_shader_lib { return unary_pipelines[key]; } - webgpu_pipeline get_rms_norm_mul_pipeline(const ggml_webgpu_shader_lib_context & context) { + webgpu_pipeline get_rms_norm_mul_pipeline(const ggml_webgpu_shader_lib_context & context, bool src_overlap) { ggml_webgpu_rms_norm_mul_pipeline_key key = {}; key.inplace = ggml_webgpu_tensor_equal(context.src0, context.dst); key.overlap = ggml_webgpu_tensor_equal(context.src1, context.dst); - key.src_overlap = ggml_webgpu_tensor_overlap(context.src0, context.src1); + key.src_overlap = src_overlap; auto it = rms_norm_mul_pipelines.find(key); if (it != rms_norm_mul_pipelines.end()) { @@ -2589,13 +2625,13 @@ class ggml_webgpu_shader_lib { return rms_norm_mul_pipelines[key]; } - webgpu_pipeline get_binary_pipeline(const ggml_webgpu_shader_lib_context & context) { + webgpu_pipeline get_binary_pipeline(const ggml_webgpu_shader_lib_context & context, bool src_overlap) { ggml_webgpu_binary_pipeline_key key = {}; key.type = context.dst->type; key.op = context.dst->op; key.inplace = ggml_webgpu_tensor_equal(context.src0, context.dst); key.overlap = ggml_webgpu_tensor_equal(context.src1, context.dst); - key.src_overlap = ggml_webgpu_tensor_overlap(context.src0, context.src1); + key.src_overlap = src_overlap; auto it = binary_pipelines.find(key); if (it != binary_pipelines.end()) { @@ -2678,10 +2714,10 @@ class ggml_webgpu_shader_lib { return pipeline; } - webgpu_pipeline get_concat_pipeline(const ggml_webgpu_shader_lib_context & context) { + webgpu_pipeline get_concat_pipeline(const ggml_webgpu_shader_lib_context & context, bool src_overlap) { ggml_webgpu_concat_pipeline_key key = {}; key.type = context.dst->type; - key.src_overlap = ggml_webgpu_tensor_overlap(context.src0, context.src1); + key.src_overlap = src_overlap; auto it = concat_pipelines.find(key); if (it != concat_pipelines.end()) { @@ -2761,7 +2797,7 @@ class ggml_webgpu_shader_lib { return repeat_pipelines[key]; } - webgpu_pipeline get_flash_attn_pipeline(const ggml_webgpu_shader_lib_context & context) { + webgpu_pipeline get_flash_attn_pipeline(const ggml_webgpu_shader_lib_context & context, bool kv_overlap) { const bool can_use_subgroup_matrix = ggml_webgpu_flash_attn_can_use_subgroup_matrix_path( context.supports_subgroup_matrix, context.sg_mat_k, context.sg_mat_n, context.src0, context.src2); ggml_webgpu_flash_attn_decisions decisions = {}; @@ -2769,14 +2805,16 @@ class ggml_webgpu_shader_lib { decisions.q_tile = decisions.use_sg_matrix ? context.sg_mat_m : GGML_WEBGPU_FLASH_ATTN_TILE_Q_TILE; ggml_webgpu_flash_attn_pipeline_key key = {}; - key.common = - ggml_webgpu_flash_attn_make_common_pipeline_key(context, decisions.use_sg_matrix ? context.sg_mat_k : 1u); - key.common.kv_direct = decisions.use_sg_matrix && key.common.kv_direct; - key.use_sg_matrix = decisions.use_sg_matrix; + key.common = ggml_webgpu_flash_attn_make_common_pipeline_key( + context, decisions.use_sg_matrix ? context.sg_mat_k : 1u, kv_overlap); + key.common.k_direct &= decisions.use_sg_matrix && key.common.k_type == GGML_TYPE_F16; + key.common.v_direct &= decisions.use_sg_matrix && key.common.v_type == GGML_TYPE_F16; + key.use_sg_matrix = decisions.use_sg_matrix; const uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile( context.wg_mem_limit_bytes, decisions.q_tile, decisions.use_sg_matrix ? context.sg_mat_n : 1u, - key.common.head_dim_qk, key.common.head_dim_v, key.common.has_mask, key.common.kv_direct); + key.common.head_dim_qk, key.common.head_dim_v, key.common.has_mask, + key.common.k_direct || key.common.v_direct); GGML_ASSERT(max_kv_tile > 0); decisions.kv_tile = decisions.use_sg_matrix ? @@ -2788,7 +2826,7 @@ class ggml_webgpu_shader_lib { std::min(context.max_wg_size, std::max(GGML_WEBGPU_FLASH_ATTN_PREFERRED_WG_SIZE, GGML_WEBGPU_FLASH_ATTN_TILE_Q_TILE * context.max_subgroup_size)); - if (key.common.kv_direct) { + if (key.common.k_direct || key.common.v_direct) { decisions.kv_tile = std::min(decisions.kv_tile, GGML_WEBGPU_KV_SEQ_PAD); while (GGML_WEBGPU_KV_SEQ_PAD % decisions.kv_tile != 0) { decisions.kv_tile -= decisions.use_sg_matrix ? context.sg_mat_n : context.min_subgroup_size; @@ -2824,9 +2862,10 @@ class ggml_webgpu_shader_lib { return flash_attn_pipelines[key]; } - webgpu_pipeline get_flash_attn_vec_pipeline(const ggml_webgpu_shader_lib_context & context) { + webgpu_pipeline get_flash_attn_vec_pipeline(const ggml_webgpu_shader_lib_context & context, bool kv_overlap) { ggml_webgpu_flash_attn_vec_pipeline_key key = {}; - key.common = ggml_webgpu_flash_attn_make_common_pipeline_key(context, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH); + key.common = ggml_webgpu_flash_attn_make_common_pipeline_key(context, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH, + kv_overlap); auto it = flash_attn_vec_pipelines.find(key); if (it != flash_attn_vec_pipelines.end()) { @@ -2834,9 +2873,9 @@ class ggml_webgpu_shader_lib { } ggml_webgpu_flash_attn_vec_decisions decisions = {}; - decisions.kv_tile = - ggml_webgpu_flash_attn_get_vec_kv_tile(context.wg_mem_limit_bytes, key.common.head_dim_qk, - key.common.head_dim_v, key.common.has_mask, key.common.kv_direct); + decisions.kv_tile = ggml_webgpu_flash_attn_get_vec_kv_tile(context.wg_mem_limit_bytes, key.common.head_dim_qk, + key.common.head_dim_v, key.common.has_mask, + key.common.k_direct || key.common.v_direct); decisions.wg_size = context.max_subgroup_size; std::string variant = "flash_attn_vec"; @@ -2848,12 +2887,10 @@ class ggml_webgpu_shader_lib { variant += "_mask_blk"; } - uint32_t d_split = context.min_subgroup_size; - if (key.common.k_type == GGML_TYPE_F16 && key.common.v_type == GGML_TYPE_F16) { - const uint32_t D = key.common.head_dim_qk | key.common.head_dim_v; - const uint32_t D_lsb = D & (~(D - 1u)); - d_split = std::min(std::min(context.min_subgroup_size, 4u), std::max(D_lsb / 4u, 1u)); - } + uint32_t d_split = context.min_subgroup_size; + const uint32_t D = key.common.head_dim_qk | key.common.head_dim_v; + const uint32_t D_lsb = D & (~(D - 1u)); + d_split = std::min(std::min(context.min_subgroup_size, 4u), std::max(D_lsb / 4u, 1u)); defines.push_back(std::string("D_SPLIT=") + std::to_string(d_split)); variant += "_dsplit" + std::to_string(d_split); @@ -2984,11 +3021,12 @@ class ggml_webgpu_shader_lib { return cpy_pipelines[key]; } - webgpu_pipeline get_glu_pipeline(const ggml_webgpu_shader_lib_context & context) { + webgpu_pipeline get_glu_pipeline(const ggml_webgpu_shader_lib_context & context, bool src_overlap) { ggml_webgpu_glu_pipeline_key key = {}; key.glu_op = ggml_get_glu_op(context.dst); key.type = context.dst->type; key.split = (context.src1 != nullptr); + key.src_overlap = src_overlap; auto it = glu_pipelines.find(key); if (it != glu_pipelines.end()) { @@ -3039,7 +3077,10 @@ class ggml_webgpu_shader_lib { GGML_ABORT("Unsupported type for GLU shader"); } - if (key.split) { + if (key.src_overlap) { + defines.push_back("SRC_OVERLAP"); + variant += "_src_overlap"; + } else if (key.split) { variant += "_split"; } else { defines.push_back("NO_SPLIT"); @@ -3048,8 +3089,9 @@ class ggml_webgpu_shader_lib { defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size)); auto processed = preprocessor.preprocess(wgsl_glu, defines); - auto decisions = std::make_shared(); + auto decisions = std::make_shared(); decisions->wg_size = context.max_wg_size; + decisions->src_overlap = key.src_overlap; webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, variant); pipeline.context = decisions; glu_pipelines[key] = pipeline; diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index 75286ec731..370f05dfe6 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -374,18 +374,59 @@ static wgpu::Buffer ggml_webgpu_tensor_buf(const ggml_tensor * tensor) { return ctx->buffer; } -static size_t ggml_webgpu_tensor_misalignment(webgpu_context & ctx, const ggml_tensor * t) { +static size_t ggml_webgpu_tensor_misalignment(const ggml_tensor * t, size_t alignment) { size_t offset = ggml_webgpu_tensor_offset(t); - return offset & (ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment - 1); + return offset & (alignment - 1); +} + +static size_t ggml_webgpu_tensor_misalignment(webgpu_context & ctx, const ggml_tensor * t) { + return ggml_webgpu_tensor_misalignment(t, ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment); +} + +static size_t ggml_webgpu_tensor_align_offset(const ggml_tensor * t, size_t alignment) { + size_t offset = ggml_webgpu_tensor_offset(t); + return offset & ~(alignment - 1); } static size_t ggml_webgpu_tensor_align_offset(webgpu_context & ctx, const ggml_tensor * t) { - size_t offset = ggml_webgpu_tensor_offset(t); - return offset & ~(ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment - 1); + return ggml_webgpu_tensor_align_offset(t, ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment); } -static size_t ggml_webgpu_tensor_binding_size(webgpu_context & ctx, ggml_tensor * t) { - return ROUNDUP_POW2(ggml_nbytes(t) + ggml_webgpu_tensor_misalignment(ctx, t), WEBGPU_STORAGE_BUF_BINDING_MULT); +static size_t ggml_webgpu_tensor_binding_size(const ggml_tensor * t, size_t alignment) { + return ROUNDUP_POW2(ggml_nbytes(t) + ggml_webgpu_tensor_misalignment(t, alignment), + WEBGPU_STORAGE_BUF_BINDING_MULT); +} + +static size_t ggml_webgpu_tensor_binding_size(webgpu_context & ctx, const ggml_tensor * t) { + return ggml_webgpu_tensor_binding_size(t, ctx->global_ctx->capabilities.limits.minStorageBufferOffsetAlignment); +} + +static bool ggml_webgpu_tensor_binding_overlap(const webgpu_global_context & global_ctx, + const ggml_tensor * a, + const ggml_tensor * b) { + if (a->buffer != b->buffer) { + return false; + } + + const size_t alignment = global_ctx->capabilities.limits.minStorageBufferOffsetAlignment; + const size_t a_offset = ggml_webgpu_tensor_align_offset(a, alignment); + const size_t b_offset = ggml_webgpu_tensor_align_offset(b, alignment); + return a_offset < b_offset + ggml_webgpu_tensor_binding_size(b, alignment) && + b_offset < a_offset + ggml_webgpu_tensor_binding_size(a, alignment); +} + +static bool ggml_webgpu_tensor_binding_overlap_range(const webgpu_global_context & global_ctx, + ggml_tensor * tensor, + ggml_backend_buffer_t buffer, + size_t offset, + size_t size) { + if (tensor->buffer != buffer) { + return false; + } + + const size_t alignment = global_ctx->capabilities.limits.minStorageBufferOffsetAlignment; + const size_t tensor_offset = ggml_webgpu_tensor_align_offset(tensor, alignment); + return tensor_offset < offset + size && offset < tensor_offset + ggml_webgpu_tensor_binding_size(tensor, alignment); } struct ggml_webgpu_merged_binding_range { @@ -1188,39 +1229,76 @@ static webgpu_encoded_op ggml_webgpu_ssm_scan(webgpu_context & ctx, ggml_webgpu_shader_lib_context shader_lib_ctx = {}; shader_lib_ctx.src0 = src0; shader_lib_ctx.src1 = src1; + shader_lib_ctx.src2 = src2; + shader_lib_ctx.src3 = src3; shader_lib_ctx.src4 = src4; shader_lib_ctx.src5 = src5; shader_lib_ctx.dst = dst; shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; shader_lib_ctx.supports_subgroups = ctx->global_ctx->capabilities.supports_subgroups; + bool xbc_overlap = ggml_webgpu_tensor_binding_overlap(ctx->global_ctx, src1, src2) || + ggml_webgpu_tensor_binding_overlap(ctx->global_ctx, src1, src4) || + ggml_webgpu_tensor_binding_overlap(ctx->global_ctx, src1, src5) || + ggml_webgpu_tensor_binding_overlap(ctx->global_ctx, src2, src4) || + ggml_webgpu_tensor_binding_overlap(ctx->global_ctx, src2, src5) || + ggml_webgpu_tensor_binding_overlap(ctx->global_ctx, src4, src5); + bool a_overlap = false; + bool ids_overlap = false; + ggml_webgpu_merged_binding_range xbc_merged_range = {}; + if (xbc_overlap) { + xbc_merged_range = ggml_webgpu_tensor_merged_binding_range(ctx, { src1, src2, src4, src5 }); + a_overlap = ggml_webgpu_tensor_binding_overlap_range(ctx->global_ctx, src3, src1->buffer, + xbc_merged_range.offset, xbc_merged_range.size); + if (a_overlap) { + xbc_merged_range = ggml_webgpu_tensor_merged_binding_range(ctx, { src1, src2, src3, src4, src5 }); + } + ids_overlap = ggml_webgpu_tensor_binding_overlap_range(ctx->global_ctx, src6, src1->buffer, + xbc_merged_range.offset, xbc_merged_range.size); + if (ids_overlap) { + xbc_merged_range = + a_overlap ? ggml_webgpu_tensor_merged_binding_range(ctx, { src1, src2, src3, src4, src5, src6 }) : + ggml_webgpu_tensor_merged_binding_range(ctx, { src1, src2, src4, src5, src6 }); + } + } - webgpu_pipeline pipeline = ctx->shader_lib->get_ssm_scan_pipeline(shader_lib_ctx); - auto * decisions = static_cast(pipeline.context.get()); - const bool xbc_overlap = decisions->xbc_overlap; + webgpu_pipeline pipeline = + ctx->shader_lib->get_ssm_scan_pipeline(shader_lib_ctx, xbc_overlap, a_overlap, ids_overlap); + auto * decisions = static_cast(pipeline.context.get()); + xbc_overlap = decisions->xbc_overlap; + a_overlap = decisions->a_overlap; + ids_overlap = decisions->ids_overlap; uint32_t offset_x = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src1) / ggml_type_size(src1->type)); + uint32_t offset_dt = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src2) / ggml_type_size(src2->type)); + uint32_t offset_A = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src3) / ggml_type_size(src3->type)); uint32_t offset_B = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src4) / ggml_type_size(src4->type)); uint32_t offset_C = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src5) / ggml_type_size(src5->type)); + uint32_t offset_ids = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src6) / ggml_type_size(src6->type)); size_t xbc_bind_offset = 0; size_t xbc_bind_size = 0; if (xbc_overlap) { - const ggml_webgpu_merged_binding_range merged_range = - ggml_webgpu_tensor_merged_binding_range(ctx, { src1, src4, src5 }); - xbc_bind_offset = merged_range.offset; - xbc_bind_size = merged_range.size; - offset_x = ggml_webgpu_tensor_merged_element_offset(src1, merged_range); - offset_B = ggml_webgpu_tensor_merged_element_offset(src4, merged_range); - offset_C = ggml_webgpu_tensor_merged_element_offset(src5, merged_range); + xbc_bind_offset = xbc_merged_range.offset; + xbc_bind_size = xbc_merged_range.size; + offset_x = ggml_webgpu_tensor_merged_element_offset(src1, xbc_merged_range); + offset_dt = ggml_webgpu_tensor_merged_element_offset(src2, xbc_merged_range); + if (a_overlap) { + offset_A = ggml_webgpu_tensor_merged_element_offset(src3, xbc_merged_range); + } + offset_B = ggml_webgpu_tensor_merged_element_offset(src4, xbc_merged_range); + offset_C = ggml_webgpu_tensor_merged_element_offset(src5, xbc_merged_range); + if (ids_overlap) { + offset_ids = ggml_webgpu_tensor_merged_element_offset(src6, xbc_merged_range); + } } std::vector params = { (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src0) / ggml_type_size(src0->type)), offset_x, - (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src2) / ggml_type_size(src2->type)), - (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src3) / ggml_type_size(src3->type)), + offset_dt, + offset_A, offset_B, offset_C, - (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src6) / ggml_type_size(src6->type)), + offset_ids, (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)), (uint32_t) (src0->nb[1] / ggml_type_size(src0->type)), @@ -1260,10 +1338,19 @@ static webgpu_encoded_op ggml_webgpu_ssm_scan(webgpu_context & ctx, if (xbc_overlap) { entries.push_back( ggml_webgpu_make_bind_group_entry(1, ggml_webgpu_tensor_buf(src1), xbc_bind_offset, xbc_bind_size)); - entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 2, src2)); - entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 3, src3)); - entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 4, src6)); - entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 5, dst)); + if (ids_overlap) { + if (!a_overlap) { + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 2, src3)); + } + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, a_overlap ? 2 : 3, dst)); + } else if (a_overlap) { + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 2, src6)); + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 3, dst)); + } else { + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 2, src3)); + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 3, src6)); + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 4, dst)); + } } else { entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, src1)); entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 2, src2)); @@ -1381,11 +1468,10 @@ static std::optional ggml_webgpu_set_rows(webgpu_context & ct (uint32_t) (idx->ne[1]), (uint32_t) (idx->ne[2]) }; - std::vector entries = { - ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src), - ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, idx), - ggml_webgpu_make_tensor_bind_group_entry(ctx, 2, dst), - }; + std::vector entries; + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src)); + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, idx)); + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 2, dst)); if (decisions->i64_idx) { entries.push_back(ggml_webgpu_make_bind_group_entry(3, ctx->set_rows_dev_error_buf, 0, @@ -1892,7 +1978,7 @@ static ggml_webgpu_flash_attn_op ggml_webgpu_flash_attn_prepare(webgpu_context & op.has_mask = mask != nullptr; op.has_sinks = sinks != nullptr; - op.kv_overlap = ggml_webgpu_tensor_overlap(K, V); + op.kv_overlap = ggml_webgpu_tensor_binding_overlap(ctx->global_ctx, K, V); uint32_t offset_k = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, K) / ggml_type_size(K->type)); uint32_t offset_v = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, V) / ggml_type_size(V->type)); @@ -1964,7 +2050,7 @@ static uint32_t ggml_webgpu_flash_attn_vec_nwg(uint32_t vec_nwg_cap, uint32_t kv } static webgpu_encoded_op ggml_webgpu_flash_attn_direct(webgpu_context & ctx, const ggml_webgpu_flash_attn_op & op) { - webgpu_pipeline pipeline = ctx->shader_lib->get_flash_attn_pipeline(op.shader_lib_ctx); + webgpu_pipeline pipeline = ctx->shader_lib->get_flash_attn_pipeline(op.shader_lib_ctx, op.kv_overlap); auto * decisions = static_cast(pipeline.context.get()); uint32_t wg_per_head = CEIL_DIV(op.shader_lib_ctx.src0->ne[1], decisions->q_tile); uint32_t wg_x = wg_per_head * op.shader_lib_ctx.src0->ne[2] * op.shader_lib_ctx.src0->ne[3]; @@ -1979,7 +2065,7 @@ static webgpu_encoded_op ggml_webgpu_flash_attn_vec(webgpu_context & ct ggml_tensor * sinks, ggml_tensor * dst, ggml_webgpu_flash_attn_op op) { - webgpu_pipeline pipeline = ctx->shader_lib->get_flash_attn_vec_pipeline(op.shader_lib_ctx); + webgpu_pipeline pipeline = ctx->shader_lib->get_flash_attn_vec_pipeline(op.shader_lib_ctx, op.kv_overlap); auto * decisions = static_cast(pipeline.context.get()); wgpu::Buffer blk_buf = {}; @@ -2249,8 +2335,9 @@ static webgpu_encoded_op ggml_webgpu_binary_op(webgpu_context & ctx, shader_lib_ctx.dst = dst; shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; - webgpu_pipeline pipeline = ctx->shader_lib->get_binary_pipeline(shader_lib_ctx); - auto * decisions = static_cast(pipeline.context.get()); + const bool src_overlap = ggml_webgpu_tensor_binding_overlap(ctx->global_ctx, src0, src1); + webgpu_pipeline pipeline = ctx->shader_lib->get_binary_pipeline(shader_lib_ctx, src_overlap); + auto * decisions = static_cast(pipeline.context.get()); uint32_t ne = (uint32_t) ggml_nelements(dst); @@ -2372,6 +2459,9 @@ static webgpu_encoded_op ggml_webgpu_concat(webgpu_context & ctx, ggml_tensor * dst) { uint32_t ne = (uint32_t) ggml_nelements(dst); uint32_t dim = (uint32_t) dst->op_params[0]; + if (ggml_nbytes(src0) == 0 && ggml_nbytes(src1) == 0) { + return {}; + } ggml_webgpu_shader_lib_context shader_lib_ctx = {}; shader_lib_ctx.src0 = src0; @@ -2379,20 +2469,34 @@ static webgpu_encoded_op ggml_webgpu_concat(webgpu_context & ctx, shader_lib_ctx.dst = dst; shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; - webgpu_pipeline pipeline = ctx->shader_lib->get_concat_pipeline(shader_lib_ctx); - auto * decisions = static_cast(pipeline.context.get()); + const bool src_overlap = ggml_webgpu_tensor_binding_overlap(ctx->global_ctx, src0, src1) || + ggml_nbytes(src0) == 0 || ggml_nbytes(src1) == 0; + webgpu_pipeline pipeline = ctx->shader_lib->get_concat_pipeline(shader_lib_ctx, src_overlap); + auto * decisions = static_cast(pipeline.context.get()); uint32_t offset_src0 = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src0) / ggml_type_size(src0->type)); uint32_t offset_src1 = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src1) / ggml_type_size(src1->type)); size_t merged_offset = 0; size_t merged_size = 0; if (decisions->src_overlap) { - const ggml_webgpu_merged_binding_range merged_range = - ggml_webgpu_tensor_merged_binding_range(ctx, { src0, src1 }); - merged_offset = merged_range.offset; - merged_size = merged_range.size; - offset_src0 = ggml_webgpu_tensor_merged_element_offset(src0, merged_range); - offset_src1 = ggml_webgpu_tensor_merged_element_offset(src1, merged_range); + if (ggml_nbytes(src0) == 0) { + merged_offset = ggml_webgpu_tensor_align_offset(ctx, src1); + merged_size = ggml_webgpu_tensor_binding_size(ctx, src1); + offset_src0 = 0; + offset_src1 = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src1) / ggml_type_size(src1->type)); + } else if (ggml_nbytes(src1) == 0) { + merged_offset = ggml_webgpu_tensor_align_offset(ctx, src0); + merged_size = ggml_webgpu_tensor_binding_size(ctx, src0); + offset_src0 = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src0) / ggml_type_size(src0->type)); + offset_src1 = 0; + } else { + const ggml_webgpu_merged_binding_range merged_range = + ggml_webgpu_tensor_merged_binding_range(ctx, { src0, src1 }); + merged_offset = merged_range.offset; + merged_size = merged_range.size; + offset_src0 = ggml_webgpu_tensor_merged_element_offset(src0, merged_range); + offset_src1 = ggml_webgpu_tensor_merged_element_offset(src1, merged_range); + } } std::vector params = { ne, @@ -2518,8 +2622,9 @@ static std::optional ggml_webgpu_rms_norm_mul(webgpu_context shader_lib_ctx.dst = dst; shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; - webgpu_pipeline pipeline = ctx->shader_lib->get_rms_norm_mul_pipeline(shader_lib_ctx); - auto * decisions = static_cast(pipeline.context.get()); + const bool src_overlap = ggml_webgpu_tensor_binding_overlap(ctx->global_ctx, rn_src, mul_src); + webgpu_pipeline pipeline = ctx->shader_lib->get_rms_norm_mul_pipeline(shader_lib_ctx, src_overlap); + auto * decisions = static_cast(pipeline.context.get()); if (decisions->src_overlap) { const ggml_webgpu_merged_binding_range merged_range = @@ -2678,15 +2783,30 @@ static webgpu_encoded_op ggml_webgpu_glu(webgpu_context & ctx, shader_lib_ctx.dst = dst; shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; - webgpu_pipeline pipeline = ctx->shader_lib->get_glu_pipeline(shader_lib_ctx); + const bool src_overlap = src1 != nullptr && ggml_webgpu_tensor_binding_overlap(ctx->global_ctx, src0, src1); + webgpu_pipeline pipeline = ctx->shader_lib->get_glu_pipeline(shader_lib_ctx, src_overlap); - auto * decisions = static_cast(pipeline.context.get()); + auto * decisions = static_cast(pipeline.context.get()); const int split = (src1 != nullptr); + uint32_t offset_src0 = (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src0) / ggml_type_size(src0->type)); + uint32_t offset_src1 = + src1 != nullptr ? (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src1) / ggml_type_size(src1->type)) : 0; + size_t merged_offset = 0; + size_t merged_size = 0; + if (decisions->src_overlap) { + const ggml_webgpu_merged_binding_range merged_range = + ggml_webgpu_tensor_merged_binding_range(ctx, { src0, src1 }); + merged_offset = merged_range.offset; + merged_size = merged_range.size; + offset_src0 = ggml_webgpu_tensor_merged_element_offset(src0, merged_range); + offset_src1 = ggml_webgpu_tensor_merged_element_offset(src1, merged_range); + } + std::vector params = { - (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src0) / ggml_type_size(src0->type)), - src1 != nullptr ? (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src1) / ggml_type_size(src1->type)) : 0, + offset_src0, + offset_src1, (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)), (uint32_t) (src0->nb[1] / ggml_type_size(src0->type)), (uint32_t) (src0->nb[2] / ggml_type_size(src0->type)), @@ -2709,11 +2829,15 @@ static webgpu_encoded_op ggml_webgpu_glu(webgpu_context & ctx, ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 3)), // limit, for swiglu_oai }; - std::vector entries = { - ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src0), - }; - uint32_t dst_binding = 1; - if (split) { + std::vector entries; + uint32_t dst_binding = 1; + if (decisions->src_overlap) { + entries.push_back( + ggml_webgpu_make_bind_group_entry(0, ggml_webgpu_tensor_buf(src0), merged_offset, merged_size)); + } else { + entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src0)); + } + if (split && !decisions->src_overlap) { dst_binding = 2; entries.push_back(ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, src1)); } @@ -3715,7 +3839,8 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer const auto & capabilities = ctx->webgpu_global_ctx->capabilities; if (ggml_webgpu_flash_attn_use_vec_path(ctx->webgpu_global_ctx, Q, K, V)) { const bool kv_direct = - ggml_webgpu_flash_attn_kv_direct(Q, K, V, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH); + ggml_webgpu_flash_attn_k_direct(Q, K, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH) || + ggml_webgpu_flash_attn_v_direct(Q, V, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH); const uint32_t kv_tile = ggml_webgpu_flash_attn_get_vec_kv_tile( capabilities.limits.maxComputeWorkgroupStorageSize, (uint32_t) Q->ne[0], (uint32_t) V->ne[0], mask != nullptr, kv_direct); @@ -4285,8 +4410,8 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const if (!supports_op) { break; } - if (ggml_webgpu_tensor_overlap(src1, src2) && src1->type != src2->type && - !ggml_is_quantized(src1->type) && !ggml_is_quantized(src2->type)) { + if (ggml_webgpu_tensor_binding_overlap(ctx->webgpu_global_ctx, src1, src2) && + src1->type != src2->type && !ggml_is_quantized(src1->type) && !ggml_is_quantized(src2->type)) { supports_op = false; break; } @@ -4324,9 +4449,10 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const const uint32_t q_tile = use_subgroup_matrix ? capabilities.sg_mat_m : GGML_WEBGPU_FLASH_ATTN_TILE_Q_TILE; const uint32_t kv_granularity = use_subgroup_matrix ? capabilities.sg_mat_n : 1u; - const bool kv_direct = use_subgroup_matrix ? - ggml_webgpu_flash_attn_kv_direct(src0, src1, src2, capabilities.sg_mat_k) : - false; + const bool kv_direct = use_subgroup_matrix ? + ggml_webgpu_flash_attn_k_direct(src0, src1, capabilities.sg_mat_k) || + ggml_webgpu_flash_attn_v_direct(src0, src2, capabilities.sg_mat_k) : + false; const uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile( capabilities.limits.maxComputeWorkgroupStorageSize, q_tile, kv_granularity, (uint32_t) src0->ne[0], (uint32_t) src2->ne[0], op->src[3] != nullptr, kv_direct); diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl index 6634fbd657..b0cf2853e0 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/common_decls.tmpl @@ -9,6 +9,12 @@ fn get_byte_i32(value: u32, index: u32) -> i32 { #endif #ifdef U32_DEQUANT_HELPERS + +fn f16_from_u16(bits: u32) -> f16 { + let packed = unpack2x16float(bits); + return f16(packed[0]); +} + #ifdef DECLARE_BYTE_LOADERS_SRC fn load_u16_at_src(byte_offset: u32) -> u32 { let word = src[byte_offset / 4u]; @@ -36,7 +42,7 @@ fn load_f16_as_f32_at_src(byte_offset: u32) -> f32 { let d_bits = (word >> shift) & 0xFFFFu; return unpack2x16float(d_bits)[0]; } -#endif +#endif // DECLARE_BYTE_LOADERS_SRC #ifdef DECLARE_BYTE_LOADERS_SRC0 fn load_u16_at_src0(byte_offset: u32) -> u32 { @@ -72,8 +78,47 @@ fn load_f16_as_f32_at_src0(byte_offset: u32) -> f32 { let d_bits = (word >> shift) & 0xFFFFu; return unpack2x16float(d_bits)[0]; } -#endif -#endif +#endif // DECLARE_BYTE_LOADERS_SRC0 + +#ifdef LOADERS_QUANTIZED_K +fn load_k_u16_at(byte_offset: u32) -> u32 { + let word = K[byte_offset / 4u]; + let shift = (byte_offset & 2u) * 8u; + return (word >> shift) & 0xFFFFu; +} + +fn load_k_u32_at(byte_offset: u32) -> u32 { + let word_idx = byte_offset / 4u; + let shift = (byte_offset & 3u) * 8u; + let lo = K[word_idx]; + if (shift == 0u) { + return lo; + } + let hi = K[word_idx + 1u]; + return (lo >> shift) | (hi << (32u - shift)); +} +#endif // LOADERS_QUANTIZED_K + +#ifdef LOADERS_QUANTIZED_V +fn load_v_u16_at(byte_offset: u32) -> u32 { + let word = V[byte_offset / 4u]; + let shift = (byte_offset & 2u) * 8u; + return (word >> shift) & 0xFFFFu; +} + +fn load_v_u32_at(byte_offset: u32) -> u32 { + let word_idx = byte_offset / 4u; + let shift = (byte_offset & 3u) * 8u; + let lo = V[word_idx]; + if (shift == 0u) { + return lo; + } + let hi = V[word_idx + 1u]; + return (lo >> shift) | (hi << (32u - shift)); +} +#endif // LOADERS_QUANTIZED_V + +#endif // U32_DEQUANT_HELPERS diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl index 9767ca3d75..75f33e68ae 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn.wgsl @@ -138,7 +138,7 @@ const FLOAT_MIN: f32 = -1.0e9; // The number of Q rows processed per workgroup var q_shmem: array; -#ifndef KV_DIRECT +#if !defined(K_DIRECT) || !defined(V_DIRECT) const kv_shmem_size = KV_TILE * max(HEAD_DIM_QK, HEAD_DIM_V); // we can reuse the same shmem for K and V since we only need one at a time var kv_shmem: array; @@ -183,13 +183,12 @@ fn load_kx4(buf: ptr>, read_write>, scalar_index: u3 return (*buf)[scalar_index >> 2u]; } -#ifndef KV_DIRECT +#if !defined(K_DIRECT) || !defined(V_DIRECT) #define QUANT_SHMEM kv_shmem #define QUANT_OUT_TYPE f16 -#include "quant_inner_loops.tmpl" #include "flash_attn_quant_staging.tmpl" -#if !defined(K_Q4_0) && !defined(K_Q8_0) +#if !defined(K_DIRECT) && !defined(K_Q4_0) && !defined(K_Q8_0) fn load_k_tile_block(local_x: u32, kv_count: u32, kv_tile: u32, k_head_offset: u32) { for (var elem_idx = local_x; elem_idx < KV_TILE * HEAD_DIM_QK; elem_idx += WG_SIZE) { let k_row = elem_idx / HEAD_DIM_QK; @@ -204,7 +203,7 @@ fn load_k_tile_block(local_x: u32, kv_count: u32, kv_tile: u32, k_head_offset: u } #endif -#if !defined(V_Q4_0) && !defined(V_Q8_0) +#if !defined(V_DIRECT) && !defined(V_Q4_0) && !defined(V_Q8_0) fn load_v_tile_block(local_x: u32, kv_count: u32, kv_tile: u32, v_head_offset: u32) { for (var elem_idx = local_x; elem_idx < KV_TILE * HEAD_DIM_V; elem_idx += WG_SIZE) { let v_row = elem_idx / HEAD_DIM_V; @@ -296,7 +295,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3, } // load k tile into shared memory -#ifndef KV_DIRECT +#ifndef K_DIRECT load_k_tile_block(local_id.x, kv_count, kv_tile, k_head_offset); #endif @@ -306,7 +305,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3, // TODO: this loop seems to be the current largest bottleneck // this bracket exists to scope the lifetime of variables, reducing register pressure { -#ifdef KV_DIRECT +#ifdef K_DIRECT let k_block_row = kv_tile + subgroup_id * SG_MAT_N; var k_global_offset = k_head_offset + k_block_row * params.stride_k1; #else @@ -318,7 +317,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3, var q_cur = subgroupMatrixLoad>(&q_shmem, 0u, false, HEAD_DIM_QK); -#ifdef KV_DIRECT +#ifdef K_DIRECT var k_cur = subgroupMatrixLoad>(&K, k_global_offset + 0u, true, params.stride_k1); #else var k_cur = subgroupMatrixLoad>(&kv_shmem, k_block_offset + 0u, true, HEAD_DIM_QK); @@ -328,7 +327,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3, for (; t + 1u < HEAD_DIM_QK / SG_MAT_K; t += 2u) { let h0 = t * SG_MAT_K; var q0 = subgroupMatrixLoad>(&q_shmem, h0, false, HEAD_DIM_QK); -#ifdef KV_DIRECT +#ifdef K_DIRECT var k0 = subgroupMatrixLoad>(&K, k_global_offset + h0, true, params.stride_k1); #else var k0 = subgroupMatrixLoad>(&kv_shmem, k_block_offset + h0, true, HEAD_DIM_QK); @@ -339,7 +338,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3, let h1 = (t + 1u) * SG_MAT_K; var q1g = subgroupMatrixLoad>(&q_shmem, h1, false, HEAD_DIM_QK); -#ifdef KV_DIRECT +#ifdef K_DIRECT var k1g = subgroupMatrixLoad>(&K, k_global_offset + h1, true, params.stride_k1); #else var k1g = subgroupMatrixLoad>(&kv_shmem, k_block_offset + h1, true, HEAD_DIM_QK); @@ -353,7 +352,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3, if (t < HEAD_DIM_QK / SG_MAT_K) { let h = t * SG_MAT_K; var qn = subgroupMatrixLoad>(&q_shmem, h, false, HEAD_DIM_QK); -#ifdef KV_DIRECT +#ifdef K_DIRECT var kn = subgroupMatrixLoad>(&K, k_global_offset + h, true, params.stride_k1); #else var kn = subgroupMatrixLoad>(&kv_shmem, k_block_offset + h, true, HEAD_DIM_QK); @@ -365,7 +364,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3, acc = subgroupMatrixMultiplyAccumulate(q_cur, k_cur, acc); -#ifdef KV_DIRECT +#ifdef K_DIRECT k_global_offset += num_subgroups * SG_MAT_N * params.stride_k1; #else k_block_offset += num_subgroups * SG_MAT_N * HEAD_DIM_QK; @@ -436,7 +435,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3, } // load v tile into shared memory -#ifndef KV_DIRECT +#ifndef V_DIRECT load_v_tile_block(local_id.x, kv_count, kv_tile, v_head_offset); #endif @@ -464,7 +463,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3, ); // load V submatrix from global or shared memory -#ifdef KV_DIRECT +#ifdef V_DIRECT let v_block_row = kv_tile + kv_block * SG_MAT_N; let v_global_offset = v_head_offset + v_block_row * params.stride_v1 + head_dim_block; var v_sg_mat: subgroup_matrix_right = subgroupMatrixLoad>( diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_quant_staging.tmpl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_quant_staging.tmpl index 8f41eb7bfd..1c23260df0 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_quant_staging.tmpl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_quant_staging.tmpl @@ -1,3 +1,5 @@ +#include "quant_inner_loops.tmpl" + #define BLOCK_SIZE 32 #define BLOCKS_K ((HEAD_DIM_QK + BLOCK_SIZE - 1) / BLOCK_SIZE) #define BLOCKS_V ((HEAD_DIM_V + BLOCK_SIZE - 1) / BLOCK_SIZE) @@ -26,49 +28,6 @@ #define V_BYTES_PER_INNER_LOOP 4u #endif -#if defined(K_Q4_0) || defined(K_Q8_0) -fn load_k_u16_at(byte_offset: u32) -> u32 { - let word = K[byte_offset / 4u]; - let shift = (byte_offset & 2u) * 8u; - return (word >> shift) & 0xFFFFu; -} - -fn load_k_u32_at(byte_offset: u32) -> u32 { - let word_idx = byte_offset / 4u; - let shift = (byte_offset & 3u) * 8u; - let lo = K[word_idx]; - if (shift == 0u) { - return lo; - } - let hi = K[word_idx + 1u]; - return (lo >> shift) | (hi << (32u - shift)); -} -#endif - -#if defined(V_Q4_0) || defined(V_Q8_0) -fn load_v_u16_at(byte_offset: u32) -> u32 { - let word = V[byte_offset / 4u]; - let shift = (byte_offset & 2u) * 8u; - return (word >> shift) & 0xFFFFu; -} - -fn load_v_u32_at(byte_offset: u32) -> u32 { - let word_idx = byte_offset / 4u; - let shift = (byte_offset & 3u) * 8u; - let lo = V[word_idx]; - if (shift == 0u) { - return lo; - } - let hi = V[word_idx + 1u]; - return (lo >> shift) | (hi << (32u - shift)); -} -#endif - -fn f16_from_u16(bits: u32) -> f16 { - let packed = unpack2x16float(bits); - return f16(packed[0]); -} - #if defined(K_Q4_0) || defined(K_Q8_0) fn load_k_tile_block(local_x: u32, kv_count: u32, kv_tile: u32, k_head_offset: u32) { for (var elem_idx = local_x * K_NQ; elem_idx < kv_count * HEAD_DIM_QK; elem_idx += WG_SIZE * K_NQ) { diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl index e68934113f..43f4fe7cac 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_tile.wgsl @@ -153,7 +153,6 @@ var p_shmem: array; #define QUANT_SHMEM kv_shmem #define QUANT_OUT_TYPE f16 -#include "quant_inner_loops.tmpl" #include "flash_attn_quant_staging.tmpl" #if !defined(K_Q4_0) && !defined(K_Q8_0) @@ -270,7 +269,9 @@ fn main(@builtin(workgroup_id) wg_id: vec3, local_scores[slot] = FLOAT_MIN; } -#ifndef KV_DIRECT + // The tile path stages K/V in shared memory so each tile can be reused across + // Q_TILE query rows. It therefore does not use the direct path. +#ifndef K_DIRECT load_k_tile_block(local_id.x, kv_count, kv_tile, k_head_offset); #endif @@ -333,7 +334,9 @@ fn main(@builtin(workgroup_id) wg_id: vec3, workgroupBarrier(); -#ifndef KV_DIRECT + // The tile path stages K/V in shared memory so each tile can be reused across + // Q_TILE query rows. It therefore does not use the direct path. +#ifndef V_DIRECT load_v_tile_block(local_id.x, kv_count, kv_tile, v_head_offset); #endif diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl index d512762419..b8e0be90d9 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/flash_attn_vec_split.wgsl @@ -196,49 +196,35 @@ struct Params { // Just a very small float value. const FLOAT_MIN: f32 = -1.0e9; +const kv_shmem_size = KV_TILE * max(HEAD_DIM_QK, HEAD_DIM_V); var q_shmem: array; - -#ifndef KV_DIRECT -const kv_shmem_size = KV_TILE * max(HEAD_DIM_QK, HEAD_DIM_V); -// we can reuse the same shmem for K and V since we only need one at a time -var kv_shmem: array; -#endif - var o_shmem: array; +// note that we reuse the same storage for both since we only need one at a time +var inter_shmem: array; #ifdef MASK // storage for mask values var mask_shmem: array; #endif -// note that we reuse the same storage for both since we only need one at a time -var inter_shmem: array; - -// Storage for row max and exp sum during online softmax -fn calc_softmax_term(kv_idx: u32, slope: f32, has_bias: bool, apply_mask: bool) -> f32 { - var v = select(FLOAT_MIN, - inter_shmem[kv_idx] * params.scale, - kv_idx < KV_TILE); -#ifdef LOGIT_SOFTCAP - v = params.logit_softcap * tanh(v); +#if defined(K_DIRECT) || defined(V_DIRECT) +// Shared memory for scale factor (d) in quantized K/V. Multiple threads use the same value, +// so caching it is more efficient, even on the direct path. +var d_shmem: array; #endif -#ifdef MASK - if (apply_mask) { - var mask_val = select(0.0, mask_shmem[kv_idx], kv_idx < KV_TILE); - v += select(mask_val, slope * mask_val, has_bias); - } -#endif - return v; -} -#ifndef KV_DIRECT +// K/V shared memory handling +#if !defined(K_DIRECT) || !defined(V_DIRECT) + +// we can reuse the same shmem for K and V since we only need one at a time +var kv_shmem: array; + #define QUANT_SHMEM kv_shmem #define QUANT_OUT_TYPE f32 -#include "quant_inner_loops.tmpl" #include "flash_attn_quant_staging.tmpl" -#if !defined(K_Q4_0) && !defined(K_Q8_0) +#if !defined(K_DIRECT) && !defined(K_Q4_0) && !defined(K_Q8_0) fn load_k_tile_block(local_x: u32, kv_count: u32, kv_tile: u32, k_head_offset: u32) { for (var elem_idx = local_x * 4u; elem_idx < KV_TILE * HEAD_DIM_QK; elem_idx += WG_SIZE * 4u) { let k_row = elem_idx / HEAD_DIM_QK; @@ -256,7 +242,7 @@ fn load_k_tile_block(local_x: u32, kv_count: u32, kv_tile: u32, k_head_offset: u } #endif -#if !defined(V_Q4_0) && !defined(V_Q8_0) +#if !defined(V_DIRECT) && !defined(V_Q4_0) && !defined(V_Q8_0) fn load_v_tile_block(local_x: u32, kv_count: u32, kv_tile: u32, v_head_offset: u32) { for (var elem_idx = local_x * 4u; elem_idx < KV_TILE * HEAD_DIM_V; elem_idx += WG_SIZE * 4u) { let v_row = elem_idx / HEAD_DIM_V; @@ -273,7 +259,24 @@ fn load_v_tile_block(local_x: u32, kv_count: u32, kv_tile: u32, v_head_offset: u } } #endif +#endif // !defined(K_DIRECT) || !defined(V_DIRECT) + +// Storage for row max and exp sum during online softmax +fn calc_softmax_term(kv_idx: u32, slope: f32, has_bias: bool, apply_mask: bool) -> f32 { + var v = select(FLOAT_MIN, + inter_shmem[kv_idx] * params.scale, + kv_idx < KV_TILE); +#ifdef LOGIT_SOFTCAP + v = params.logit_softcap * tanh(v); #endif +#ifdef MASK + if (apply_mask) { + var mask_val = select(0.0, mask_shmem[kv_idx], kv_idx < KV_TILE); + v += select(mask_val, slope * mask_val, has_bias); + } +#endif + return v; +} @compute @workgroup_size(WG_SIZE) fn main(@builtin(workgroup_id) wg_id: vec3, @@ -355,12 +358,31 @@ fn main(@builtin(workgroup_id) wg_id: vec3, inter_shmem[elem_idx] = 0.0; } - // load k tile into shared memory -#ifndef KV_DIRECT - load_k_tile_block(local_id.x, kv_count, kv_tile, k_head_offset); +#ifdef K_DIRECT + // load only the scale factor (d) from each quantized block into shared memory on the direct path. +#if defined(K_Q8_0) + for (var j = local_id.x * 32; j < KV_TILE * HEAD_DIM_QK; j += WG_SIZE * 32) { + let kv_row = kv_tile + j / HEAD_DIM_QK; + let block_idx = (j % HEAD_DIM_QK) / 32; + let block_byte_base = 34 * (k_head_offset + kv_row * params.stride_k1 + block_idx); + let d = f32(f16_from_u16(load_k_u16_at(block_byte_base))); + d_shmem[j / 32] = d; + } +#elif defined(K_Q4_0) + for (var j = local_id.x * 32; j < KV_TILE * HEAD_DIM_QK; j += WG_SIZE * 32) { + let kv_row = kv_tile + j / HEAD_DIM_QK; + let block_idx = (j % HEAD_DIM_QK) / 32; + let block_byte_base = 18 * (k_head_offset + kv_row * params.stride_k1 + block_idx); + let d = f32(f16_from_u16(load_k_u16_at(block_byte_base))); + d_shmem[j / 32] = d; + } #endif +#else + // load k tile into shared memory + load_k_tile_block(local_id.x, kv_count, kv_tile, k_head_offset); +#endif // defined(K_DIRECT) - workgroupBarrier(); + workgroupBarrier(); // accumulate q block * k block into registers across the entire KV tile if (!skip_tile) { @@ -381,9 +403,40 @@ fn main(@builtin(workgroup_id) wg_id: vec3, q_shmem[q_off + 1u], q_shmem[q_off + 2u], q_shmem[q_off + 3u]); -#ifdef KV_DIRECT +#ifdef K_DIRECT +#if defined(K_Q8_0) + let kv_row = kv_tile + kv_idx; + let block_idx = (i * 4u) / 32; + let id_in_block = (i * 4u) % 32; + let block_byte_base = 34 * (k_head_offset + kv_row * params.stride_k1 + block_idx); + let q_byte_base = block_byte_base + 2u; + let d = d_shmem[(kv_idx * HEAD_DIM_QK) / 32 + block_idx]; + let q8u4 = load_k_u32_at(q_byte_base + id_in_block); + let kv = vec4( + d * f32(get_byte_i32(q8u4, 0)), + d * f32(get_byte_i32(q8u4, 1)), + d * f32(get_byte_i32(q8u4, 2)), + d * f32(get_byte_i32(q8u4, 3)), + ); +#elif defined(K_Q4_0) + let kv_row = kv_tile + kv_idx; + let block_idx = (i * 4u) / 32; + let id_in_block = (i * 4u) % 32; + let phase = id_in_block / 16; + let block_byte_base = 18 * (k_head_offset + kv_row * params.stride_k1 + block_idx); + let q_byte_base = block_byte_base + 2u; + let d = d_shmem[(kv_idx * HEAD_DIM_QK) / 32 + block_idx]; + let q8u4 = load_k_u32_at(q_byte_base + (id_in_block - phase * 16u)); + let kv = vec4( + d * (f32((get_byte(q8u4, 0) >> (phase * 4u)) & 0xFu) - 8.0), + d * (f32((get_byte(q8u4, 1) >> (phase * 4u)) & 0xFu) - 8.0), + d * (f32((get_byte(q8u4, 2) >> (phase * 4u)) & 0xFu) - 8.0), + d * (f32((get_byte(q8u4, 3) >> (phase * 4u)) & 0xFu) - 8.0), + ); +#else let idx = k_head_offset + (kv_tile + kv_idx) * params.stride_k1 + (i * 4u); let kv = vec4(K[idx >> 2u]); +#endif #else let idx = kv_idx * HEAD_DIM_QK + (i * 4u); let kv = vec4( @@ -391,7 +444,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3, kv_shmem[idx + 1u], kv_shmem[idx + 2u], kv_shmem[idx + 3u]); -#endif +#endif // defined(K_DIRECT) partial_sum += dot(qv, kv); } } @@ -473,12 +526,32 @@ fn main(@builtin(workgroup_id) wg_id: vec3, } } - // load v tile into shared memory -#ifndef KV_DIRECT - load_v_tile_block(local_id.x, kv_count, kv_tile, v_head_offset); -#endif - workgroupBarrier(); +#ifdef V_DIRECT + // load only `d` of quantized block into shared memory in the direct path +#if defined(V_Q8_0) + for (var j = local_id.x * 32; j < KV_TILE * HEAD_DIM_V; j += WG_SIZE * 32) { + let v_row = kv_tile + j / HEAD_DIM_V; + let block_idx = (j % HEAD_DIM_V) / 32; + let block_byte_base = 34 * (v_head_offset + v_row * params.stride_v1 + block_idx); + let d = f32(f16_from_u16(load_v_u16_at(block_byte_base))); + d_shmem[j / 32] = d; + } +#elif defined(V_Q4_0) + for (var j = local_id.x * 32; j < KV_TILE * HEAD_DIM_V; j += WG_SIZE * 32) { + let v_row = kv_tile + j / HEAD_DIM_V; + let block_idx = (j % HEAD_DIM_V) / 32; + let block_byte_base = 18 * (v_head_offset + v_row * params.stride_v1 + block_idx); + let d = f32(f16_from_u16(load_v_u16_at(block_byte_base))); + d_shmem[j / 32] = d; + } +#endif +#else + // load v tile into shared memory + load_v_tile_block(local_id.x, kv_count, kv_tile, v_head_offset); +#endif // V_DIRECT + + workgroupBarrier(); if (!skip_tile) { // we have P (KV_TILE) in inter_shmem and V (KV_TILE x head_dim_v) in kv_shmem @@ -501,9 +574,38 @@ fn main(@builtin(workgroup_id) wg_id: vec3, } let p = inter_shmem[kv_idx]; -#ifdef KV_DIRECT +#ifdef V_DIRECT +#if defined(V_Q8_0) + let block_idx = (vec_col * 4u) / 32; + let id_in_block = (vec_col * 4u) % 32; + let block_byte_base = 34 * (v_head_offset + v_row * params.stride_v1 + block_idx); + let q_byte_base = block_byte_base + 2u; + let d = d_shmem[(kv_idx * HEAD_DIM_V) / 32 + block_idx]; + let q8u4 = load_v_u32_at(q_byte_base + id_in_block); + let v4 = vec4( + d * f32(get_byte_i32(q8u4, 0)), + d * f32(get_byte_i32(q8u4, 1)), + d * f32(get_byte_i32(q8u4, 2)), + d * f32(get_byte_i32(q8u4, 3)), + ); +#elif defined(V_Q4_0) + let block_idx = (vec_col * 4u) / 32; + let id_in_block = (vec_col * 4u) % 32; + let phase = id_in_block / 16; + let block_byte_base = 18 * (v_head_offset + v_row * params.stride_v1 + block_idx); + let q_byte_base = block_byte_base + 2u; + let d = d_shmem[(kv_idx * HEAD_DIM_V) / 32 + block_idx]; + let q8u4 = load_v_u32_at(q_byte_base + (id_in_block - phase * 16u)); + let v4 = vec4( + d * (f32((get_byte(q8u4, 0) >> (phase * 4u)) & 0xFu) - 8.0), + d * (f32((get_byte(q8u4, 1) >> (phase * 4u)) & 0xFu) - 8.0), + d * (f32((get_byte(q8u4, 2) >> (phase * 4u)) & 0xFu) - 8.0), + d * (f32((get_byte(q8u4, 3) >> (phase * 4u)) & 0xFu) - 8.0), + ); +#else let v_idx = v_head_offset + v_row * params.stride_v1 + vec_col * 4u; let v4 = vec4(V[v_idx >> 2u]); +#endif #else let v_idx = kv_idx * HEAD_DIM_V + vec_col * 4u; let v4 = vec4( @@ -511,7 +613,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3, kv_shmem[v_idx + 1u], kv_shmem[v_idx + 2u], kv_shmem[v_idx + 3u]); -#endif +#endif // defined(V_DIRECT) lo += p * v4; } diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl index e6d7608cec..d03f1c207d 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/glu.wgsl @@ -96,7 +96,22 @@ struct Params { @group(0) @binding(0) var src0: array; -#ifdef NO_SPLIT +#ifdef SRC_OVERLAP +@group(0) @binding(1) +var dst: array; + +@group(0) @binding(2) +var params: Params; + +fn a_value(base: u32) -> DataType { + return src0[base]; +} + +fn b_value(base: u32) -> DataType { + return src0[base]; +} + +#elif defined(NO_SPLIT) @group(0) @binding(1) var dst: array; diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl index 05761dec35..66bfdd6401 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/ssm_scan.wgsl @@ -46,12 +46,29 @@ struct Params { @group(0) @binding(0) var s_in: array; #ifdef XBC_OVERLAP -@group(0) @binding(1) var x_B_C_merged: array; -@group(0) @binding(2) var dt: array; -@group(0) @binding(3) var A: array; -@group(0) @binding(4) var ids: array; -@group(0) @binding(5) var dst: array; -@group(0) @binding(6) var params: Params; +#ifdef IDS_OVERLAP +@group(0) @binding(1) var x_dt_B_C_ids_merged: array; +#ifdef A_OVERLAP +@group(0) @binding(2) var dst: array; +@group(0) @binding(3) var params: Params; +#else +@group(0) @binding(2) var A: array; +@group(0) @binding(3) var dst: array; +@group(0) @binding(4) var params: Params; +#endif +#else +@group(0) @binding(1) var x_dt_B_C_merged: array; +#ifdef A_OVERLAP +@group(0) @binding(2) var ids: array; +@group(0) @binding(3) var dst: array; +@group(0) @binding(4) var params: Params; +#else +@group(0) @binding(2) var A: array; +@group(0) @binding(3) var ids: array; +@group(0) @binding(4) var dst: array; +@group(0) @binding(5) var params: Params; +#endif +#endif #else @group(0) @binding(1) var x: array; @group(0) @binding(2) var dt: array; @@ -71,6 +88,24 @@ fn reduce_base(token_in_tile: u32) -> u32 { return token_in_tile * WG_SIZE; } +#ifdef XBC_OVERLAP +fn read_merged_f32(idx: u32) -> f32 { +#ifdef IDS_OVERLAP + return bitcast(x_dt_B_C_ids_merged[idx]); +#else + return x_dt_B_C_merged[idx]; +#endif +} +#endif + +fn read_state_slot(i3: u32) -> u32 { +#ifdef IDS_OVERLAP + return x_dt_B_C_ids_merged[params.offset_ids + i3]; +#else + return u32(ids[params.offset_ids + i3]); +#endif +} + @compute @workgroup_size(WG_SIZE) fn main( @builtin(local_invocation_id) local_id: vec3, @@ -90,13 +125,18 @@ fn main( let ir = head_seq % params.n_head; let i3 = head_seq / params.n_head; - let state_slot = u32(ids[params.offset_ids + i3]); + let state_slot = read_state_slot(i3); let g = ir / (params.n_head / params.n_group); let s_idx = params.offset_s + tid + i1 * params.stride_s1 + ir * params.stride_s2 + state_slot * params.stride_s3; var s_prev = s_in[s_idx]; - let A0 = A[params.offset_A + (tid % params.a_ne0) + ir * params.stride_A1]; + let a_idx = params.offset_A + (tid % params.a_ne0) + ir * params.stride_A1; +#ifdef A_OVERLAP + let A0 = read_merged_f32(a_idx); +#else + let A0 = A[a_idx]; +#endif for (var token_base = 0u; token_base < params.n_seq_tokens; token_base += TOKENS_PER_TILE) { if (tid < TOKENS_PER_TILE) { @@ -104,11 +144,15 @@ fn main( if (token < params.n_seq_tokens) { let x_idx = params.offset_x + i1 + ir * params.stride_x1 + token * params.stride_x2 + i3 * params.stride_x3; let dt_idx = params.offset_dt + ir + token * params.stride_dt1 + i3 * params.stride_dt2; +#ifdef XBC_OVERLAP + let dt0 = read_merged_f32(dt_idx); +#else let dt0 = dt[dt_idx]; +#endif let dtsp = select(log(1.0 + exp(dt0)), dt0, dt0 > 20.0); shared_dtsp[tid] = dtsp; #ifdef XBC_OVERLAP - shared_x_dt[tid] = x_B_C_merged[x_idx] * dtsp; + shared_x_dt[tid] = read_merged_f32(x_idx) * dtsp; #else shared_x_dt[tid] = x[x_idx] * dtsp; #endif @@ -130,7 +174,7 @@ fn main( let b_idx = params.offset_B + tid + g * params.stride_B1 + token * params.stride_B2 + i3 * params.stride_B3; let c_idx = params.offset_C + tid + g * params.stride_C1 + token * params.stride_C2 + i3 * params.stride_C3; #ifdef XBC_OVERLAP - let s = s_prev * dA + x_B_C_merged[b_idx] * x_dt; + let s = s_prev * dA + read_merged_f32(b_idx) * x_dt; #else let s = s_prev * dA + B[b_idx] * x_dt; #endif @@ -138,7 +182,7 @@ fn main( #ifdef USE_SUBGROUP_REDUCTION #ifdef XBC_OVERLAP - let subgroup_partial = subgroupAdd(s * x_B_C_merged[c_idx]); + let subgroup_partial = subgroupAdd(s * read_merged_f32(c_idx)); #else let subgroup_partial = subgroupAdd(s * C[c_idx]); #endif @@ -147,7 +191,7 @@ fn main( } #else #ifdef XBC_OVERLAP - shared_reduce[reduce_idx] = s * x_B_C_merged[c_idx]; + shared_reduce[reduce_idx] = s * read_merged_f32(c_idx); #else shared_reduce[reduce_idx] = s * C[c_idx]; #endif diff --git a/ggml/src/ggml-zendnn/CMakeLists.txt b/ggml/src/ggml-zendnn/CMakeLists.txt index e4ba9cfbd0..87d721f6d7 100644 --- a/ggml/src/ggml-zendnn/CMakeLists.txt +++ b/ggml/src/ggml-zendnn/CMakeLists.txt @@ -28,7 +28,7 @@ if (NOT ZENDNN_ROOT OR ZENDNN_ROOT STREQUAL "" OR ZENDNN_ROOT STREQUAL "OFF") ExternalProject_Add( zendnn GIT_REPOSITORY https://github.com/amd/ZenDNN.git - GIT_TAG 253b94ce0d7e9284c265fefb485714944caff9d3 # ZenDNN-2026-WW19 + GIT_TAG 1f399a75cc0993778374a51bea49b64a57879595 # ZenDNN-2026-WW28 PREFIX ${ZENDNN_PREFIX} SOURCE_DIR ${ZENDNN_SOURCE_DIR} BINARY_DIR ${ZENDNN_BUILD_DIR} diff --git a/ggml/src/ggml-zendnn/ggml-zendnn.cpp b/ggml/src/ggml-zendnn/ggml-zendnn.cpp index 3c33dcb11a..e6a9b51b79 100644 --- a/ggml/src/ggml-zendnn/ggml-zendnn.cpp +++ b/ggml/src/ggml-zendnn/ggml-zendnn.cpp @@ -30,6 +30,29 @@ zendnnl::common::data_type_t ggml_to_zendnn_type() { } } +/** + * Builds the matmul_params shared by ggml_zendnn_matmul() and ggml_zendnn_group_matmul(): + * dtype selection plus, for Q8_0 weights, dynamic-quant setup. Callers still need to set + * quant_params.src_scale.dims themselves, since that depends on the batch size(s) in use. + */ +template +static zendnnl::lowoha::matmul::matmul_params ggml_zendnn_make_matmul_params(ggml_backend_zendnn_context * ctx) { + zendnnl::lowoha::matmul::matmul_params params; + params.dtypes.src = ggml_to_zendnn_type(); + params.dtypes.wei = ggml_to_zendnn_type(); + params.dtypes.dst = ggml_to_zendnn_type(); + params.num_threads = ctx->n_threads; + + if constexpr (std::is_same_v) { + params.dtypes.compute = zendnnl::common::data_type_t::s8; + params.dynamic_quant = true; + params.quant_params.src_scale.buff = nullptr; + params.quant_params.src_scale.dt = zendnnl::common::data_type_t::bf16; + params.packing.pack_format_b = 1; + } + return params; +} + /** * ZenDNN matmul: computes C = B * A. * @@ -47,22 +70,12 @@ static bool ggml_zendnn_matmul(ggml_backend_zendnn_context * ctx, int64_t m, int const TA * A, int64_t lda, const TB * B, int64_t ldb, TC * C, int64_t ldc) { - zendnnl::lowoha::matmul::matmul_params params; - params.dtypes.src = ggml_to_zendnn_type(); - params.dtypes.wei = ggml_to_zendnn_type(); - params.dtypes.dst = ggml_to_zendnn_type(); - params.num_threads = ctx->n_threads; + zendnnl::lowoha::matmul::matmul_params params = ggml_zendnn_make_matmul_params(ctx); zendnnl::lowoha::matmul::matmul_batch_params_t batch_params; if constexpr (std::is_same_v) { - params.dtypes.compute = zendnnl::common::data_type_t::s8; - const int64_t num_groups = k / QK8_0; - params.dynamic_quant = true; - params.quant_params.src_scale.buff = nullptr; - params.quant_params.src_scale.dt = zendnnl::common::data_type_t::bf16; - params.quant_params.src_scale.dims = {n, num_groups}; - params.packing.pack_format_b = 1; + params.quant_params.src_scale.dims = {n, k / QK8_0}; } zendnnl::error_handling::status_t status = zendnnl::lowoha::matmul::matmul_direct( @@ -223,6 +236,99 @@ struct mmid_row_mapping { int32_t i2; }; +/** + * ZenDNN batched matmul: computes C[i] = B[i] * A[i] for every active expert i via a single + * group_matmul_direct() call. Batched analogue of ggml_zendnn_matmul() - see its docs for the + * per-expert A/B/C shape convention. m and k are shared by every expert; n (batch size) varies + * per expert, hence the vector. + */ +template +static bool ggml_zendnn_group_matmul(ggml_backend_zendnn_context * ctx, int64_t m, int64_t k, + const std::vector & n, + const std::vector & A, int64_t lda, + const std::vector & B, int64_t ldb, + const std::vector & C, int64_t ldc) { + + const int n_experts = n.size(); + + zendnnl::lowoha::matmul::matmul_params base_params = ggml_zendnn_make_matmul_params(ctx); + + std::vector layout(n_experts, 'r'); + std::vector trans_a(n_experts, false); + std::vector trans_b(n_experts, true); + std::vector batch_m(n_experts); + std::vector batch_n(n_experts, m); + std::vector batch_k(n_experts, k); + std::vector alpha(n_experts, 1.0f); + std::vector beta(n_experts, 0.0f); + std::vector bias(n_experts, nullptr); + std::vector lda_v(n_experts, lda); + std::vector ldb_v(n_experts, ldb); + std::vector ldc_v(n_experts, ldc); + std::vector is_wei_const(n_experts, true); + std::vector params(n_experts, base_params); + + for (int i = 0; i < n_experts; i++) { + batch_m[i] = n[i]; + + // src_scale.dims depends on this expert's row count, unlike the rest of base_params + if constexpr (std::is_same_v) { + params[i].quant_params.src_scale.dims = {n[i], k / QK8_0}; + } + } + + zendnnl::error_handling::status_t status = zendnnl::lowoha::matmul::group_matmul_direct( + layout, trans_a, trans_b, batch_m, batch_n, batch_k, alpha, + B, ldb_v, A, lda_v, bias, beta, + C, ldc_v, is_wei_const, params); + + if (status != zendnnl::error_handling::status_t::success) { + GGML_LOG_ERROR("%s, ZenDNN group matmul failed: status=%d\n", __func__, static_cast(status)); + return false; + } + return true; +} + +static bool ggml_zendnn_group_gemm(ggml_backend_zendnn_context * ctx, int64_t m, int64_t k, + const std::vector & n, + const std::vector & A, int64_t lda, + const std::vector & B, int64_t ldb, + const std::vector & C, int64_t ldc, + int Atype, int Btype, int Ctype) { + + assert(m >= 0); + for (size_t i = 0; i < n.size(); i++) { + assert(n[i] >= 0); + } + assert(k >= 0); + assert(lda >= k); + assert(ldb >= k); + assert(ldc >= m); + + // categorize types + switch (Atype) { + case GGML_TYPE_F32: + if (Btype != GGML_TYPE_F32 || Ctype != GGML_TYPE_F32) + return false; + return ggml_zendnn_group_matmul(ctx, m, k, n, A, lda, B, ldb, C, ldc); + case GGML_TYPE_BF16: + if (Btype != GGML_TYPE_BF16) + return false; + if (Ctype == GGML_TYPE_BF16) + return ggml_zendnn_group_matmul( + ctx, m, k, n, A, lda, B, ldb, C, ldc); + if (Ctype == GGML_TYPE_F32) + return ggml_zendnn_group_matmul(ctx, m, k, n, A, lda, B, ldb, C, ldc); + return false; + case GGML_TYPE_Q8_0: + if (Btype != GGML_TYPE_F32 || Ctype != GGML_TYPE_F32) + return false; + return ggml_zendnn_group_matmul(ctx, m, k, n, A, lda, B, ldb, C, ldc); + default: + return false; // unsupported type + } +} + static void ggml_zendnn_compute_forward_mul_mat_id( ggml_backend_zendnn_context * ctx, ggml_tensor * dst) { @@ -262,7 +368,8 @@ static void ggml_zendnn_compute_forward_mul_mat_id( std::vector matrix_row_counts(n_as, 0); std::vector> matrix_rows(n_as); - int64_t max_rows = 0; + int64_t total_rows = 0; + int n_active_experts = 0; // group rows by expert (preprocessing step) for (int64_t iid1 = 0; iid1 < ids->ne[1]; ++iid1) { for (int id = 0; id < n_ids; ++id) { @@ -270,66 +377,74 @@ static void ggml_zendnn_compute_forward_mul_mat_id( GGML_ASSERT(i02 >= 0 && i02 < n_as); + if (matrix_row_counts[i02] == 0) { + n_active_experts++; + } matrix_rows[i02].push_back({id, iid1}); matrix_row_counts[i02]++; - if (matrix_row_counts[i02] > max_rows) { - max_rows = matrix_row_counts[i02]; - } + total_rows++; } } - if (max_rows == 0) { + if (total_rows == 0) { return; // no rows to process } const size_t row_size = ggml_row_size(vec_dot_type, ne10); - // size for converting src1 rows to vec_dot_type if needed - const size_t nbw1 = row_size; - const size_t nbw2 = nbw1 * ne11; - const size_t nbw3 = nbw2 * ne12; - const size_t src1_conv_size = (src1->type != vec_dot_type && src0->type != GGML_TYPE_Q8_0) ? ne13 * nbw3 : 0; - // For Q8_0, src1 is always F32; the gather buffer must hold F32 rows (ne10*4 bytes), // not Q8_0-encoded rows (row_size ≈ ne10/32*34 bytes) — they differ by ~4x. const size_t f32_row_size = (size_t)ne10 * sizeof(float); const size_t gather_row_size = (src0->type == GGML_TYPE_Q8_0) ? f32_row_size : row_size; + if (src1->type != vec_dot_type && src0->type != GGML_TYPE_Q8_0) { + GGML_ASSERT(src1->type == GGML_TYPE_F32); + } + // size for MoE gather/scatter buffers - const size_t wdata_cur_size = max_rows * gather_row_size; - const size_t dst_cur_size = max_rows * ggml_row_size(dst->type, ne01); + const size_t wdata_cur_size = total_rows * gather_row_size; + const size_t dst_cur_size = total_rows * ggml_row_size(dst->type, ne01); // allocate single buffer for all needs - const size_t total_size = src1_conv_size + wdata_cur_size + dst_cur_size; + const size_t total_size = wdata_cur_size + dst_cur_size; if (ctx->work_size < total_size) { ctx->work_data.reset(new char[total_size]); ctx->work_size = total_size; } // partition the buffer - char * work_data = ctx->work_data.get(); - char * wdata_cur = work_data + src1_conv_size; + char * wdata_cur = ctx->work_data.get(); char * dst_cur = wdata_cur + wdata_cur_size; - // ZenDNN requires FP32 for dynamic quantization, so conversion is skipped - if (src1->type != vec_dot_type && src0->type != GGML_TYPE_Q8_0) { - GGML_ASSERT(src1->type == GGML_TYPE_F32); + // per-expert data collected during gather, handed to ggml_zendnn_group_gemm() as one batch + std::vector expert_row_count(n_active_experts); + std::vector batch_src(n_active_experts); + std::vector batch_wei(n_active_experts); + std::vector batch_dst(n_active_experts); - #pragma omp parallel for collapse(3) num_threads(ctx->n_threads) schedule(static) - for (int64_t i13 = 0; i13 < ne13; ++i13) { - for (int64_t i12 = 0; i12 < ne12; ++i12) { - for (int64_t i11 = 0; i11 < ne11; ++i11) { - const float * src1_f32 = (float *)((char *)src1->data + i11*nb11 + i12*nb12 + i13*nb13); - void * src1_conv = (char *)work_data + i11*nbw1 + i12*nbw2 + i13*nbw3; - from_float(src1_f32, src1_conv, ne10); - } + // precompute per-expert buffer offsets and batch indices for the parallel loop below + std::vector expert_wdata_off(n_as, 0); + std::vector expert_dst_off(n_as, 0); + std::vector expert_batch_idx(n_as, -1); + { + int64_t w_off = 0; + int64_t d_off = 0; + int batch_idx = 0; + for (int64_t cur_a = 0; cur_a < n_as; ++cur_a) { + if (matrix_row_counts[cur_a] == 0) { + continue; } + expert_wdata_off[cur_a] = w_off; + expert_dst_off[cur_a] = d_off; + expert_batch_idx[cur_a] = batch_idx; + w_off += matrix_row_counts[cur_a] * gather_row_size; + d_off += matrix_row_counts[cur_a] * ggml_row_size(dst->type, ne01); + batch_idx++; } } - const void * wdata = (src1->type == vec_dot_type || src0->type == GGML_TYPE_Q8_0) ? src1->data : work_data; - - // process each expert with gather -> gemm -> scatter pattern + // gather + inline-convert input rows into each expert's batch slot + #pragma omp parallel for num_threads(ctx->n_threads) schedule(static) for (int64_t cur_a = 0; cur_a < n_as; ++cur_a) { const int64_t cne1 = matrix_row_counts[cur_a]; @@ -337,42 +452,57 @@ static void ggml_zendnn_compute_forward_mul_mat_id( continue; } - const char * src0_cur = (const char *) src0->data + cur_a*nb02; + const int64_t w_off = expert_wdata_off[cur_a]; + const int64_t d_off = expert_dst_off[cur_a]; + const int batch_idx = expert_batch_idx[cur_a]; - // gather input rows for this expert - #pragma omp parallel for num_threads(ctx->n_threads) schedule(static) for (int64_t ir1 = 0; ir1 < cne1; ++ir1) { const mmid_row_mapping & row_mapping = matrix_rows[cur_a][ir1]; - const int64_t id = row_mapping.i1; + const int64_t id = row_mapping.i1; const int64_t i11 = id % ne11; const int64_t i12 = row_mapping.i2; - std::memcpy( - wdata_cur + ir1 * gather_row_size, - (const char *) wdata + (i11 + i12*ne11) * gather_row_size, - gather_row_size - ); + const char * src_row = (const char *) src1->data + i11*nb11 + i12*nb12; + void * dst_row = wdata_cur + w_off + ir1 * gather_row_size; + + if (src1->type != vec_dot_type && src0->type != GGML_TYPE_Q8_0) { + from_float((const float *) src_row, dst_row, ne10); + } else { + // no conversion: src1 already matches vec_dot_type, or src0 is Q8_0, whose + // ZenDNN dynamic quantization requires the row to stay in F32 + std::memcpy(dst_row, src_row, gather_row_size); + } } - // batched gemm for all tokens in this expert - if (!ggml_zendnn_gemm(ctx, - ne01, // m - cne1, // n - ne10, // k - src0_cur, - ne00, // lda - wdata_cur, - ne10, // ldb - dst_cur, - ne01, // ldc - src0->type, - src0->type == GGML_TYPE_Q8_0 ? GGML_TYPE_F32 : vec_dot_type, - dst->type)) { - GGML_ABORT("%s: ZenDNN gemm failed\n", __func__); + expert_row_count[batch_idx] = cne1; + batch_src[batch_idx] = wdata_cur + w_off; + batch_wei[batch_idx] = (const char *) src0->data + cur_a * nb02; + batch_dst[batch_idx] = dst_cur + d_off; + } + + if (!ggml_zendnn_group_gemm(ctx, + ne01, // m + ne10, // k + expert_row_count, // n (per expert) + batch_wei, ne00, // A: weights, lda + batch_src, ne10, // B: input, ldb + batch_dst, ne01, // C: output, ldc + src0->type, + src0->type == GGML_TYPE_Q8_0 ? GGML_TYPE_F32 : vec_dot_type, + dst->type)) + GGML_ABORT("%s: ZenDNN group gemm failed\n", __func__); + + // scatter output rows to destination + #pragma omp parallel for num_threads(ctx->n_threads) schedule(static) + for (int64_t cur_a = 0; cur_a < n_as; ++cur_a) { + const int64_t cne1 = matrix_row_counts[cur_a]; + + if (cne1 == 0) { + continue; } - // scatter output rows to destination - #pragma omp parallel for num_threads(ctx->n_threads) schedule(static) + const int64_t d_off = expert_dst_off[cur_a]; + for (int64_t ir1 = 0; ir1 < cne1; ++ir1) { const mmid_row_mapping & row_mapping = matrix_rows[cur_a][ir1]; const int64_t id = row_mapping.i1; @@ -381,7 +511,7 @@ static void ggml_zendnn_compute_forward_mul_mat_id( std::memcpy( (char *) dst->data + i1*nb1 + i2*nb2, - dst_cur + ir1 * ggml_row_size(dst->type, ne01), + dst_cur + d_off + ir1 * ggml_row_size(dst->type, ne01), ggml_row_size(dst->type, ne01) ); } @@ -591,22 +721,26 @@ static bool ggml_backend_zendnn_device_supports_op(ggml_backend_dev_t dev, const if(K <= 256 || N <= 128 || M <= 96) { return false; } + + // MUL_MAT_ID's gather+matmul+scatter approach favors a moderate expert count + if (op->op == GGML_OP_MUL_MAT_ID) { + const int64_t n_experts = weights->ne[2]; + const int64_t max_experts = 32; + if (n_experts > max_experts) { + return false; + } + + // fall back once the average rows per expert (N / n_experts) is too thin + // to amortize each per-expert GEMM's overhead + if (N / n_experts <= 32) { + return false; + } + } } else if (ne0 < min_batch || ne1 < min_batch || ne10 < min_batch) { return false; } - // MUL_MAT_ID performs best with a moderate number of experts due to its - // gather + batched matmul + scatter approach. Future versions will leverage - // ZenDNN's grouped_gemm for better scalability with larger expert counts: - // https://github.com/amd/ZenDNN/blob/main/docs/operator/lowoha_group_gemm_operator.md - if (op->op == GGML_OP_MUL_MAT_ID) { - const int64_t n_experts = weights->ne[2]; - const int64_t max_experts = 32; - if (n_experts > max_experts) { - return false; - } - } switch (weights->type) { case GGML_TYPE_F32: case GGML_TYPE_BF16: diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index a7d1fe7d94..59191c663e 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -7854,7 +7854,9 @@ void ggml_set_input(struct ggml_tensor * tensor) { } void ggml_set_output(struct ggml_tensor * tensor) { - tensor->flags |= GGML_TENSOR_FLAG_OUTPUT; + for (struct ggml_tensor * cur = tensor; cur != NULL; cur = cur->view_src) { + cur->flags |= GGML_TENSOR_FLAG_OUTPUT; + } } void ggml_set_param(struct ggml_tensor * tensor) { diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index ee5c4c85f0..7a2ba560a8 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -145,6 +145,8 @@ class Keys: TOKEN_SHIFT_COUNT = "{arch}.token_shift_count" INTERLEAVE_MOE_LAYER_STEP = "{arch}.interleave_moe_layer_step" FULL_ATTENTION_INTERVAL = "{arch}.full_attention_interval" + NUM_LOOPS = "{arch}.num_loops" + SKIP_LOOP_FINAL_NORM = "{arch}.skip_loop_final_norm" HASH_LAYER_COUNT = "{arch}.hash_layer_count" ACTIVATION_SPARSITY_SCALE = "{arch}.activation_sparsity_scale" ALTUP_ACTIVE_IDX = "{arch}.altup.active_idx" @@ -159,6 +161,7 @@ class Keys: TARGET_HIDDEN_SIZE = "{arch}.target_hidden_size" BLOCK_SIZE = "{arch}.block_size" NORM_BEFORE_RESIDUAL = "{arch}.norm_before_residual" + NORM_BEFORE_FC = "{arch}.norm_before_fc" class Attention: HEAD_COUNT = "{arch}.attention.head_count" @@ -351,6 +354,7 @@ class Keys: class Attention: HEAD_COUNT = "clip.vision.attention.head_count" HEAD_COUNT_KV = "clip.vision.attention.head_count_kv" # used by mimovl (GQA) + HEAD_DIM = "clip.vision.attention.head_dim" # set when qkv width != n_embd LAYERNORM_EPS = "clip.vision.attention.layer_norm_epsilon" class Projector: @@ -371,10 +375,17 @@ class Keys: FEED_FORWARD_LENGTH = "clip.audio.feed_forward_length" PROJECTION_DIM = "clip.audio.projection_dim" BLOCK_COUNT = "clip.audio.block_count" + SUBSAMPLING_FACTOR = "clip.audio.subsampling_factor" CHUNK_SIZE = "clip.audio.chunk_size" CONV_KERNEL_SIZE = "clip.audio.conv_kernel_size" MAX_POS_EMB = "clip.audio.max_pos_emb" FEATURE_LAYERS = "clip.audio.feature_layer" # Granite Speech Plus + RVQ_NUM_QUANTIZERS = "clip.audio.rvq.num_quantizers" + RVQ_CODEBOOK_SIZE = "clip.audio.rvq.codebook_size" + WA_PATTERN_MODE = "clip.audio.wa_pattern_mode" # per-layer -1 (full) / 0 (windowed) + WINDOW_SIZE = "clip.audio.window_size" + LOCAL_BLOCK_COUNT = "clip.audio.local_block_count" # mimo-v2.5: input_local_transformer layer count + LOCAL_GROUP_SIZE = "clip.audio.local_group_size" # mimo-v2.5: input_local_transformer grouping size class Attention: HEAD_COUNT = "clip.audio.attention.head_count" @@ -547,6 +558,7 @@ class MODEL_ARCH(IntEnum): KIMI_LINEAR = auto() TALKIE = auto() MELLUM = auto() + NANBEIGE = auto() class VISION_PROJECTOR_TYPE(IntEnum): @@ -859,6 +871,8 @@ class MODEL_TENSOR(IntEnum): V_MM_UP = auto() # cogvlm V_MM_DOWN = auto() # cogvlm V_MM_GATE = auto() # cogvlm + V_MM_MERGER_FC1 = auto() # minimax-m3 (patch-merge MLP) + V_MM_MERGER_FC2 = auto() # minimax-m3 (patch-merge MLP) V_TOK_BOI = auto() # cogvlm V_TOK_EOI = auto() # cogvlm V_TOK_IMG_BEGIN = auto() # hunyuanvl @@ -942,6 +956,9 @@ class MODEL_TENSOR(IntEnum): A_ENC_FFN_SCALE_1 = auto() # gemma3n A_ENC_FFN_GATE_1 = auto() # lfm2, gemma3n A_ENC_FFN_DOWN_1 = auto() # lfm2, gemma3n + A_ENC_DOWNSAMPLE_CONV = auto() # mimo-audio-tokenizer: post-transformer downsample conv + A_ENC_DOWNSAMPLE_NORM = auto() # mimo-audio-tokenizer: post-transformer downsample norm + A_ENC_RVQ_CODEBOOK = auto() # mimo-audio-tokenizer: residual vector quantizer codebook, per quantizer index A_MMPROJ = auto() A_MMPROJ_FC = auto() A_MM_NORM_PRE = auto() @@ -950,6 +967,17 @@ class MODEL_TENSOR(IntEnum): A_MM_HARD_EMB_NORM = auto() # gemma3n A_MM_SOFT_EMB_NORM = auto() # gemma3n A_MM_INP_PROJ = auto() # gemma3n + A_MM_CODE_EMBD = auto() # mimo: text-side RVQ code embedding table ("text codebook"), merged 3D [n_channels, vocab, dim] + A_MM_LOCAL_ATTN_Q = auto() # mimo: input_local_transformer (LLM-side connector) + A_MM_LOCAL_ATTN_K = auto() + A_MM_LOCAL_ATTN_V = auto() + A_MM_LOCAL_ATTN_OUT = auto() + A_MM_LOCAL_FFN_GATE = auto() + A_MM_LOCAL_FFN_UP = auto() + A_MM_LOCAL_FFN_DOWN = auto() + A_MM_LOCAL_LN1 = auto() + A_MM_LOCAL_LN2 = auto() + A_MM_LOCAL_NORM = auto() # final norm after all input_local_transformer layers A_PER_DIM_K_SCALE = auto() # gemma4 A_PER_DIM_SCALE = auto() # gemma4 # nextn/mtp @@ -964,6 +992,10 @@ class MODEL_TENSOR(IntEnum): # eagle3 FC = auto() # feature fusion layer D2T = auto() # draft to target vocabulary mapping + # dspark + DSPARK_MARKOV_W1 = auto() # markov head: prev-token embed + DSPARK_MARKOV_W2 = auto() # markov head: bias projection + DSPARK_CONF_PROJ = auto() # confidence head # lfm2 audio A_ENC_NORM_CONV = auto() A_ENC_LINEAR_POS = auto() @@ -974,6 +1006,10 @@ class MODEL_TENSOR(IntEnum): A_ENC_CONV_NORM = auto() # SSM conv A_ENC_CONV_PW1 = auto() A_ENC_CONV_PW2 = auto() + A_ENC_CONV_NORM_MEAN = auto() # parakeet + A_ENC_CONV_NORM_VAR = auto() # parakeet + A_ENC_MEL_FILTERS = auto() # parakeet + A_ENC_WINDOW = auto() # parakeet A_CTC_OUT = auto() A_CTC_OUT_MID = auto() A_ENC_ATTN_REL_POS_EMB = auto() @@ -1135,6 +1171,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = { MODEL_ARCH.KIMI_LINEAR: "kimi-linear", MODEL_ARCH.TALKIE: "talkie", MODEL_ARCH.MELLUM: "mellum", + MODEL_ARCH.NANBEIGE: "nanbeige", } VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = { @@ -1444,6 +1481,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.V_MM_UP: "mm.up", MODEL_TENSOR.V_MM_DOWN: "mm.down", MODEL_TENSOR.V_MM_GATE: "mm.gate", + MODEL_TENSOR.V_MM_MERGER_FC1: "mm.merger.fc1", + MODEL_TENSOR.V_MM_MERGER_FC2: "mm.merger.fc2", MODEL_TENSOR.V_TOK_BOI: "v.boi", MODEL_TENSOR.V_TOK_EOI: "v.eoi", MODEL_TENSOR.V_MM_PRE_NORM: "mm.pre_norm", @@ -1527,6 +1566,9 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.A_ENC_FFN_UP_1: "a.blk.{bid}.ffn_up_1", MODEL_TENSOR.A_ENC_FFN_GATE_1: "a.blk.{bid}.ffn_gate_1", MODEL_TENSOR.A_ENC_FFN_DOWN_1: "a.blk.{bid}.ffn_down_1", + MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: "a.downsample.conv", + MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: "a.downsample.norm", + MODEL_TENSOR.A_ENC_RVQ_CODEBOOK: "a.rvq.codebook", MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}", MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc", MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre", @@ -1535,6 +1577,17 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.A_MM_SOFT_EMB_NORM: "mm.a.soft_emb_norm", # gemma3n MODEL_TENSOR.A_MM_EMBEDDING: "mm.a.embedding", # gemma3n MODEL_TENSOR.A_MM_HARD_EMB_NORM: "mm.a.hard_emb_norm", # gemma3n + MODEL_TENSOR.A_MM_CODE_EMBD: "mm.a.code_embd", + MODEL_TENSOR.A_MM_LOCAL_ATTN_Q: "mm.a.local_blk.{bid}.attn_q", + MODEL_TENSOR.A_MM_LOCAL_ATTN_K: "mm.a.local_blk.{bid}.attn_k", + MODEL_TENSOR.A_MM_LOCAL_ATTN_V: "mm.a.local_blk.{bid}.attn_v", + MODEL_TENSOR.A_MM_LOCAL_ATTN_OUT: "mm.a.local_blk.{bid}.attn_out", + MODEL_TENSOR.A_MM_LOCAL_FFN_GATE: "mm.a.local_blk.{bid}.ffn_gate", + MODEL_TENSOR.A_MM_LOCAL_FFN_UP: "mm.a.local_blk.{bid}.ffn_up", + MODEL_TENSOR.A_MM_LOCAL_FFN_DOWN: "mm.a.local_blk.{bid}.ffn_down", + MODEL_TENSOR.A_MM_LOCAL_LN1: "mm.a.local_blk.{bid}.ln1", + MODEL_TENSOR.A_MM_LOCAL_LN2: "mm.a.local_blk.{bid}.ln2", + MODEL_TENSOR.A_MM_LOCAL_NORM: "mm.a.local_norm", MODEL_TENSOR.A_PER_DIM_K_SCALE: "a.blk.{bid}.per_dim_k_scale", # gemma4 MODEL_TENSOR.A_PER_DIM_SCALE: "a.blk.{bid}.per_dim_scale", # gemma4 # lfm2 audio @@ -1547,6 +1600,10 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.A_ENC_CONV_NORM: "a.blk.{bid}.conv_norm", MODEL_TENSOR.A_ENC_CONV_PW1: "a.blk.{bid}.conv_pw1", MODEL_TENSOR.A_ENC_CONV_PW2: "a.blk.{bid}.conv_pw2", + MODEL_TENSOR.A_ENC_CONV_NORM_MEAN: "a.blk.{bid}.conv_norm_mean", + MODEL_TENSOR.A_ENC_CONV_NORM_VAR: "a.blk.{bid}.conv_norm_var", + MODEL_TENSOR.A_ENC_MEL_FILTERS: "a.mel_filters", + MODEL_TENSOR.A_ENC_WINDOW: "a.window", MODEL_TENSOR.A_CTC_OUT: "a.enc_ctc_out", MODEL_TENSOR.A_CTC_OUT_MID: "a.enc_ctc_out_mid", MODEL_TENSOR.A_ENC_ATTN_REL_POS_EMB: "a.blk.{bid}.attn_rel_pos_emb", @@ -1577,6 +1634,9 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD: "blk.{bid}.nextn.shared_head_head", MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM: "blk.{bid}.nextn.shared_head_norm", MODEL_TENSOR.FC: "fc", + MODEL_TENSOR.DSPARK_MARKOV_W1: "markov_w1", + MODEL_TENSOR.DSPARK_MARKOV_W2: "markov_w2", + MODEL_TENSOR.DSPARK_CONF_PROJ: "conf_proj", MODEL_TENSOR.D2T: "d2t", } @@ -1640,6 +1700,8 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.V_RESMPL_QUERY, MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK, MODEL_TENSOR.V_MM_PATCH_MERGER, + MODEL_TENSOR.V_MM_MERGER_FC1, + MODEL_TENSOR.V_MM_MERGER_FC2, MODEL_TENSOR.V_DS_NORM, MODEL_TENSOR.V_DS_FC1, MODEL_TENSOR.V_DS_FC2, @@ -1734,10 +1796,24 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.A_ENC_FFN_UP_1, MODEL_TENSOR.A_ENC_FFN_GATE_1, MODEL_TENSOR.A_ENC_FFN_DOWN_1, + MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV, + MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM, + MODEL_TENSOR.A_ENC_RVQ_CODEBOOK, MODEL_TENSOR.A_MMPROJ, MODEL_TENSOR.A_MMPROJ_FC, MODEL_TENSOR.A_MM_NORM_PRE, MODEL_TENSOR.A_MM_NORM_MID, + MODEL_TENSOR.A_MM_CODE_EMBD, + MODEL_TENSOR.A_MM_LOCAL_ATTN_Q, + MODEL_TENSOR.A_MM_LOCAL_ATTN_K, + MODEL_TENSOR.A_MM_LOCAL_ATTN_V, + MODEL_TENSOR.A_MM_LOCAL_ATTN_OUT, + MODEL_TENSOR.A_MM_LOCAL_FFN_GATE, + MODEL_TENSOR.A_MM_LOCAL_FFN_UP, + MODEL_TENSOR.A_MM_LOCAL_FFN_DOWN, + MODEL_TENSOR.A_MM_LOCAL_LN1, + MODEL_TENSOR.A_MM_LOCAL_LN2, + MODEL_TENSOR.A_MM_LOCAL_NORM, MODEL_TENSOR.A_ENC_NORM_CONV, MODEL_TENSOR.A_ENC_LINEAR_POS, MODEL_TENSOR.A_ENC_POS_BIAS_U, @@ -1747,6 +1823,10 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.A_ENC_CONV_NORM, MODEL_TENSOR.A_ENC_CONV_PW1, MODEL_TENSOR.A_ENC_CONV_PW2, + MODEL_TENSOR.A_ENC_CONV_NORM_MEAN, + MODEL_TENSOR.A_ENC_CONV_NORM_VAR, + MODEL_TENSOR.A_ENC_MEL_FILTERS, + MODEL_TENSOR.A_ENC_WINDOW, MODEL_TENSOR.A_MM_INP_PROJ, MODEL_TENSOR.A_MM_SOFT_EMB_NORM, MODEL_TENSOR.A_MM_EMBEDDING, @@ -4307,6 +4387,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FC, + MODEL_TENSOR.ENC_OUTPUT_NORM, MODEL_TENSOR.D2T, ], MODEL_ARCH.DFLASH: [ @@ -4324,6 +4405,10 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FC, MODEL_TENSOR.ENC_OUTPUT_NORM, + # optional DSpark heads + MODEL_TENSOR.DSPARK_MARKOV_W1, + MODEL_TENSOR.DSPARK_MARKOV_W2, + MODEL_TENSOR.DSPARK_CONF_PROJ, ], MODEL_ARCH.MISTRAL4: [ MODEL_TENSOR.TOKEN_EMBD, @@ -4378,8 +4463,11 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_EXP_PROBS_B, MODEL_TENSOR.LAYER_OUT_NORM, MODEL_TENSOR.NEXTN_EH_PROJ, + MODEL_TENSOR.NEXTN_EMBED_TOKENS, MODEL_TENSOR.NEXTN_ENORM, MODEL_TENSOR.NEXTN_HNORM, + MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, + MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ], MODEL_ARCH.STEP35: [ MODEL_TENSOR.TOKEN_EMBD, @@ -4521,7 +4609,22 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.FFN_DOWN_EXP, MODEL_TENSOR.FFN_UP_EXP, ], - # TODO + MODEL_ARCH.NANBEIGE: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.ATTN_Q, + MODEL_TENSOR.ATTN_K, + MODEL_TENSOR.ATTN_V, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.ATTN_ROT_EMBD, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.FFN_GATE, + MODEL_TENSOR.FFN_DOWN, + MODEL_TENSOR.FFN_UP, + ], } # tensors that will not be serialized @@ -4588,6 +4691,10 @@ MODEL_TENSOR_SKIP: dict[MODEL_ARCH, list[MODEL_TENSOR]] = { MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_ROT_EMBD, ], + MODEL_ARCH.NANBEIGE: [ + MODEL_TENSOR.ROPE_FREQS, + MODEL_TENSOR.ATTN_ROT_EMBD, + ], } # @@ -4793,9 +4900,12 @@ class VisionProjectorType: YOUTUVL = "youtuvl" NEMOTRON_V2_VL = "nemotron_v2_vl" HUNYUANVL = "hunyuanvl" + PARAKEET = "parakeet" # audio + MINIMAXM3 = "minimax_m3" MINICPMV4_6 = "minicpmv4_6" GRANITE_SPEECH = "granite_speech" # audio MIMOVL = "mimovl" + MIMO_AUDIO = "mimo_audio" GRANITE4_VISION = "granite4_vision" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index a2f2bf384a..af14e24fe4 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -908,6 +908,12 @@ class GGUFWriter: def add_token_shift_count(self, count: int) -> None: self.add_uint32(Keys.LLM.TOKEN_SHIFT_COUNT.format(arch=self.arch), count) + def add_num_loops(self, count: int) -> None: + self.add_uint32(Keys.LLM.NUM_LOOPS.format(arch=self.arch), count) + + def add_skip_loop_final_norm(self, value: bool) -> None: + self.add_bool(Keys.LLM.SKIP_LOOP_FINAL_NORM.format(arch=self.arch), value) + def add_interleave_moe_layer_step(self, value: int) -> None: self.add_uint32(Keys.LLM.INTERLEAVE_MOE_LAYER_STEP.format(arch=self.arch), value) @@ -968,6 +974,9 @@ class GGUFWriter: def add_norm_before_residual(self, value: bool) -> None: self.add_bool(Keys.LLM.NORM_BEFORE_RESIDUAL.format(arch=self.arch), value) + def add_norm_before_fc(self, value: bool) -> None: + self.add_bool(Keys.LLM.NORM_BEFORE_FC.format(arch=self.arch), value) + def add_attention_output_group_count(self, count: int) -> None: self.add_uint32(Keys.Attention.OUTPUT_GROUP_COUNT.format(arch=self.arch), count) @@ -1220,6 +1229,9 @@ class GGUFWriter: def add_vision_head_count_kv(self, value: int) -> None: self.add_uint32(Keys.ClipVision.Attention.HEAD_COUNT_KV, value) + def add_vision_head_dim(self, value: int) -> None: + self.add_uint32(Keys.ClipVision.Attention.HEAD_DIM, value) + def add_vision_attention_layernorm_eps(self, value: float) -> None: self.add_float32(Keys.ClipVision.Attention.LAYERNORM_EPS, value) @@ -1347,9 +1359,30 @@ class GGUFWriter: def add_audio_num_mel_bins(self, value: int) -> None: self.add_uint32(Keys.ClipAudio.NUM_MEL_BINS, value) + def add_audio_rvq_num_quantizers(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.RVQ_NUM_QUANTIZERS, value) + + def add_audio_rvq_codebook_size(self, values: Sequence[int]) -> None: + self.add_array(Keys.ClipAudio.RVQ_CODEBOOK_SIZE, values) + + def add_audio_wa_pattern_mode(self, modes: Sequence[int]) -> None: + self.add_array(Keys.ClipAudio.WA_PATTERN_MODE, modes) + + def add_audio_window_size(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.WINDOW_SIZE, value) + + def add_audio_local_block_count(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.LOCAL_BLOCK_COUNT, value) + + def add_audio_local_group_size(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.LOCAL_GROUP_SIZE, value) + def add_audio_stack_factor(self, value: int) -> None: self.add_uint32(Keys.ClipAudio.Projector.STACK_FACTOR, value) + def add_audio_subsampling_factor(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.SUBSAMPLING_FACTOR, value) + def add_audio_chunk_size(self, value: int) -> None: self.add_uint32(Keys.ClipAudio.CHUNK_SIZE, value) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index ff27a5eb04..79539c46d1 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -1304,6 +1304,18 @@ class TensorNameMap: "model.fc", # dflash ), + MODEL_TENSOR.DSPARK_MARKOV_W1: ( + "model.markov_head.markov_w1", # dspark + ), + + MODEL_TENSOR.DSPARK_MARKOV_W2: ( + "model.markov_head.markov_w2", # dspark + ), + + MODEL_TENSOR.DSPARK_CONF_PROJ: ( + "model.confidence_head.proj", # dspark + ), + MODEL_TENSOR.CLS: ( "classifier", # jina "classifier.dense", # roberta @@ -1838,6 +1850,14 @@ class TensorNameMap: "visual.downsample", # glm4v ), + MODEL_TENSOR.V_MM_MERGER_FC1: ( + "patch_merge_mlp.linear_1", # minimax-m3 + ), + + MODEL_TENSOR.V_MM_MERGER_FC2: ( + "patch_merge_mlp.linear_2", # minimax-m3 + ), + MODEL_TENSOR.V_DS_NORM: ( "model.visual.deepstack_merger_list.{bid}.norm", # deepstack in qwen3vl ), @@ -2087,6 +2107,8 @@ class TensorNameMap: "conformer.pre_encode.conv.{bid}", # lfm2 "model.audio_tower.subsample_conv_projection.conv_{bid}.conv", # gemma3n "conformer.subsample_conv_projection.layer{bid}.conv", # gemma4 + "sound_encoder.encoder.subsampling.layers.{bid}", # parakeet + "encoder.conv{bid}", # mimo-audio-tokenizer ), MODEL_TENSOR.A_ENC_CONV1D_NORM: ( @@ -2111,6 +2133,7 @@ class TensorNameMap: MODEL_TENSOR.A_POST_NORM: ( "audio_tower.layer_norm", # ultravox "audio_tower.ln_post", # qwen2omni + "encoder.layer_norm", # mimo-audio-tokenizer ), MODEL_TENSOR.A_ENC_ATTN_Q: ( @@ -2118,7 +2141,9 @@ class TensorNameMap: "conformer.layers.{bid}.self_attn.linear_q", # lfm2 "conformer.layers.{bid}.attention.attn.q_proj", # gemma3n "conformer.layers.{bid}.self_attn.q_proj", # gemma4 + "sound_encoder.encoder.layers.{bid}.self_attn.q_proj", # parakeet "encoder.layers.{bid}.attn.to_q", # granite_speech + "encoder.layers.{bid}.self_attn.q_proj", # mimo-audio-tokenizer ), MODEL_TENSOR.A_ENC_ATTN_K: ( @@ -2126,7 +2151,9 @@ class TensorNameMap: "conformer.layers.{bid}.self_attn.linear_k", # lfm2 "conformer.layers.{bid}.attention.attn.k_proj", # gemma3n "conformer.layers.{bid}.self_attn.k_proj", # gemma4 + "sound_encoder.encoder.layers.{bid}.self_attn.k_proj", # parakeet "encoder.layers.{bid}.attn.to_k", # granite_speech (split from to_kv) + "encoder.layers.{bid}.self_attn.k_proj", # mimo-audio-tokenizer ), MODEL_TENSOR.A_ENC_ATTN_V: ( @@ -2134,7 +2161,9 @@ class TensorNameMap: "conformer.layers.{bid}.self_attn.linear_v", # lfm2 "conformer.layers.{bid}.attention.attn.v_proj", # gemma3n "conformer.layers.{bid}.self_attn.v_proj", # gemma4 + "sound_encoder.encoder.layers.{bid}.self_attn.v_proj", # parakeet "encoder.layers.{bid}.attn.to_v", # granite_speech (split from to_kv) + "encoder.layers.{bid}.self_attn.v_proj", # mimo-audio-tokenizer ), MODEL_TENSOR.A_ENC_ATTN_K_REL: ( @@ -2162,7 +2191,9 @@ class TensorNameMap: "audio_tower.layers.{bid}.self_attn_layer_norm", # ultravox "conformer.layers.{bid}.norm_self_att", # lfm2 "conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n + "sound_encoder.encoder.layers.{bid}.norm_self_att", # parakeet "encoder.layers.{bid}.attn.pre_norm", # granite_speech + "encoder.layers.{bid}.self_attn_layer_norm", # mimo-audio-tokenizer ), MODEL_TENSOR.A_ENC_OUTPUT: ( @@ -2170,20 +2201,25 @@ class TensorNameMap: "conformer.layers.{bid}.self_attn.linear_out", # lfm2 "conformer.layers.{bid}.attention.post", # gemma3n "conformer.layers.{bid}.self_attn.post", # gemma4 + "sound_encoder.encoder.layers.{bid}.self_attn.o_proj", # parakeet "encoder.layers.{bid}.attn.to_out", # granite_speech + "encoder.layers.{bid}.self_attn.out_proj", # mimo-audio-tokenizer ), MODEL_TENSOR.A_ENC_OUTPUT_NORM: ( "audio_tower.layers.{bid}.final_layer_norm", # ultravox "conformer.layers.{bid}.norm_out", # lfm2 "conformer.layers.{bid}.attention.post_norm", # gemma3n + "sound_encoder.encoder.layers.{bid}.norm_out", # parakeet "encoder.layers.{bid}.post_norm", # granite_speech + "encoder.layers.{bid}.final_layer_norm", # mimo-audio-tokenizer ), MODEL_TENSOR.A_ENC_FFN_NORM: ( "conformer.layers.{bid}.norm_feed_forward1", # lfm2 "conformer.layers.{bid}.ffw_layer_start.pre_layer_norm", # gemma3n "conformer.layers.{bid}.feed_forward1.pre_layer_norm", # gemma4 + "sound_encoder.encoder.layers.{bid}.norm_feed_forward1", # parakeet "encoder.layers.{bid}.ff1.pre_norm", # granite_speech ), @@ -2201,7 +2237,9 @@ class TensorNameMap: "conformer.layers.{bid}.feed_forward1.linear1", # lfm2 "conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n "conformer.layers.{bid}.feed_forward1.ffw_layer_1", # gemma4 + "sound_encoder.encoder.layers.{bid}.feed_forward1.linear1", # parakeet "encoder.layers.{bid}.ff1.up_proj", # granite_speech + "encoder.layers.{bid}.fc1", # mimo-audio-tokenizer ), MODEL_TENSOR.A_ENC_FFN_GATE: (), @@ -2211,13 +2249,16 @@ class TensorNameMap: "conformer.layers.{bid}.feed_forward1.linear2", # lfm2 "conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n "conformer.layers.{bid}.feed_forward1.ffw_layer_2", # gemma4 + "sound_encoder.encoder.layers.{bid}.feed_forward1.linear2", # parakeet "encoder.layers.{bid}.ff1.down_proj", # granite_speech + "encoder.layers.{bid}.fc2", # mimo-audio-tokenizer ), MODEL_TENSOR.A_ENC_FFN_UP_1: ( "conformer.layers.{bid}.feed_forward2.linear1", # lfm2 "conformer.layers.{bid}.ffw_layer_end.ffw_layer_1", # gemma3n "conformer.layers.{bid}.feed_forward2.ffw_layer_1", # gemma4 + "sound_encoder.encoder.layers.{bid}.feed_forward2.linear1", # parakeet "encoder.layers.{bid}.ff2.up_proj", # granite_speech ), @@ -2225,6 +2266,7 @@ class TensorNameMap: "conformer.layers.{bid}.feed_forward2.linear2", # lfm2 "conformer.layers.{bid}.ffw_layer_end.ffw_layer_2", # gemma3n "conformer.layers.{bid}.feed_forward2.ffw_layer_2", # gemma4 + "sound_encoder.encoder.layers.{bid}.feed_forward2.linear2", # parakeet "encoder.layers.{bid}.ff2.down_proj", # granite_speech ), @@ -2232,9 +2274,23 @@ class TensorNameMap: "conformer.layers.{bid}.norm_feed_forward2", # lfm2 "conformer.layers.{bid}.ffw_layer_end.pre_layer_norm", # gemma3n "conformer.layers.{bid}.feed_forward2.pre_layer_norm", # gemma4 + "sound_encoder.encoder.layers.{bid}.norm_feed_forward2", # parakeet "encoder.layers.{bid}.ff2.pre_norm", # granite_speech ), + MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: ( + "encoder.down_sample_layer.0", # mimo-audio-tokenizer + ), + + MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: ( + "encoder.down_sample_norm", # mimo-audio-tokenizer + ), + + # note: the raw per-quantizer "encoder.quantizer.vq.layers.{i}._codebook.embed" + # tensors are merged (padded + stacked, like MoE experts) into this single 3D + # tensor in conversion code, so no raw-name mapping is registered here. + MODEL_TENSOR.A_ENC_RVQ_CODEBOOK: (), + MODEL_TENSOR.A_ENC_FFN_POST_NORM_1: ( "conformer.layers.{bid}.ffw_layer_end.post_layer_norm", # gemma3n "conformer.layers.{bid}.feed_forward2.post_layer_norm", # gemma4 @@ -2247,20 +2303,24 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_LINEAR_POS: ( "conformer.layers.{bid}.self_attn.linear_pos", # lfm2 "conformer.layers.{bid}.attention.attn.relative_position_embedding.pos_proj", # gemma3n + "sound_encoder.encoder.layers.{bid}.self_attn.relative_k_proj", # parakeet ), MODEL_TENSOR.A_ENC_POS_BIAS_U: ( "conformer.layers.{bid}.self_attn.pos_bias_u", # lfm2 + "sound_encoder.encoder.layers.{bid}.self_attn.bias_u", # parakeet ), MODEL_TENSOR.A_ENC_POS_BIAS_V: ( "conformer.layers.{bid}.self_attn.pos_bias_v", # lfm2 + "sound_encoder.encoder.layers.{bid}.self_attn.bias_v", # parakeet ), MODEL_TENSOR.A_ENC_OUT: ( "conformer.pre_encode.out", # lfm2 "model.audio_tower.subsample_conv_projection.input_proj_linear", # gemma3n (note: it should be A_ENC_INP_PROJ, this is a mistake; it should be corrected in C++ code when it's supported) "conformer.output_proj", # gemma4 + "sound_encoder.encoder.subsampling.linear", # parakeet ), # note: some tensors below has "audio." pseudo-prefix, to prevent conflicts with vision tensors @@ -2270,6 +2330,7 @@ class TensorNameMap: "audio.multi_modal_projector.linear_{bid}", # ultravox, meralion "audio_adapter.model.{bid}", # lfm2 "audio_tower.proj{bid}", # qwen3omni + "sound_projection.linear{bid}", # parakeet (linear1, linear2) ), MODEL_TENSOR.A_MMPROJ_FC: ( @@ -2280,39 +2341,89 @@ class TensorNameMap: MODEL_TENSOR.A_MM_NORM_PRE: ( "audio.multi_modal_projector.ln_pre", # ultravox + "sound_projection.norm", # parakeet ), MODEL_TENSOR.A_MM_NORM_MID: ( "audio.multi_modal_projector.ln_mid", # ultravox ), + # note: the raw per-channel "speech_embeddings.{i}" tensors are merged + # (stacked, like MoE experts) into this single 3D tensor in conversion + # code, so no raw-name mapping is registered here. + MODEL_TENSOR.A_MM_CODE_EMBD: (), + + MODEL_TENSOR.A_MM_LOCAL_ATTN_Q: ( + "audio_encoder.input_local_transformer.layers.{bid}.self_attn.q_proj", # mimo-v2.5 + ), + MODEL_TENSOR.A_MM_LOCAL_ATTN_K: ( + "audio_encoder.input_local_transformer.layers.{bid}.self_attn.k_proj", # mimo-v2.5 + ), + MODEL_TENSOR.A_MM_LOCAL_ATTN_V: ( + "audio_encoder.input_local_transformer.layers.{bid}.self_attn.v_proj", # mimo-v2.5 + ), + MODEL_TENSOR.A_MM_LOCAL_ATTN_OUT: ( + "audio_encoder.input_local_transformer.layers.{bid}.self_attn.o_proj", # mimo-v2.5 + ), + MODEL_TENSOR.A_MM_LOCAL_FFN_GATE: ( + "audio_encoder.input_local_transformer.layers.{bid}.mlp.gate_proj", # mimo-v2.5 + ), + MODEL_TENSOR.A_MM_LOCAL_FFN_UP: ( + "audio_encoder.input_local_transformer.layers.{bid}.mlp.up_proj", # mimo-v2.5 + ), + MODEL_TENSOR.A_MM_LOCAL_FFN_DOWN: ( + "audio_encoder.input_local_transformer.layers.{bid}.mlp.down_proj", # mimo-v2.5 + ), + MODEL_TENSOR.A_MM_LOCAL_LN1: ( + "audio_encoder.input_local_transformer.layers.{bid}.input_layernorm", # mimo-v2.5 + ), + MODEL_TENSOR.A_MM_LOCAL_LN2: ( + "audio_encoder.input_local_transformer.layers.{bid}.post_attention_layernorm", # mimo-v2.5 + ), + MODEL_TENSOR.A_MM_LOCAL_NORM: ( + "audio_encoder.input_local_transformer.norm", # mimo-v2.5 + ), + MODEL_TENSOR.A_ENC_CONV_DW: ( "conformer.layers.{bid}.conv.depthwise_conv", # lfm2 "conformer.layers.{bid}.lconv1d.depthwise_conv1d", # gemma3n + "sound_encoder.encoder.layers.{bid}.conv.depthwise_conv", # parakeet "encoder.layers.{bid}.conv.depth_conv.conv", # granite_speech ), MODEL_TENSOR.A_ENC_CONV_NORM: ( "conformer.layers.{bid}.conv.batch_norm", # lfm2 "conformer.layers.{bid}.lconv1d.pre_layer_norm", # gemma3n + "sound_encoder.encoder.layers.{bid}.conv.norm", # parakeet + ), + + MODEL_TENSOR.A_ENC_CONV_NORM_MEAN: ( + "sound_encoder.encoder.layers.{bid}.conv.norm.running_mean", # parakeet + ), + + MODEL_TENSOR.A_ENC_CONV_NORM_VAR: ( + "sound_encoder.encoder.layers.{bid}.conv.norm.running_var", # parakeet "encoder.layers.{bid}.conv.batch_norm", # granite_speech ), MODEL_TENSOR.A_ENC_CONV_PW1: ( "conformer.layers.{bid}.conv.pointwise_conv1", # lfm2 "conformer.layers.{bid}.lconv1d.linear_start", # gemma3n + "sound_encoder.encoder.layers.{bid}.conv.pointwise_conv1", # parakeet "encoder.layers.{bid}.conv.up_conv", # granite_speech ), MODEL_TENSOR.A_ENC_CONV_PW2: ( "conformer.layers.{bid}.conv.pointwise_conv2", # lfm2 "conformer.layers.{bid}.lconv1d.linear_end", # gemma3n + "sound_encoder.encoder.layers.{bid}.conv.pointwise_conv2", # parakeet "encoder.layers.{bid}.conv.down_conv", # granite_speech ), MODEL_TENSOR.A_ENC_NORM_CONV: ( "conformer.layers.{bid}.norm_conv", # lfm2 "conformer.layers.{bid}.lconv1d.conv_norm", # gemma3n + "sound_encoder.encoder.layers.{bid}.norm_conv", # parakeet "encoder.layers.{bid}.conv.norm", # granite_speech ), @@ -2324,6 +2435,14 @@ class TensorNameMap: "conformer.layers.{bid}.attention.attn.per_dim_scale", # gemma4 ), + MODEL_TENSOR.A_ENC_MEL_FILTERS: ( + "sound_encoder.encoder.feature_extractor.featurizer.fb", # parakeet + ), + + MODEL_TENSOR.A_ENC_WINDOW: ( + "sound_encoder.encoder.feature_extractor.featurizer.window", # parakeet + ), + MODEL_TENSOR.A_MM_EMBEDDING: ( "model.embed_audio.embedding", # gemma3n ), diff --git a/include/llama.h b/include/llama.h index 9fab693170..6e53e22972 100644 --- a/include/llama.h +++ b/include/llama.h @@ -203,10 +203,11 @@ extern "C" { }; enum llama_load_mode { - LLAMA_LOAD_MODE_NONE = 0, // no special loading mode - LLAMA_LOAD_MODE_MMAP = 1, // memory map the model - LLAMA_LOAD_MODE_MLOCK = 2, // mmap + force system to keep model in RAM rather than swapping or compressing - LLAMA_LOAD_MODE_DIRECT_IO = 3, // use direct I/O if available + LLAMA_LOAD_MODE_NONE = 0, // no special loading mode + LLAMA_LOAD_MODE_MMAP = 1, // memory map the model + LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing + LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available }; LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode); @@ -336,6 +337,7 @@ extern "C" { bool use_extra_bufts; // use extra buffer types (used for weight repacking) bool no_host; // bypass host buffer allowing extra buffers to be used bool no_alloc; // only load metadata and simulate memory allocations + bool load_mtp; // whether to load MTP layers }; struct llama_sampler_seq_config { @@ -1101,6 +1103,9 @@ extern "C" { LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab); LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab); + // model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens) + LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens); + LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab); LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab); diff --git a/models/templates/MiniMax-M3.jinja b/models/templates/MiniMax-M3.jinja new file mode 100644 index 0000000000..93022eb9ce --- /dev/null +++ b/models/templates/MiniMax-M3.jinja @@ -0,0 +1,247 @@ +{# ---------- special token variables ---------- #} +{%- set ns_token = ']<]minimax[>[' -%} +{%- set bod_token = ']~!b[' -%} +{%- set bos_token = ']~b]' -%} +{%- set eos_token = '[e~[' -%} +{%- set toolcall_begin_token = ns_token ~ '' -%} +{%- set toolcall_end_token = ns_token ~ '' -%} +{%- set think_begin_token = '' -%} +{%- set think_end_token = '' -%} +{%- set image_token = ']<]image[>[' -%} +{%- set video_token = ']<]video[>[' -%} +{#- Thinking mode: "enabled" / "disabled" / "adaptive" / not defined -#} +{#- Recursive XML renderer for tool_call arguments ======================== -#} +{#- None values are intentionally skipped in mapping iteration so that + `null` (which would round-trip to the literal string "null") + never appears in the rendered tool_call. The convention is: omit the + field entirely. The top-level `_args` loop applies the same rule. + The `val is none` branch below is a safety net only — upstream cleaning + (drop_none_in_tool_arguments) should ensure no None ever reaches here. -#} +{%- macro to_xml(val, ns) -%} +{%- if val is mapping -%} +{%- for k, v in val.items() if v is not none -%} +{{ ns }}<{{ k }}>{{ to_xml(v, ns) }}{{ ns }} +{%- endfor -%} +{%- elif val is iterable and val is not string -%} +{%- for item in val -%} +{{ ns }}{{ to_xml(item, ns) }}{{ ns }} +{%- endfor -%} +{%- elif val is none -%} +{#- Should be unreachable when upstream cleaning is applied. -#} +{%- elif val is boolean -%} +{{ val | tojson }} +{%- else -%} +{{ val }} +{%- endif -%} +{%- endmacro -%} +{#- Tool Rendering Functions ============================================== -#} +{%- macro render_tool_namespace(namespace_name, tool_list) -%} +{%- for tool in tool_list -%} +{{ tool.function | tojson(ensure_ascii=False) }} +{% endfor -%} +{%- endmacro -%} +{%- macro visible_text(content) -%} + {%- if content is string -%} + {{ content }} + {%- elif content is iterable and content is not mapping -%} + {%- for item in content -%} + {%- if item is mapping and item.type == 'text' -%} + {{- item.text }} + {%- elif item is mapping and item.type == 'image' -%} + {{- image_token }} + {%- elif item is mapping and item.type == 'video' -%} + {{- video_token}} + {%- elif item is string -%} + {{- item }} + {%- endif -%} + {%- endfor -%} + {%- elif content is none -%} + {{- '' }} + {%- else -%} + {{- content }} + {%- endif -%} +{%- endmacro -%} +{#- System Message Construction ============================================ -#} +{%- macro build_system_message(system_message) -%} + {%- if system_message and system_message.content -%} + {{- visible_text(system_message.content) }} + {%- else -%} + {{- 'Your model version is MiniMax-M3, developed by MiniMax. Knowledge cutoff: January 2026. Founded in early 2022, MiniMax is a global AI foundation model company committed to advancing the frontiers of AI towards AGI.' }} + {%- endif -%} + + {#- Thinking mode instructions -#} + {{- '\n\n\n' }} + {{- 'You have a thinking capability that allows you to reason step by step before responding. When thinking is enabled, wrap your reasoning in ' ~ think_begin_token ~ think_end_token ~ ' tags before your response. When thinking is disabled, begin your response directly after the ' ~ think_end_token ~ ' prefix. When thinking is adaptive, decide on your own whether to think for the current turn.\n' }} + {%- if thinking_mode is defined -%} + {%- if thinking_mode == "enabled" -%} + {{- 'Current thinking mode: enabled. You MUST think step by step before every response, including after receiving function/tool results.\n' }} + {%- elif thinking_mode == "disabled" -%} + {{- 'Current thinking mode: disabled. Do not output any thinking process.\n' }} + {%- elif thinking_mode == "adaptive" -%} + {{- 'Current thinking mode: adaptive. You are encouraged to think for complex decision-making, multi-step reasoning, or when analyzing function/tool results.\n' }} + {%- endif -%} + {%- else -%} + {{- 'Current thinking mode: adaptive. You are encouraged to think for complex decision-making, multi-step reasoning, or when analyzing function/tool results.\n' }} + {%- endif -%} + {{- '' }} +{%- endmacro -%} +{%- macro build_developer_message(developer_message) -%} + {%- if developer_message and developer_message.content -%} + {{- visible_text(developer_message.content) }} + {%- else -%} + {%- if model_identity is not defined -%} + {%- set model_identity = "You are a helpful assistant." -%} + {%- endif -%} + {{- model_identity }} + {%- endif -%} +{%- endmacro -%} +{#- Main Template Logic ================================================= -#} +{#- Role mapping: root -> system sp (high priority), system/developer -> developer sp (low priority) -#} +{%- set system_message = none -%} +{%- set developer_message = none -%} +{%- set conversation_messages = messages -%} +{%- if messages and messages[0].role == "root" -%} + {%- set system_message = messages[0] -%} + {%- set conversation_messages = messages[1:] -%} + {%- if conversation_messages and conversation_messages[0].role in ["system", "developer"] -%} + {%- set developer_message = conversation_messages[0] -%} + {%- set conversation_messages = conversation_messages[1:] -%} + {%- endif -%} +{%- elif messages and messages[0].role in ["system", "developer"] -%} + {%- set developer_message = messages[0] -%} + {%- set conversation_messages = messages[1:] -%} +{%- endif -%} +{#- Render system sp (higher priority, root role only) -#} +{{- bod_token ~ bos_token ~ 'system' ~ '\n' }} +{{- build_system_message(system_message) }} +{{- eos_token ~ '\n' }} + +{#- Render developer sp (lower priority: system/developer role + tools) -#} +{{- bos_token ~ 'developer' ~ '\n' }} +{{- build_developer_message(developer_message) }} +{%- if tools -%} + {{- '\n\n' ~ '# Tools' ~ '\n' ~ 'You may call one or more tools to assist with the user query.\nHere are the tools available in JSONSchema format:' ~ '\n' }} + {{- '\n' ~ '' ~ '\n' }} + {{- render_tool_namespace("functions", tools) }} + {{- '' ~ '\n\n' }} + {{- 'To call tools, wrap all invocations in a single ' ~ toolcall_begin_token ~ toolcall_end_token ~ ' block. Parameter values containing nested objects or arrays are recursively expanded into XML elements. Example:\n' }} + {{- '\n' ~ toolcall_begin_token ~ '\n' }} + {{- ns_token + '' }} + {{- ns_token + 'value-1' + ns_token + '' }} + {{- ns_token + '' }} + {{- ns_token + '' }} + {{- ns_token + 'val-a' + ns_token + '' }} + {{- ns_token + 'val-b' + ns_token + '' }} + {{- ns_token + '' }} + {{- ns_token + '' }} + {{- ns_token + '\n' }} + {{- ns_token + '' }} + {{- ns_token + 'value-1' + ns_token + '' }} + {{- ns_token + '\n' }} + {{- toolcall_end_token }} +{%- endif -%} +{{- eos_token ~ '\n' }} + +{#- Render messages -#} +{%- set last_tool_call = namespace(name=none) -%} +{%- for message in conversation_messages -%} + {%- if message.role == 'assistant' -%} + {{- bos_token ~ 'ai' ~ '\n' }} + + {%- set reasoning_content = '' %} + {%- set content = visible_text(message.content) %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if think_end_token in content %} + {%- set reasoning_content = content.split(think_end_token)[0].strip('\n').split(think_begin_token)[-1].strip('\n') %} + {%- set content = content.split(think_end_token)[-1].strip('\n') %} + {%- endif %} + {%- endif %} + + {%- if reasoning_content -%} + {#- Render thinking for every assistant turn (all-turn visible) -#} + {{- think_begin_token ~ reasoning_content ~ think_end_token }} + {%- else -%} + {#- No thinking rendered → prefix with think_end_token -#} + {{- think_end_token }} + {%- endif -%} + + {%- if content -%} + {{- content }} + {%- endif -%} + {%- if message.tool_calls -%} + {{- toolcall_begin_token ~ '\n' }} + + {%- for tool_call in message.tool_calls -%} + {%- if tool_call.function -%} + {%- set tool_call = tool_call.function -%} + {%- endif -%} +{{- ns_token + '' }} +{%- set _args = tool_call.arguments -%} +{%- for k, v in _args.items() if v is not none %} +{{- ns_token + '<' + k + '>' -}} +{{- to_xml(v, ns_token) -}} +{{- ns_token + '' }} +{%- endfor -%} +{{- ns_token + '' ~ '\n' }} + {%- endfor -%} + + {{- toolcall_end_token }} + {%- if message.tool_calls[-1].function -%} + {%- set last_tool_call.name = message.tool_calls[-1].function.name -%} + {%- else -%} + {%- set last_tool_call.name = message.tool_calls[-1].name -%} + {%- endif -%} + {%- else -%} + {%- set last_tool_call.name = none -%} + {%- endif -%} + {{- eos_token ~ '\n' }} + + {%- elif message.role == 'tool' -%} + {%- if last_tool_call.name is none -%} + {{- raise_exception("Message has tool role, but there was no previous assistant message with a tool call!") }} + {%- endif -%} + {%- if loop.first or (conversation_messages[loop.index0 - 1].role != 'tool') -%} + {{- bos_token ~ 'tool' }} + {%- endif -%} + {{- '\n' }} + {%- if message.content is string -%} + {{- message.content }} + {%- else -%} + {%- for tr in message.content -%} + {%- if tr is mapping and tr.type is defined and tr.type == 'image' -%} + {{- image_token }} + {%- elif tr is mapping and tr.type is defined and tr.type == 'video' -%} + {{- video_token }} + {%- else -%} + {{- tr.output if tr.output is defined else (tr.text if tr.type == 'text' and tr.text is defined else tr) }} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {{- '' }} + {%- if loop.last or (conversation_messages[loop.index0 + 1].role != 'tool') -%} + {{- eos_token ~ '\n' -}} + {%- endif -%} + + {%- elif message.role == 'user' -%} + {{- bos_token ~ 'user' ~ '\n' }} + {{- visible_text(message.content) }} + {{- eos_token ~ '\n' }} + {%- endif -%} +{%- endfor -%} + +{#- Generation prompt -#} +{%- if add_generation_prompt -%} +{{- bos_token ~ 'ai' ~ '\n' }} +{%- if thinking_mode is defined and thinking_mode == "disabled" -%} + {{- think_end_token }} +{%- elif thinking_mode is defined and thinking_mode == "adaptive" -%} + {#- adaptive: no prefix, let model decide -#} +{%- elif thinking_mode is defined and thinking_mode == "enabled" -%} + {#- enabled or not defined: default to think -#} + {{- think_begin_token }} +{%- else -%} + {#- adaptive: no prefix, let model decide -#} +{%- endif -%} +{%- endif -%} diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 24e27e6f02..230e59a95b 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -9be313313c8ecb9488911bd64550190e3ed80f38 +06ca97616793248fadb410ea8d69c7511b2005e4 diff --git a/skills/add-new-model/SKILL.md b/skills/add-new-model/SKILL.md index 68be866c7b..f76d1abfd7 100644 --- a/skills/add-new-model/SKILL.md +++ b/skills/add-new-model/SKILL.md @@ -76,6 +76,7 @@ These recur often enough in review comments on past add-model PRs that they're w - Don't ship unfinished or unverified speculative-decoding (e.g. MTP) scaffolding in the base model PR - if it hasn't actually been confirmed to work, pull it out and land it as its own follow-up. - Conversion code should call into the base class's existing hparam logic (e.g. `super().set_gguf_parameters()`) rather than re-deriving it - large blocks of code that duplicate what `TextModel`/`MmprojModel` already provide will get flagged as redundant. - Do constant tensor modifications (e.g. `norm(1 + weight)`) and permutations/chunking at conversion time, not in the graph - see HOWTO-add-model.md's "Prefer conversion-time tensor modifications" tip (Gemma 3 folds its `1 +` into the weights, Qwen3-Next permutes in `modify_tensors`). Doing these at runtime in the graph is very likely to be rejected as over-complicated; if you genuinely can't do it at conversion time, open a discussion first explaining why rather than implementing it in the graph. + - Exception: a plain `weight * scale` with a constant scale is usually better applied at inference time instead of being folded into the weight at conversion. The scale conceptually applies to the activation, not the weight, so folding it in can hurt numerical stability, and it shifts the weight's value range in a way that can make quantization worse. ## Validation checklist diff --git a/skills/code-review/SKILL.md b/skills/code-review/SKILL.md index 84075fea32..ba76c48115 100644 --- a/skills/code-review/SKILL.md +++ b/skills/code-review/SKILL.md @@ -110,6 +110,15 @@ Public API changes carry a higher bar than internal ones (`CONTRIBUTING.md`). Re - Security: don't trust client-supplied headers (e.g. `X-Forwarded-For`) or add footguns; things like IP allowlisting belong at a reverse proxy unless there's a trusted-proxy design. - Wire new behavior into the existing request/response and checkpoint paths correctly; watch for resource leaks across requests. +## Multimodal (`tools/mtmd/`) + +- Tensor names must be prefixed by `v.`, `a.`, `mm.` or `a.mm.` (legacy naming doesn't follow this convention - this is expected, but new code should follow it). +- Do not use explicit sin/cos for RoPE; use `ggml_rope_ext` instead, see `HOWTO-add-model.md`. If it can't express the needed behavior, that's a design discussion, not a PR. +- New GGML ops must not be introduced in the same PR, you must push it as a separate PR. +- In most cases, `build_vit` should be enough to build the transformer graph for vision models. Do not add a loop to build the transformer graph manually, unless you have a very good reason to do so. If you do, please explain why in the PR description. +- If you need a dedicated preprocessor, there is a high chance that it can be a derived class from one of the existing preprocessors. Check carefully before adding a new preprocessor class. +- If the model need a new public API in `mtmd.h`, open a discussion first. + ## General (always) Enforce the `AGENTS.md` / `CONTRIBUTING.md` coding and naming guidelines on every changed line - this is a distinct pass from checking that the code works, and matters just as much for review speed: diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 2c88567b67..661409a7ca 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -144,6 +144,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_KIMI_LINEAR, "kimi-linear" }, { LLM_ARCH_TALKIE, "talkie" }, { LLM_ARCH_MELLUM, "mellum" }, + { LLM_ARCH_NANBEIGE, "nanbeige" }, { LLM_ARCH_UNKNOWN, "(unknown)" }, }; @@ -222,6 +223,8 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_TOKEN_SHIFT_COUNT, "%s.token_shift_count" }, { LLM_KV_INTERLEAVE_MOE_LAYER_STEP, "%s.interleave_moe_layer_step" }, { LLM_KV_FULL_ATTENTION_INTERVAL, "%s.full_attention_interval" }, + { LLM_KV_NUM_LOOPS, "%s.num_loops" }, + { LLM_KV_SKIP_LOOP_FINAL_NORM, "%s.skip_loop_final_norm" }, { LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" }, { LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" }, @@ -316,6 +319,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_TARGET_LAYERS, "%s.target_layers" }, { LLM_KV_TARGET_HIDDEN_SIZE, "%s.target_hidden_size" }, { LLM_KV_NORM_BEFORE_RESIDUAL, "%s.norm_before_residual" }, + { LLM_KV_NORM_BEFORE_FC, "%s.norm_before_fc" }, { LLM_KV_SHORTCONV_L_CACHE, "%s.shortconv.l_cache" }, // sentence-transformers dense modules feature dims @@ -614,6 +618,9 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_MASKED_EMBD_ORDERING, "masked_embd_ordering" }, { LLM_TENSOR_FC, "fc" }, { LLM_TENSOR_D2T, "d2t" }, + { LLM_TENSOR_DSPARK_MARKOV_W1, "markov_w1" }, + { LLM_TENSOR_DSPARK_MARKOV_W2, "markov_w2" }, + { LLM_TENSOR_DSPARK_CONF_PROJ, "conf_proj" }, }; // declare information about the model weight tensors: @@ -868,6 +875,10 @@ static const std::map LLM_TENSOR_INFOS = { // eagle3 {LLM_TENSOR_FC, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, {LLM_TENSOR_D2T, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}}, + // dspark + {LLM_TENSOR_DSPARK_MARKOV_W1, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_GET_ROWS}}, + {LLM_TENSOR_DSPARK_MARKOV_W2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DSPARK_CONF_PROJ, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, }; LLM_KV::LLM_KV(llm_arch arch, const char * suffix) : arch(arch), suffix(suffix) {} diff --git a/src/llama-arch.h b/src/llama-arch.h index ceb334cc6c..7e97292b25 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -149,6 +149,7 @@ enum llm_arch { LLM_ARCH_EAGLE3, LLM_ARCH_MINIMAX_M3, LLM_ARCH_DFLASH, + LLM_ARCH_NANBEIGE, LLM_ARCH_UNKNOWN, }; @@ -227,6 +228,8 @@ enum llm_kv { LLM_KV_TOKEN_SHIFT_COUNT, LLM_KV_INTERLEAVE_MOE_LAYER_STEP, LLM_KV_FULL_ATTENTION_INTERVAL, + LLM_KV_NUM_LOOPS, + LLM_KV_SKIP_LOOP_FINAL_NORM, LLM_KV_ATTENTION_HEAD_COUNT, LLM_KV_ATTENTION_HEAD_COUNT_KV, @@ -362,6 +365,7 @@ enum llm_kv { LLM_KV_TARGET_LAYERS, LLM_KV_TARGET_HIDDEN_SIZE, LLM_KV_NORM_BEFORE_RESIDUAL, + LLM_KV_NORM_BEFORE_FC, LLM_KV_SHORTCONV_L_CACHE, @@ -622,6 +626,9 @@ enum llm_tensor { LLM_TENSOR_MASKED_EMBD_ORDERING, LLM_TENSOR_FC, LLM_TENSOR_D2T, + LLM_TENSOR_DSPARK_MARKOV_W1, + LLM_TENSOR_DSPARK_MARKOV_W2, + LLM_TENSOR_DSPARK_CONF_PROJ, }; diff --git a/src/llama-context.cpp b/src/llama-context.cpp index c512477c0e..5ef7becf6f 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -474,6 +474,9 @@ llama_context::llama_context( } llama_context::~llama_context() { + // wait for any pending asynchronous copies into the output buffers before they are freed + synchronize(); + if (!model.hparams.no_alloc) { for (size_t i = 0; i < backend_ptrs.size(); ++i) { ggml_backend_t backend = backend_ptrs[i]; @@ -1417,13 +1420,17 @@ int llama_context::encode(const llama_batch & batch_inp) { // micro-batching is not possible for non-causal encoding, so we process the batch in a single shot GGML_ASSERT(cparams.n_ubatch >= n_tokens && "encoder requires n_ubatch >= n_tokens"); + // TODO: this clear of the buffer can easily be forgotten - need something better + // sync first so any in-flight async copies into embd_seq complete before it is freed + if (!embd_seq.empty()) { + synchronize(); + } + embd_seq.clear(); + if (t_compute_start_us == 0) { t_compute_start_us = ggml_time_us(); } - // TODO: this clear of the buffer can easily be forgotten - need something better - embd_seq.clear(); - sched_reserve(); n_queued_tokens += n_tokens; @@ -1762,13 +1769,18 @@ int llama_context::decode(const llama_batch & batch_inp) { GGML_ASSERT((cparams.causal_attn || cparams.n_ubatch >= n_tokens_all) && "non-causal attention requires n_ubatch >= n_tokens"); + // TODO: this clear of the buffer can easily be forgotten - need something better + // sync first so any in-flight async copies into embd_seq complete before it is freed + if (!embd_seq.empty()) { + synchronize(); + } + embd_seq.clear(); + if (t_compute_start_us == 0) { t_compute_start_us = ggml_time_us(); } n_queued_tokens += n_tokens_all; - // TODO: this clear of the buffer can easily be forgotten - need something better - embd_seq.clear(); output_swaps.clear(); sched_reserve(); @@ -2339,6 +2351,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE || model.arch == LLM_ARCH_DEEPSEEK4 || + model.arch == LLM_ARCH_NANBEIGE || model.arch == LLM_ARCH_MINIMAX_M3) { return std::max(n_tokens * 40, 32u * model.n_tensors()); } @@ -2473,11 +2486,12 @@ llm_graph_cb llama_context::graph_get_cb() const { ggml_set_name(cur, name); } - // norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends + // - norm may be automatically assigned to the backend of the previous layer, increasing data transfer between backends + // - force the last op of the layer on the specified backend to avoid running it on the backend of the next layer due to scheduling // FIXME: fix in ggml_backend_sched const bool full_offload = model.n_gpu_layers() > model.hparams.n_layer_all; if (ubatch.n_tokens < 32 || full_offload) { - if (il != -1 && strcmp(name, "norm") == 0) { + if (il != -1 && (strcmp(name, "norm") == 0 || strcmp(name, "l_last") == 0)) { const auto & dev_layer = model.dev_layer(il); for (const auto & backend : backends) { if (ggml_backend_get_device(backend.get()) == dev_layer) { @@ -3539,6 +3553,22 @@ llama_context * llama_init_from_model( } } + if ((model->hparams.is_mla() || model->arch == LLM_ARCH_DEEPSEEK4) && params.type_k != params.type_v) { + LLAMA_LOG_ERROR("%s: model does not support different K (%s) and V (%s) cache types\n", __func__, ggml_type_name(params.type_k), ggml_type_name(params.type_v)); + return nullptr; + } + + if (ggml_is_quantized(params.type_v) && params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_ENABLED) { + if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO) { + LLAMA_LOG_INFO("%s: enabling flash_attn since it is required for quantized V cache\n", __func__); + params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED; + } + if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_DISABLED) { + LLAMA_LOG_ERROR("%s: quantized V cache requires flash_attn to be enabled\n", __func__); + return nullptr; + } + } + if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_k)) { const uint32_t blck_size = ggml_blck_size(params.type_k); for (uint32_t il = 0; il < model->hparams.n_layer(); ++il) { @@ -3561,11 +3591,6 @@ llama_context * llama_init_from_model( } } - if (ggml_is_quantized(params.type_v) && params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_DISABLED) { - LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__); - return nullptr; - } - if (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != model->hparams.pooling_type) { //user-specified pooling-type is different from the model default diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 07b1847f58..ce430d4364 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -47,6 +47,7 @@ struct llama_hparams { bool use_par_res; bool swin_norm; bool norm_before_residual = false; + bool norm_before_fc = false; uint32_t n_ctx_train; // context size the model was trained on uint32_t n_embd; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 43447f57d3..df8313e819 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -526,6 +526,7 @@ llama_model_loader::llama_model_loader( llama_load_mode load_mode, bool check_tensors, bool no_alloc, + bool load_mtp, const llama_model_kv_override * param_overrides_p, const llama_model_tensor_buft_override * param_tensor_buft_overrides_p) : metadata(meta), set_tensor_data(set_tensor_data), set_tensor_data_ud(set_tensor_data_ud) { @@ -542,7 +543,7 @@ llama_model_loader::llama_model_loader( tensor_buft_overrides = param_tensor_buft_overrides_p; - this->use_mmap = load_mode == LLAMA_LOAD_MODE_MMAP || load_mode == LLAMA_LOAD_MODE_MLOCK; + this->use_mmap = load_mode == LLAMA_LOAD_MODE_MMAP || load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK; this->use_direct_io = load_mode == LLAMA_LOAD_MODE_DIRECT_IO; if (!fname.empty()) { @@ -812,6 +813,7 @@ llama_model_loader::llama_model_loader( this->check_tensors = check_tensors; this->no_alloc = no_alloc; + this->load_mtp = load_mtp; } std::string llama_model_loader::get_arch_name() const { diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h index 75a3652d06..7ad3807822 100644 --- a/src/llama-model-loader.h +++ b/src/llama-model-loader.h @@ -79,6 +79,7 @@ struct llama_model_loader { bool use_direct_io = false; bool check_tensors; bool no_alloc; + bool load_mtp; llama_files files; llama_ftype ftype; @@ -129,6 +130,7 @@ struct llama_model_loader { llama_load_mode load_mode, bool check_tensors, bool no_alloc, + bool load_mtp, const llama_model_kv_override * param_overrides_p, const llama_model_tensor_buft_override * param_tensor_buft_overrides_p); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 8ca752e3b6..2180d36b2a 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -85,6 +85,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_stablelm(params); case LLM_ARCH_MELLUM: return new llama_model_mellum(params); + case LLM_ARCH_NANBEIGE: + return new llama_model_nanbeige(params); case LLM_ARCH_QWEN: return new llama_model_qwen(params); case LLM_ARCH_QWEN2: @@ -818,6 +820,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_100B_A6B: return "100B.A6B"; case LLM_TYPE_102B_A12B: return "102B.A12B"; case LLM_TYPE_106B_A12B: return "106B.A12B"; + case LLM_TYPE_118B_A8B: return "118B.A8B"; case LLM_TYPE_120B_A12B: return "120B.A12B"; case LLM_TYPE_122B_A10B: return "122B.A10B"; case LLM_TYPE_196B_A11B: return "196B.A11B"; @@ -1251,7 +1254,7 @@ void llama_model_base::load_vocab(llama_model_loader & ml) { bool llama_model_base::load_tensors(llama_model_loader & ml) { const auto & split_mode = params.split_mode; - const bool use_mlock = params.load_mode == LLAMA_LOAD_MODE_MLOCK; + const bool use_mlock = params.load_mode == LLAMA_LOAD_MODE_MLOCK || params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK; const auto & tensor_split = params.tensor_split; const int n_layer_all = hparams.n_layer_all; @@ -2072,7 +2075,6 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, res = nullptr; } break; case LLM_ARCH_DEEPSEEK32: - case LLM_ARCH_GLM_DSA: { res = new llama_kv_cache_dsa( *this, @@ -2089,6 +2091,56 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, nullptr, nullptr); } break; + case LLM_ARCH_GLM_DSA: + { + if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP && hparams.n_layer_nextn > 0) { + // The NextN/MTP draft head runs dense MLA (no DSA indexer), so the + // MTP context uses a plain attention KV cache holding only the + // nextn layer(s) - same pattern as the hybrid Qwen3.5 MTP context. + llama_kv_cache::layer_filter_cb filter = + [&](uint32_t il) { return il >= hparams.n_layer(); }; + + res = new llama_kv_cache( + *this, + hparams, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + 1, + hparams.n_swa, + hparams.swa_type, + nullptr, + filter, + nullptr, + nullptr); + } else { + // Main context: DSA cache for the trunk layers only - the nextn + // layer(s) are never attended by the trunk graph. + llama_kv_cache::layer_filter_cb filter = nullptr; + if (hparams.n_layer_nextn > 0) { + filter = [&](uint32_t il) { return il < hparams.n_layer(); }; + } + + res = new llama_kv_cache_dsa( + *this, + params.type_k, + params.type_v, + !cparams.flash_attn, + cparams.offload_kqv, + cparams.kv_unified, + cparams.n_ctx_seq, + cparams.n_seq_max, + 1, + hparams.n_swa, + hparams.swa_type, + filter, + nullptr); + } + } break; // Models that need standard caching should rely on recurrent/hybrid // checks default: @@ -2194,7 +2246,9 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params, filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; } - if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3) && hparams.n_layer_nextn > 0) { + if ((arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_HY_V3 || arch == LLM_ARCH_GLM_DSA || + arch == LLM_ARCH_MIMO2) && + hparams.n_layer_nextn > 0) { if (params.ctx_type == LLAMA_CONTEXT_TYPE_MTP) { filter = [&](uint32_t il) { return il >= hparams.n_layer(); }; } else { @@ -2339,6 +2393,7 @@ llama_model_params llama_model_default_params() { /*.use_extra_bufts =*/ true, /*.no_host =*/ false, /*.no_alloc =*/ false, + /*.load_mtp =*/ false, }; return result; @@ -2495,6 +2550,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_LLAMA_EMBED: case LLM_ARCH_MAINCODER: case LLM_ARCH_GLM_DSA: + case LLM_ARCH_NANBEIGE: return LLAMA_ROPE_TYPE_NORM; // the pairs of head values are offset by n_rot/2 diff --git a/src/llama-model.h b/src/llama-model.h index 36d0480e5e..056a6efa59 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -130,6 +130,7 @@ enum llm_type { LLM_TYPE_100B_A6B, LLM_TYPE_102B_A12B, // Solar-Open LLM_TYPE_106B_A12B, // GLM-4.5-Air + LLM_TYPE_118B_A8B, // Laguna-S-2 LLM_TYPE_120B_A12B, // Nemotron 3 Super LLM_TYPE_122B_A10B, // Qwen3.5 LLM_TYPE_196B_A11B, // Step3.5-Flash @@ -606,6 +607,12 @@ struct llama_model { struct ggml_tensor * fc = nullptr; // feature fusion layer struct ggml_tensor * d2t = nullptr; // draft to target vocabulary mapping + // dspark + struct ggml_tensor * dspark_markov_w1 = nullptr; + struct ggml_tensor * dspark_markov_w2 = nullptr; + struct ggml_tensor * dspark_conf_proj = nullptr; + struct ggml_tensor * dspark_conf_proj_b = nullptr; + // unified vector to store target-model extracted layer ids in eagle3, dflash, etc. std::vector target_layer_ids; diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index caf7733a5b..fd6e787bd7 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -326,6 +326,10 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param quantize &= name.find("ssm_conv1d") == std::string::npos; quantize &= name.find("shortconv.conv.weight") == std::string::npos; + // do not quantize MiniMax's indexer projection weights, they are tiny + quantize &= name.find("indexer.k_proj.weight") == std::string::npos; + quantize &= name.find("indexer.q_proj.weight") == std::string::npos; + // do not quantize RWKV's small yet 2D weights quantize &= name.find("time_mix_first.weight") == std::string::npos; quantize &= name.find("time_mix_w0.weight") == std::string::npos; @@ -355,6 +359,10 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param quantize &= name.find(".patch_embd") == std::string::npos; quantize &= name.find(".patch_merger") == std::string::npos; + // audio codebook + quantize &= name.find("a.rvq.codebook") == std::string::npos; + quantize &= name.find("mm.a.code_embd") == std::string::npos; + return quantize; } @@ -885,7 +893,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: const llama_model_kv_override * kv_overrides = params->kv_overrides; std::vector splits = {}; llama_model_loader ml(/*metadata*/ nullptr, /*set_tensor_data*/ nullptr, /*set_tensor_data_ud*/ nullptr, - fname_inp, splits, /*file*/ nullptr, /*load_mode*/ load_mode, /*check_tensors*/ true, /*no_alloc*/ false, kv_overrides, nullptr); + fname_inp, splits, /*file*/ nullptr, /*load_mode*/ load_mode, /*check_tensors*/ true, /*no_alloc*/ false, /*load_mtp*/ true, kv_overrides, nullptr); ml.init_mappings(false); // no prefetching auto mparams = llama_model_default_params(); diff --git a/src/llama-sampler.cpp b/src/llama-sampler.cpp index 6520e4181e..a9cb6bee5f 100644 --- a/src/llama-sampler.cpp +++ b/src/llama-sampler.cpp @@ -993,7 +993,9 @@ static void llama_sampler_greedy_backend_apply( GGML_UNUSED(gf); GGML_UNUSED(smpl); - struct ggml_tensor * curl = ggml_argmax(ctx, data->logits); + struct ggml_tensor * logits = ggml_reshape_1d(ctx, data->logits, ggml_nelements(data->logits)); + + struct ggml_tensor * curl = ggml_argmax(ctx, logits); ggml_set_name(curl, "greedy_argmax"); data->sampled = curl; @@ -1158,7 +1160,10 @@ static void llama_sampler_dist_backend_apply( ggml_set_name (sctx->inp_uniform, "uniform"); ggml_set_input(sctx->inp_uniform); - struct ggml_tensor * probs = ggml_soft_max(ctx, data->logits); + // flatten + struct ggml_tensor * logits = ggml_reshape_1d(ctx, data->logits, ggml_nelements(data->logits)); + + struct ggml_tensor * probs = ggml_soft_max(ctx, logits); ggml_set_name(probs, "dist_probs"); struct ggml_tensor * cumsum = ggml_cumsum(ctx, probs); @@ -1289,22 +1294,22 @@ static void llama_sampler_top_k_backend_apply( struct llama_sampler_data * data) { auto * sctx = (llama_sampler_top_k *) smpl->ctx; - struct ggml_tensor * top_k = ggml_top_k(ctx, data->logits, sctx->k); + struct ggml_tensor * logits = ggml_reshape_1d(ctx, data->logits, ggml_nelements(data->logits)); + + struct ggml_tensor * top_k = ggml_top_k(ctx, logits, sctx->k); ggml_set_name(top_k, "top_k"); if (data->candidates) { struct ggml_tensor * candidates_rows = ggml_reshape_2d(ctx, data->candidates, 1, data->candidates->ne[0]); data->candidates = ggml_get_rows(ctx, candidates_rows, top_k); - data->candidates = ggml_reshape_1d(ctx, data->candidates, sctx->k); ggml_set_name(data->candidates, "top_k_candidates"); } else { data->candidates = top_k; } - struct ggml_tensor * logits_rows = ggml_reshape_2d(ctx, data->logits, 1, data->logits->ne[0]); - struct ggml_tensor * top_k_rows = ggml_get_rows(ctx, logits_rows, top_k); - data->logits = ggml_reshape_1d(ctx, top_k_rows, sctx->k); - ggml_set_name(top_k_rows, "top_k_rows"); + struct ggml_tensor * logits_rows = ggml_reshape_2d(ctx, logits, 1, logits->ne[0]); + data->logits = ggml_get_rows(ctx, logits_rows, top_k); + ggml_set_name(data->logits, "top_k_rows"); GGML_UNUSED(gf); } @@ -1435,21 +1440,25 @@ static void llama_sampler_top_p_backend_apply( struct llama_sampler_data * data) { auto * sctx = (llama_sampler_top_p *) smpl->ctx; + // flatten + struct ggml_tensor * logits = ggml_reshape_1d(ctx, data->logits, ggml_nelements(data->logits)); + auto ggml_sort = [ctx](struct ggml_tensor * a, struct ggml_tensor * b) { GGML_ASSERT(ggml_nrows(a) == 1); struct ggml_tensor * a_reshaped = ggml_reshape_2d(ctx, a, 1, a->ne[0]); struct ggml_tensor * a_sorted = ggml_get_rows(ctx, a_reshaped, b); - return ggml_reshape_1d(ctx, a_sorted, a->ne[0]); + return a_sorted; }; // Get the sorted logits in descending order. - struct ggml_tensor * sorted_idx = ggml_argsort(ctx, data->logits, GGML_SORT_ORDER_DESC); + struct ggml_tensor * sorted_idx = ggml_argsort(ctx, logits, GGML_SORT_ORDER_DESC); ggml_set_name(sorted_idx, "top_p_sorted_idx"); // Do the sorting via reshape + get_rows - struct ggml_tensor * sorted_logits = ggml_sort(data->logits, sorted_idx); + struct ggml_tensor * sorted_logits = ggml_sort(logits, sorted_idx); ggml_set_name(sorted_logits, "top_p_sorted_logits"); + sorted_logits = ggml_reshape_1d(ctx, sorted_logits, ggml_nelements(sorted_logits)); struct ggml_tensor * softmax = ggml_soft_max(ctx, sorted_logits); ggml_set_name(softmax, "top_p_softmax"); @@ -1626,10 +1635,12 @@ static void llama_sampler_min_p_backend_apply( struct llama_sampler_data * data) { auto * sctx = (llama_sampler_min_p *) smpl->ctx; - struct ggml_tensor * max_idx = ggml_argmax(ctx, data->logits); + struct ggml_tensor * logits = ggml_reshape_1d(ctx, data->logits, ggml_nelements(data->logits)); + + struct ggml_tensor * max_idx = ggml_argmax(ctx, logits); ggml_set_name(max_idx, "max_idx"); - struct ggml_tensor * logits_rows = ggml_reshape_2d(ctx, data->logits, 1, data->logits->ne[0]); + struct ggml_tensor * logits_rows = ggml_reshape_2d(ctx, logits, 1, logits->ne[0]); ggml_set_name(logits_rows, "logits_rows"); struct ggml_tensor * max_logit = ggml_get_rows(ctx, logits_rows, max_idx); @@ -1640,7 +1651,7 @@ static void llama_sampler_min_p_backend_apply( ggml_set_name(threshold, "min_p_threshold"); // Subtract the threshold from logits. - struct ggml_tensor * sub = ggml_sub(ctx, data->logits, threshold); + struct ggml_tensor * sub = ggml_sub(ctx, logits, threshold); // Create a mask where logits below the threshold are 0 (discard), // and others are 1 (keep). @@ -1652,7 +1663,7 @@ static void llama_sampler_min_p_backend_apply( struct ggml_tensor * min_p_bias = ggml_log(ctx, mask); ggml_set_name(min_p_bias, "min_p_bias"); - data->logits = ggml_add(ctx, data->logits, min_p_bias); + data->logits = ggml_add(ctx, logits, min_p_bias); ggml_set_name(data->logits, "min_p_logits"); GGML_UNUSED(gf); @@ -1829,18 +1840,20 @@ static void llama_sampler_backend_temp_sampling( struct llama_sampler_data * data, float temp) { if (temp <= 0.0f) { + struct ggml_tensor * logits = ggml_reshape_1d(ctx, data->logits, ggml_nelements(data->logits)); + // Find the most probable token index. - struct ggml_tensor * max_idx = ggml_argmax(ctx, data->logits); + struct ggml_tensor * max_idx = ggml_argmax(ctx, logits); ggml_set_name(max_idx, "temp_max_idx"); if (data->candidates) { - struct ggml_tensor * candidates_rows = ggml_reshape_2d(ctx, data->candidates, 1, data->candidates->ne[0]); + struct ggml_tensor * candidates_rows = ggml_reshape_2d(ctx, data->candidates, 1, ggml_nelements(data->candidates)); data->candidates = ggml_get_rows(ctx, candidates_rows, max_idx); } else { data->candidates = max_idx; } - struct ggml_tensor * logits_rows = ggml_reshape_2d(ctx, data->logits, 1, data->logits->ne[0]); + struct ggml_tensor * logits_rows = ggml_reshape_2d(ctx, logits, 1, ggml_nelements(logits)); data->logits = ggml_get_rows(ctx, logits_rows, max_idx); return; @@ -2019,13 +2032,15 @@ static void llama_sampler_temp_ext_backend_apply( return; } + struct ggml_tensor * logits = ggml_reshape_1d(ctx, data->logits, ggml_nelements(data->logits)); + // Calculate min_temp, max_temp, and max_entropy. const float min_temp = std::max(0.0f, sctx->temp - sctx->delta); const float max_temp = sctx->temp + sctx->delta; - const float max_entropy = logf(data->logits->ne[0]); + const float max_entropy = logf(logits->ne[0]); // Calculate the probabilities. - struct ggml_tensor * probs = ggml_soft_max(ctx, data->logits); + struct ggml_tensor * probs = ggml_soft_max(ctx, logits); ggml_set_name(probs, "temp_ext_softmax_probs"); // Clamp probabilities to avoid log(0) which would give -inf @@ -2063,7 +2078,7 @@ static void llama_sampler_temp_ext_backend_apply( ggml_set_name(dyn_temp, "temp_ext_dyn_temp"); // Scale the logits by the dynamic temperature - struct ggml_tensor * scaled_logits = ggml_div(ctx, data->logits, dyn_temp); + struct ggml_tensor * scaled_logits = ggml_div(ctx, logits, dyn_temp); ggml_set_name(scaled_logits, "temp_ext_scaled_logits"); data->logits = scaled_logits; diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index 9164a4dd88..443cd46408 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -2578,7 +2578,14 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { if (suppress_idx != -1) { const int n = gguf_get_arr_n(ctx, suppress_idx); const int32_t * data = (const int32_t *) gguf_get_arr_data(ctx, suppress_idx); - suppress_tokens.assign(data, data + n); + // drop out-of-range ids + suppress_tokens.reserve(n); + for (int i = 0; i < n; ++i) { + const int32_t id = data[i]; + if (id >= 0 && id < (int) id_to_token.size()) { + suppress_tokens.push_back(id); + } + } } } @@ -4205,6 +4212,14 @@ bool llama_vocab_get_add_sep(const struct llama_vocab * vocab) { return vocab->get_add_sep(); } +const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens) { + const std::vector & tokens = vocab->get_suppress_tokens(); + if (n_suppress_tokens) { + *n_suppress_tokens = (int32_t) tokens.size(); + } + return tokens.data(); +} + llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab) { return vocab->token_fim_pre(); } diff --git a/src/llama.cpp b/src/llama.cpp index 11ac9656d9..d6e0bbfefa 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -54,6 +54,8 @@ const char * llama_load_mode_name(enum llama_load_mode load_mode) { return "mmap"; case LLAMA_LOAD_MODE_MLOCK: return "mlock"; + case LLAMA_LOAD_MODE_MMAP_MLOCK: + return "mmap+mlock"; case LLAMA_LOAD_MODE_DIRECT_IO: return "dio"; } @@ -61,10 +63,11 @@ const char * llama_load_mode_name(enum llama_load_mode load_mode) { } enum llama_load_mode llama_load_mode_from_str(const char * str) { - if (std::strcmp(str, "none") == 0) { return LLAMA_LOAD_MODE_NONE; } - if (std::strcmp(str, "mmap") == 0) { return LLAMA_LOAD_MODE_MMAP; } - if (std::strcmp(str, "mlock") == 0) { return LLAMA_LOAD_MODE_MLOCK; } - if (std::strcmp(str, "dio") == 0) { return LLAMA_LOAD_MODE_DIRECT_IO; } + if (std::strcmp(str, "none") == 0) { return LLAMA_LOAD_MODE_NONE; } + if (std::strcmp(str, "mmap") == 0) { return LLAMA_LOAD_MODE_MMAP; } + if (std::strcmp(str, "mlock") == 0) { return LLAMA_LOAD_MODE_MLOCK; } + if (std::strcmp(str, "mmap+mlock") == 0) { return LLAMA_LOAD_MODE_MMAP_MLOCK; } + if (std::strcmp(str, "dio") == 0) { return LLAMA_LOAD_MODE_DIRECT_IO; } throw std::invalid_argument(std::string("unknown load mode: ") + str); } @@ -302,7 +305,7 @@ static std::pair llama_model_load(struct gguf_context * meta const std::string & fname, std::vector & splits, FILE * file, llama_model_params & params) { try { llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode, - params.check_tensors, params.no_alloc, params.kv_overrides, params.tensor_buft_overrides); + params.check_tensors, params.no_alloc, params.load_mtp, params.kv_overrides, params.tensor_buft_overrides); ml.print_info(); std::unique_ptr model_ptr(llama_model_create(ml, params)); diff --git a/src/models/cohere2moe.cpp b/src/models/cohere2moe.cpp index 499c73a1c4..3acb7e77af 100644 --- a/src/models/cohere2moe.cpp +++ b/src/models/cohere2moe.cpp @@ -55,7 +55,11 @@ void llama_model_cohere2moe::load_arch_tensors(llama_model_loader & ml) { const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; - const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + + if (!ml.load_mtp) { + mtp_flags |= TENSOR_SKIP; + } tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 5ad6473ce2..2d41dace0b 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -1133,6 +1133,10 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p &post, &comb, il); cb(cur, "hc_ffn_pre", il); + ggml_build_forward_expand(gf, residual); + ggml_build_forward_expand(gf, post); + ggml_build_forward_expand(gf, comb); + cur = build_norm(cur, model.layers[il].ffn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "ffn_norm", il); @@ -1175,7 +1179,7 @@ llama_model_deepseek4::graph::graph(const llama_model & model, const llm_graph_p inpL = build_hc_post(cur, residual, post, comb, il); inpL = build_cvec(inpL, il); - cb(inpL, "l_out", il); + cb(inpL, "l_last", il); } if (inp_out_ids) { diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index 427eed4594..dcff3aec9f 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -37,6 +37,23 @@ void llama_model_dflash::load_arch_tensors(llama_model_loader &) { const int64_t n_embd_inp = hparams.n_embd_inp_enc(); + // DSpark = DFlash + a semi-autoregressive Markov head and Confidence head + // + // TODO: only Qwen3-style backbones are supported for now; other backbones (e.g. Gemma4) + // need their own conversion path and graph tweaks + const struct ggml_tensor * markov_meta = ml->get_tensor_meta("markov_w1.weight"); + if (markov_meta) { + const int64_t dspark_markov_rank = markov_meta->ne[0]; + + dspark_markov_w1 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W1, "weight"), { dspark_markov_rank, n_vocab }, 0); + dspark_markov_w2 = create_tensor(tn(LLM_TENSOR_DSPARK_MARKOV_W2, "weight"), { dspark_markov_rank, n_vocab }, 0); + + dspark_conf_proj = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "weight"), { n_embd + dspark_markov_rank, 1 }, 0); + dspark_conf_proj_b = create_tensor(tn(LLM_TENSOR_DSPARK_CONF_PROJ, "bias"), { 1 }, TENSOR_NOT_REQUIRED); + + LLAMA_LOG_INFO("%s: DFlash with DSpark markov head (rank = %lld)\n", __func__, (long long) dspark_markov_rank); + } + fc = create_tensor(tn(LLM_TENSOR_FC, "weight"), { n_embd_inp, n_embd }, 0); output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), { n_embd }, 0); // encoder hidden_norm (after fc) output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), { n_embd }, 0); // decoder final norm @@ -105,6 +122,94 @@ llama_model_dflash::graph::graph(const llama_model & model, const llm_grap ggml_build_forward_expand(gf, cur); } +// DSpark (DFlash + Markov & Confidence head): Markov bias on the draft logits, chained per block position +static void build_dspark_markov_head(llm_graph_context & g, const llama_model & model, ggml_tensor * tokens) { + ggml_context * ctx0 = g.ctx0; + auto & res = g.res; + + ggml_tensor * w1 = model.dspark_markov_w1; + ggml_tensor * w2 = model.dspark_markov_w2; + GGML_ASSERT(w1 && w2 && model.dspark_conf_proj && "DSpark markov/confidence weights not loaded"); + + ggml_tensor * base = res->t_logits; // [n_vocab, n_tokens] + const int64_t n_vocab = base->ne[0]; + const int64_t n_tok = base->ne[1]; + + const auto it = model.gguf_kv.find("dflash.block_size"); + GGML_ASSERT(it != model.gguf_kv.end() && "DSpark draft requires 'dflash.block_size' in GGUF metadata"); + const int64_t block_size = std::stoi(it->second); + GGML_ASSERT(block_size > 0); + + const int64_t n_blocks = g.ubatch.n_seqs_unq; + GGML_ASSERT(n_blocks > 0 && n_tok % n_blocks == 0 && "DSpark markov head requires equal-size blocks"); + // runtime tokens per block in this ubatch (anchor + drafted positions), bounded by training block_size + const int64_t block_drafts = n_tok / n_blocks; + if (block_drafts > block_size) { + return; + } + + // anchor (committed last) token of every block: token 0 of each block, i.e. a strided view + const size_t token_stride = (size_t) block_drafts * tokens->nb[0]; + const size_t base_stride = (size_t) block_drafts * base->nb[1]; + + ggml_tensor * prev = ggml_view_2d(ctx0, tokens, 1, n_blocks, token_stride, 0); + prev = ggml_cont_1d(ctx0, prev, n_blocks); + + // confidence head input: predicts per-position acceptance + ggml_tensor * conf_inp = res->t_embd; // [n_embd, n_tok] + + ggml_tensor * cat = nullptr; + ggml_tensor * cat_conf = nullptr; + + // TODO: the in-graph chain is greedy (argmax); sampling params affect only the final + // token pick, not the Markov conditioning path + for (int64_t i = 0; i < block_drafts; ++i) { + ggml_tensor * w1_prev = ggml_get_rows(ctx0, w1, prev); // [R, n_blocks] + ggml_tensor * bias = ggml_mul_mat(ctx0, w2, w1_prev); // [n_vocab, n_blocks] + + // position i of every block: strided view [n_vocab, n_blocks] + ggml_tensor * base_i = ggml_view_2d(ctx0, base, n_vocab, n_blocks, base_stride, i*base->nb[1]); + ggml_tensor * col = ggml_add(ctx0, base_i, bias); + + cat = cat ? ggml_concat(ctx0, cat, col, 1) : col; + + // conf(i) = sigmoid(conf_proj . [conf_inp(i); markov_w1[prev(i)]] + b) -- [1, n_blocks] + ggml_tensor * conf_inp_i = ggml_view_2d(ctx0, conf_inp, conf_inp->ne[0], n_blocks, + (size_t) block_drafts * conf_inp->nb[1], i*conf_inp->nb[1]); + ggml_tensor * feat = ggml_concat(ctx0, ggml_cont(ctx0, conf_inp_i), w1_prev, 0); + ggml_tensor * conf = ggml_mul_mat(ctx0, model.dspark_conf_proj, feat); + if (model.dspark_conf_proj_b) { + conf = ggml_add(ctx0, conf, model.dspark_conf_proj_b); + } + conf = ggml_sigmoid(ctx0, conf); + + cat_conf = cat_conf ? ggml_concat(ctx0, cat_conf, conf, 1) : conf; + + if (i + 1 < block_drafts) { + prev = ggml_argmax(ctx0, col); + } + } + + // cat is position-major; restore ubatch block-major order + ggml_tensor * out = ggml_reshape_3d(ctx0, cat, n_vocab, n_blocks, block_drafts); + out = ggml_cont(ctx0, ggml_permute(ctx0, out, 0, 2, 1, 3)); // [n_vocab, block_drafts, n_blocks] + out = ggml_reshape_2d(ctx0, out, n_vocab, n_tok); + + { + ggml_tensor * conf = ggml_reshape_3d(ctx0, cat_conf, 1, n_blocks, block_drafts); + conf = ggml_cont(ctx0, ggml_permute(ctx0, conf, 0, 2, 1, 3)); + conf = ggml_reshape_2d(ctx0, conf, 1, n_tok); + + // note: broadcast the [1, n_tok] confidences to n_embd-wide rows to be able to reuse `llama_get_embeddings_nextn` + conf = ggml_repeat(ctx0, conf, res->t_embd); + res->t_h_nextn = conf; + ggml_build_forward_expand(g.gf, conf); + } + + res->t_logits = out; + ggml_build_forward_expand(g.gf, out); +} + // DFlash decoder, dual-mode by batch type: // * embd batch -> fused target features: project + inject K/V into the cache. // * token batch -> noise-block diffusion: attend over [committed, MASK...] to generate draft tokens @@ -210,6 +315,8 @@ llama_model_dflash::graph::graph(const llama_model & model, const llm_gra inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); ggml_set_input(inp->tokens); + ggml_tensor * inp_tokens = inp->tokens; + ggml_tensor * inpL = ggml_get_rows(ctx0, tok_embd, inp->tokens); cb(inpL, "inp_noise_embd", -1); @@ -290,4 +397,9 @@ llama_model_dflash::graph::graph(const llama_model & model, const llm_gra res->t_logits = cur; ggml_build_forward_expand(gf, cur); + + // DSpark: bias the draft logits with the Markov head + if (model.dspark_markov_w1) { + build_dspark_markov_head(*this, model, inp_tokens); + } } diff --git a/src/models/eagle3.cpp b/src/models/eagle3.cpp index 9d96fae594..be466056df 100644 --- a/src/models/eagle3.cpp +++ b/src/models/eagle3.cpp @@ -28,6 +28,10 @@ void llama_model_eagle3::load_arch_hparams(llama_model_loader & ml) { LLAMA_LOG_INFO("%s: EAGLE3gnorm_before_residual = true\n", __func__); } + // eagle3 norm_before_fc (optional, default false) + // compatible with eagle3.1 (e.g. nvidia/gpt-oss-120b-Eagle3-v3) + ml.get_key(LLM_KV_NORM_BEFORE_FC, hparams.norm_before_fc, false); + type = LLM_TYPE_UNKNOWN; } @@ -53,6 +57,11 @@ void llama_model_eagle3::load_arch_tensors(llama_model_loader &) { // Feature fusion layer: projects 3 target layers to draft hidden size fc = create_tensor(tn(LLM_TENSOR_FC, "weight"), {n_embd_inp, n_embd}, 0); + // RMSNorm on the fused target features (input to fc), only when norm_before_fc is set. + if (hparams.norm_before_fc) { + output_norm_enc = create_tensor(tn(LLM_TENSOR_ENC_OUTPUT_NORM, "weight"), {n_embd_inp}, 0); + } + // Output layer (uses draft vocab size) output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_draft_vocab}, TENSOR_NOT_REQUIRED); @@ -130,6 +139,12 @@ llama_model_eagle3::graph::graph(const llama_model & model, const llm_grap cur = build_inp_embd_enc(); + // RMSNorm on the fused target features before fc + if (hparams.norm_before_fc) { + cur = build_norm(cur, model.output_norm_enc, NULL, LLM_NORM_RMS, -1); + cb(cur, "enc_input_norm", -1); + } + // Feature fusion layer cur = build_lora_mm(model.fc, cur); cb(cur, "fc_out", -1); diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp index 6a96979ceb..e44f423bdb 100644 --- a/src/models/gemma4.cpp +++ b/src/models/gemma4.cpp @@ -142,33 +142,6 @@ static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, in idx * x->ne[0] * x->ne[1] * ggml_element_size(x)); } -// TODO @ngxson : maybe improve this in the future -class llm_graph_input_logits_bias : public llm_graph_input_i { -public: - llm_graph_input_logits_bias(const llama_vocab & vocab) { - arr.resize(vocab.n_tokens(), 0.0f); - for (llama_token id : vocab.get_suppress_tokens()) { - if (0 <= id && id < (int32_t)vocab.n_tokens()) { - arr[id] = -INFINITY; - } - } - } - virtual ~llm_graph_input_logits_bias() = default; - - void set_input(const llama_ubatch * /*ubatch*/) override { - const int64_t n_vocab = arr.size(); - ggml_backend_tensor_set(logits_bias, arr.data(), 0, n_vocab*ggml_element_size(logits_bias)); - } - - bool can_reuse(const llm_graph_params & /*params*/) override { - return true; - } - - ggml_tensor * logits_bias = nullptr; // F32 [n_vocab] - - std::vector arr; -}; - llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params), model(model), @@ -429,16 +402,6 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping); } - // apply logits bias if needed (e.g. for gemma4_unified patch) - // this is to mirror the suppress_tokens patch on transformers, to avoid model from outputing and tokens (which is a known issue related to the checkpoint) - // TODO: maybe handle this inside the sampling system in the future - if (!model.vocab.get_suppress_tokens().empty()) { - auto inp_bias = std::make_unique(model.vocab); - inp_bias->logits_bias = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, inp_bias->arr.size()); - cur = ggml_add(ctx0, cur, inp_bias->logits_bias); - res->add_input(std::move(inp_bias)); - } - cb(cur, "result_output", -1); res->t_logits = cur; diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp index df190e1f63..360c2ee773 100644 --- a/src/models/glm-dsa.cpp +++ b/src/models/glm-dsa.cpp @@ -72,15 +72,31 @@ void llama_model_glm_dsa::load_arch_hparams(llama_model_loader & ml) { ml.get_key_or_arr(LLM_KV_ATTENTION_INDEXER_TYPES, hparams.is_indexer_full_impl, hparams.n_layer(), false); switch (hparams.n_layer()) { - case 78: type = LLM_TYPE_744B_A40B; break; + case 78: // GGUF with NextN/MTP metadata: n_layer() excludes the nextn layer + case 79: + type = LLM_TYPE_744B_A40B; break; default: type = LLM_TYPE_UNKNOWN; } } -void llama_model_glm_dsa::load_arch_tensors(llama_model_loader &) { +void llama_model_glm_dsa::load_arch_tensors(llama_model_loader & ml) { LLAMA_LOAD_LOCALS; const int64_t n_expert_shared = hparams.n_expert_shared; + // MTP-only: the GGUF carries only the NextN/MTP block(s) (user split target/draft). + const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr); + // Trunk-only: the GGUF declares MTP layers in metadata but the actual MTP + // tensors live in a separate file (or were stripped at conversion). Mark + // MTP tensors NOT_REQUIRED so the trunk loads cleanly. + const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; + const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); + const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + + if (!ml.load_mtp) { + mtp_flags |= TENSOR_SKIP; + } + const bool is_mla = hparams.is_mla(); if (!is_mla) { throw std::runtime_error("GLM_DSA architecture requires MLA"); @@ -109,12 +125,9 @@ void llama_model_glm_dsa::load_arch_tensors(llama_model_loader &) { } for (int i = 0; i < n_layer_all; ++i) { - int flags = 0; - if (i >= n_layer) { - // skip all tensors in the NextN layers - // TODO @ngxson : TENSOR_NOT_REQUIRED was a hack, need to remove it later - flags |= TENSOR_SKIP | TENSOR_NOT_REQUIRED; - } + // NextN/MTP layers (i >= n_layer) are full decoder blocks used by the + // LLM_GRAPH_TYPE_DECODER_MTP draft head; load them like qwen35moe/step35/hy_v3. + const int flags = (i >= n_layer) ? mtp_flags : trunk_flags; auto & layer = layers[i]; @@ -167,7 +180,7 @@ void llama_model_glm_dsa::load_arch_tensors(llama_model_loader &) { layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_exp * n_expert_shared}, flags); } - // NextN/MTP tensors (preserved but unused) - conditionally load for last n_layer_nextn + // NextN/MTP tensors - the NextN-specific wiring around the extra decoder block if (i >= n_layer) { layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), { 2 * n_embd, n_embd }, flags); layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), { n_embd }, flags); @@ -182,6 +195,9 @@ void llama_model_glm_dsa::load_arch_tensors(llama_model_loader &) { } std::unique_ptr llama_model_glm_dsa::build_arch_graph(const llm_graph_params & params) const { + if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { + return std::make_unique(*this, params); + } return std::make_unique(*this, params); } @@ -469,7 +485,9 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par Qcur, Kcur, Vcur, nullptr, nullptr, model.layers[il].wv_b, top_k, kq_scale, il); } } - if (il == n_layer - 1 && inp_out_ids) { + // when unmasked nextn embeddings are requested, t_h_nextn must keep all rows, + // so the early output masking has to be skipped (it is applied after the final norm instead) + if (il == n_layer - 1 && inp_out_ids && (!cparams.embeddings_nextn || cparams.embeddings_nextn_masked)) { cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); } @@ -532,6 +550,14 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + // post-norm hidden state feeds the NextN/MTP draft head + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + if (cparams.embeddings_nextn && !cparams.embeddings_nextn_masked && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + cb(cur, "result_norm", -1); res->t_embd = cur; @@ -543,3 +569,242 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par ggml_build_forward_expand(gf, cur); } + +// LLM_GRAPH_TYPE_DECODER_MTP draft head for GLM-5.2 (GLM_DSA). +// Semantics mirror the deepseek-family NextN/MTP layer: +// enorm(embed) + hnorm(prev_hidden) -> concat(e, h) -> eh_proj -> +// full glm_dsa decoder block (dense MLA attention + sigmoid-gated MoE FFN +// with shared expert, exactly as the trunk deepseek2 graph builds it) -> +// shared_head_norm (fallback output_norm) -> shared LM head. +// The DSA indexer is not used at runtime (same as the trunk graph). +llama_model_glm_dsa::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) + : llm_graph_context(params) { + GGML_ASSERT(hparams.n_layer_nextn > 0 && "GLM_DSA MTP requires n_layer_nextn > 0"); + GGML_ASSERT(hparams.n_layer_nextn == 1 && "GLM_DSA MTP currently only supports a single MTP block"); + GGML_ASSERT(hparams.is_mla() && "GLM_DSA MTP requires MLA"); + + const int il = hparams.n_layer() + cparams.nextn_layer_offset; + GGML_ASSERT(cparams.nextn_layer_offset >= 0 && + cparams.nextn_layer_offset < (int) hparams.n_layer_nextn && + "nextn_layer_offset out of range [0, n_layer_nextn)"); + const auto & layer = model.layers[il]; + + GGML_ASSERT(layer.nextn.eh_proj && "MTP block missing nextn.eh_proj"); + GGML_ASSERT(layer.nextn.enorm && "MTP block missing nextn.enorm"); + GGML_ASSERT(layer.nextn.hnorm && "MTP block missing nextn.hnorm"); + GGML_ASSERT(layer.ffn_gate_inp && "MTP block missing ffn_gate_inp"); + + // note: these are the actual head sizes you get when treating as MHA or after "decompression" using wv_b for MLA + const int64_t n_embd_head_k = hparams.n_embd_head_k_mla(); + + const int64_t n_embd_head_qk_rope = hparams.n_rot(); + const int64_t n_embd_head_qk_nope = n_embd_head_k - n_embd_head_qk_rope; + + const uint32_t kv_lora_rank = hparams.n_lora_kv; + + // We have to pre-scale kq_scale and attn_factor to make the YaRN RoPE work correctly. + // See the deepseek2 trunk graph for the detailed explanation - this must match it EXACTLY. + GGML_ASSERT(ext_factor >= 0.0f); + const float attn_factor_org = attn_factor * (1.0f + 0.1f * logf(1.0f / freq_scale)); + + const float mscale = attn_factor_org * (1.0f + 0.1f * hparams.rope_yarn_log_mul * logf(1.0f / freq_scale)); + const float kq_scale = 1.0f * mscale * mscale / sqrtf(float(n_embd_head_k)); + + // TODO: extract in a common llm_graph_context::build_inp_embd_h() + auto inp = std::make_unique(hparams.n_embd); + + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); + ggml_set_input(inp->embd); + + ggml_tensor * tok_embd; + if (ubatch.token) { + ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; + + tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens); + } else { + tok_embd = inp->embd; + } + cb(tok_embd, "mtp_tok_embd", il); + + inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); + ggml_set_input(inp->h); + ggml_set_name(inp->h, "mtp_h_input"); + + ggml_tensor * h_embd = inp->h; + + res->add_input(std::move(inp)); + + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + // MLA with the absorption optimization uses a K-only cache (V is a view of K) + auto * inp_attn = build_attn_inp_k(); + + ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); + cb(h_norm, "mtp_hnorm", il); + + ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il); + cb(e_norm, "mtp_enorm", il); + + ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0); + cb(concat, "mtp_concat", il); + + ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s); + cb(cur, "mtp_eh_proj", il); + + ggml_tensor * inpSA = cur; + + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_attn_norm", il); + + // self-attention: dense MLA, same construction as the deepseek2 trunk graph + { + ggml_tensor * q = ggml_mul_mat(ctx0, layer.wq_a, cur); + cb(q, "mtp_q", il); + + q = build_norm(q, layer.attn_q_a_norm, nullptr, LLM_NORM_RMS, il); + cb(q, "mtp_q", il); + + q = ggml_mul_mat(ctx0, layer.wq_b, q); + cb(q, "mtp_q", il); + + // split into {n_embd_head_qk_nope, n_head, n_tokens} + ggml_tensor * q_nope = + ggml_view_3d(ctx0, q, n_embd_head_qk_nope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k), + ggml_row_size(q->type, n_embd_head_k) * n_head, 0); + cb(q_nope, "mtp_q_nope", il); + + // and {n_embd_head_qk_rope, n_head, n_tokens} + ggml_tensor * q_pe = ggml_view_3d( + ctx0, q, n_embd_head_qk_rope, n_head, n_tokens, ggml_row_size(q->type, n_embd_head_k), + ggml_row_size(q->type, n_embd_head_k) * n_head, ggml_row_size(q->type, n_embd_head_qk_nope)); + cb(q_pe, "mtp_q_pe", il); + + ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur); + cb(kv_cmpr_pe, "mtp_kv_cmpr_pe", il); + + // split into {kv_lora_rank, n_tokens} + ggml_tensor * kv_cmpr = + ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0); + cb(kv_cmpr, "mtp_kv_cmpr", il); + + // and {n_embd_head_qk_rope, 1, n_tokens} + ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens, + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), + ggml_row_size(kv_cmpr_pe->type, kv_lora_rank)); + cb(k_pe, "mtp_k_pe", il); + + q_pe = ggml_rope_ext(ctx0, q_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(q_pe, "mtp_q_pe", il); + + k_pe = ggml_rope_ext(ctx0, k_pe, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + cb(k_pe, "mtp_k_pe", il); + + kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il); + cb(kv_cmpr, "mtp_kv_cmpr", il); + + // {n_embd_head_qk_nope, n_tokens, n_head} + q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3); + cb(q_nope, "mtp_q_nope_perm", il); + + // {n_embd_head_qk_nope, kv_lora_rank, n_head} x {n_embd_head_qk_nope, n_tokens, n_head} + ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope); + cb(q_nope_absorbed, "mtp_q_nope_absorbed", il); + + // {kv_lora_rank, n_head, n_tokens} + q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3); + cb(q_nope_absorbed, "mtp_q_nope_absorbed_perm", il); + + // {n_embd_head_qk_rope + kv_lora_rank, n_head, n_tokens} + // note: rope must go first for in-place context shifting in build_rope_shift() + ggml_tensor * Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0); + cb(Qcur, "mtp_Qcur", il); + + kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens); + cb(kv_cmpr, "mtp_kv_cmpr_reshape", il); + + // {n_embd_head_qk_rope + kv_lora_rank, 1, n_tokens} + ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0); + cb(Kcur, "mtp_Kcur", il); + + // {kv_lora_rank, 1, n_tokens} + ggml_tensor * Vcur = kv_cmpr; + cb(Vcur, "mtp_Vcur", il); + + // note: MLA with the absorption optimization converts into MQA (ie: GQA with 1 group) + cur = build_attn(inp_attn, + layer.wo, NULL, layer.wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale, il); + cb(cur, "mtp_attn_out", il); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "mtp_ffn_inp", il); + + cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "mtp_ffn_norm", il); + + // MoE FFN with shared expert - same construction as the deepseek2 trunk graph + ggml_tensor * moe_out = build_moe_ffn(cur, + layer.ffn_gate_inp, + layer.ffn_up_exps, + layer.ffn_gate_exps, + layer.ffn_down_exps, + layer.ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, hparams.expert_weights_norm, + hparams.expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, + il, + nullptr, + layer.ffn_gate_up_exps, + layer.ffn_up_exps_s, + layer.ffn_gate_exps_s, + layer.ffn_down_exps_s); + cb(moe_out, "mtp_ffn_moe_out", il); + + // FFN shared expert + ggml_tensor * ffn_shexp = + build_ffn(cur, + layer.ffn_up_shexp, NULL, layer.ffn_up_shexp_s, + layer.ffn_gate_shexp, NULL, layer.ffn_gate_shexp_s, + layer.ffn_down_shexp, NULL, layer.ffn_down_shexp_s, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(ffn_shexp, "mtp_ffn_shexp", il); + + cur = ggml_add(ctx0, moe_out, ffn_shexp); + cb(cur, "mtp_ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "mtp_post_ffn", il); + + // shared_head_norm applied after the decoder block, before the shared LM head. + // The post-norm hidden state seeds the next MTP step. + ggml_tensor * head_norm_w = layer.nextn.shared_head_norm + ? layer.nextn.shared_head_norm + : model.output_norm; + GGML_ASSERT(head_norm_w && "GLM_DSA MTP: missing both nextn.shared_head_norm and output_norm"); + cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1); + + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + cb(cur, "mtp_shared_head_norm", -1); + + ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output; + ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s; + GGML_ASSERT(head_w && "GLM_DSA MTP: missing LM head (nextn.shared_head_head or model.output)"); + cur = build_lora_mm(head_w, cur, head_s); + cb(cur, "result_output", -1); + + res->t_logits = cur; + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/hy-v3.cpp b/src/models/hy-v3.cpp index 47a0beaf21..61db93af85 100644 --- a/src/models/hy-v3.cpp +++ b/src/models/hy-v3.cpp @@ -33,7 +33,11 @@ void llama_model_hy_v3::load_arch_tensors(llama_model_loader & ml) { const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; - const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + + if (!ml.load_mtp) { + mtp_flags |= TENSOR_SKIP; + } tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp index fb55ec12f9..82c9a9538c 100644 --- a/src/models/laguna.cpp +++ b/src/models/laguna.cpp @@ -58,6 +58,7 @@ void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) { switch (hparams.n_layer()) { case 40: type = LLM_TYPE_30B_A3B; break; // Laguna-XS.2 + case 48: type = LLM_TYPE_118B_A8B; break; // Laguna-S.2 case 70: type = LLM_TYPE_230B_A10B; break; // Laguna-M.1 default: type = LLM_TYPE_UNKNOWN; } diff --git a/src/models/mimo2.cpp b/src/models/mimo2.cpp index 8898916057..4080a934cb 100644 --- a/src/models/mimo2.cpp +++ b/src/models/mimo2.cpp @@ -25,9 +25,13 @@ void llama_model_mimo2::load_arch_hparams(llama_model_loader & ml) { } } -void llama_model_mimo2::load_arch_tensors(llama_model_loader &) { +void llama_model_mimo2::load_arch_tensors(llama_model_loader & ml) { LLAMA_LOAD_LOCALS; + const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; + const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); + const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); // output @@ -40,41 +44,46 @@ void llama_model_mimo2::load_arch_tensors(llama_model_loader &) { uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(i); uint32_t n_head = hparams.n_head(i); - // NextN/MTP layers (the last n_nextn blocks) are preserved but disabled pending support const bool is_nextn = i >= n_layer; - const int skip = is_nextn ? TENSOR_SKIP : 0; + const int flags = is_nextn ? mtp_flags : 0; - create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, skip); - layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_v * n_head, n_embd }, skip); + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, flags); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_v * n_head, n_embd }, flags); - layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, skip); - layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, TENSOR_NOT_REQUIRED | skip); + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags); + layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, TENSOR_NOT_REQUIRED | flags); - layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, skip); + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags); // non-MoE branch - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED | skip); - layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, TENSOR_NOT_REQUIRED | skip); - layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED | skip); + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED | flags); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, TENSOR_NOT_REQUIRED | flags); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED | flags); // MoE branch int64_t n_ff_exp = hparams.n_ff_exp; - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED | skip); - layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED | skip); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, TENSOR_NOT_REQUIRED | skip); - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED | skip); - layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED | skip); + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED | flags); + layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED | flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, TENSOR_NOT_REQUIRED | flags); + layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED | flags); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED | flags); if (is_nextn) { - layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), {2 * n_embd, n_embd}, skip); - layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), {n_embd}, skip); - layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), {n_embd}, skip); - layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, skip); + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), {2 * n_embd, n_embd}, flags); + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), {n_embd}, flags); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), {n_embd}, flags); + layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", i), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED | flags); + layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED | flags); + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED | flags); + layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED | flags); } } } std::unique_ptr llama_model_mimo2::build_arch_graph(const llm_graph_params & params) const { + if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) { + return std::make_unique(*this, params); + } return std::make_unique(*this, params); } @@ -89,6 +98,8 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_param ggml_tensor * inp_out_ids = build_inp_out_ids(); const float v_scale = hparams.f_attn_value_scale; + const bool emit_h_nextn = cparams.embeddings_nextn; + const bool crop_last_layer = inp_out_ids && (!emit_h_nextn || cparams.embeddings_nextn_masked); for (int il = 0; il < n_layer; ++il) { ggml_tensor * inpSA = inpL; @@ -168,7 +179,7 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_param } } - if (il == n_layer - 1 && inp_out_ids) { + if (il == n_layer - 1 && crop_last_layer) { cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); } @@ -218,6 +229,15 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_param cur = inpL; + if (emit_h_nextn) { + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + if (!cparams.embeddings_nextn_masked && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + } + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); @@ -233,3 +253,143 @@ llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_param ggml_build_forward_expand(gf, cur); } + +// Mirrors MiMo's appended NextN block: normalize and fuse token and hidden inputs, run the decoder block, +// expose its pre-head-norm state to the next draft step, then apply the shared output norm and LM head. +// Converted checkpoints may store that shared norm as layer_out_norm, so it remains in the fallback chain. +llama_model_mimo2::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) + : llm_graph_context(params) { + GGML_ASSERT(hparams.n_layer_nextn > 0 && "MIMO2 MTP requires n_layer_nextn > 0"); + + const int il = hparams.n_layer() + cparams.nextn_layer_offset; + GGML_ASSERT(cparams.nextn_layer_offset >= 0 && + cparams.nextn_layer_offset < (int) hparams.n_layer_nextn && + "nextn_layer_offset out of range [0, n_layer_nextn)"); + + const auto & layer = model.layers[il]; + GGML_ASSERT(layer.nextn.eh_proj && "MIMO2 MTP block missing nextn.eh_proj"); + GGML_ASSERT(layer.nextn.enorm && "MIMO2 MTP block missing nextn.enorm"); + GGML_ASSERT(layer.nextn.hnorm && "MIMO2 MTP block missing nextn.hnorm"); + GGML_ASSERT(layer.wqkv && "MIMO2 MTP requires fused attn_qkv"); + + const uint32_t n_head_l = hparams.n_head(il); + const uint32_t n_head_kv_l = hparams.n_head_kv(il); + + const float freq_base_l = model.get_rope_freq_base(cparams, il); + const float freq_scale_l = model.get_rope_freq_scale(cparams, il); + const float v_scale = hparams.f_attn_value_scale; + + auto inp = std::make_unique(hparams.n_embd); + + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); + ggml_set_input(inp->embd); + ggml_set_name(inp->embd, "mtp_h_input"); + + ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; + ggml_tensor * h_input = inp->embd; + ggml_tensor * tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens); + cb(tok_embd, "mtp_tok_embd", il); + + res->add_input(std::move(inp)); + + ggml_tensor * inp_pos = build_inp_pos(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + auto * inp_attn = build_attn_inp_kv_iswa(); + + ggml_tensor * h_norm = build_norm(h_input, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); + cb(h_norm, "mtp_hnorm", il); + + ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il); + cb(e_norm, "mtp_enorm", il); + + ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0); + cb(concat, "mtp_concat", il); + + ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s); + cb(cur, "mtp_eh_proj", il); + + ggml_tensor * inpSA = cur; + + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_attn_norm", il); + + ggml_tensor * qkv = build_lora_mm(layer.wqkv, cur, layer.wqkv_s); + cb(qkv, "mtp_wqkv", il); + + const size_t row_k = ggml_row_size(qkv->type, n_embd_head_k); + const size_t row_v = ggml_row_size(qkv->type, n_embd_head_v); + const size_t row_full = qkv->nb[1]; + const size_t k_off = row_k * n_head_l; + const size_t v_off = k_off + row_k * n_head_kv_l; + + ggml_tensor * Qcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_l, n_tokens, row_k, row_full, 0); + ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_kv_l, n_tokens, row_k, row_full, k_off); + ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_kv_l, n_tokens, row_v, row_full, v_off); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, nullptr, + n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l, + ext_factor, attn_factor, beta_fast, beta_slow); + + cb(Qcur, "mtp_Qcur", il); + cb(Kcur, "mtp_Kcur", il); + cb(Vcur, "mtp_Vcur", il); + + cur = build_attn(inp_attn, + layer.wo, nullptr, layer.wo_s, + Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, nullptr, + 1.0f / sqrtf(float(n_embd_head_k)), il); + cb(cur, "mtp_attn_out", il); + + if (v_scale) { + cur = ggml_scale(ctx0, cur, v_scale); + cb(cur, "mtp_attn_out_scaled", il); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "mtp_ffn_inp", il); + + cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_ffn_norm", il); + + GGML_ASSERT(layer.ffn_gate && layer.ffn_down && layer.ffn_up && "MIMO2 MTP requires dense FFN tensors"); + cur = build_ffn(cur, + layer.ffn_up, layer.ffn_up_b, nullptr, + layer.ffn_gate, layer.ffn_gate_b, nullptr, + layer.ffn_down, layer.ffn_down_b, nullptr, + nullptr, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "mtp_ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "mtp_post_ffn", il); + + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + + cb(cur, "h_nextn", -1); + res->t_h_nextn = cur; + + ggml_tensor * head_norm_w = layer.nextn.shared_head_norm + ? layer.nextn.shared_head_norm + : (layer.layer_out_norm ? layer.layer_out_norm : model.output_norm); + GGML_ASSERT(head_norm_w && "MIMO2 MTP missing head norm fallback"); + cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1); + cb(cur, "mtp_shared_head_norm", -1); + + ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output; + ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s; + GGML_ASSERT(head_w && "MIMO2 MTP missing LM head fallback"); + cur = build_lora_mm(head_w, cur, head_s); + cb(cur, "result_output", -1); + + res->t_logits = cur; + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp index 6068fc6b87..0773ad5435 100644 --- a/src/models/minimax-m3.cpp +++ b/src/models/minimax-m3.cpp @@ -2,7 +2,6 @@ #include "llama-kv-cache.h" #include #include -#include #include // MiniMax-M3: MiniMax-M2 style GQA (per-head QK-norm, partial rotary) with @@ -126,68 +125,6 @@ public: int64_t nblk; }; -// pooled score of a block with no visible token: -inf from the mask, or -FLT_MAX from the -// max-pool identity when every element of the block is -inf -static inline bool msa_score_masked(float x) { return x <= -1e30f; } - -// MSA block selection (batch regime) -// CPU custom op, the token-level expansion and the combination with the causal mask happen on the GPU. -static void msa_block_mask_op(struct ggml_tensor * dst, int ith, int nth, void * userdata) { - const struct ggml_tensor * bs = dst->src[0]; - const struct ggml_tensor * bias = dst->src[1]; - const msa_params * p = (const msa_params *) userdata; - - const int nblk = (int) bs->ne[0]; - const int Hd = (int) bs->ne[1]; - const int S = (int) bs->ne[2]; - - GGML_ASSERT(bs->type == GGML_TYPE_F32 && ggml_is_contiguous(bs)); - GGML_ASSERT(bias->type == GGML_TYPE_F32 && ggml_is_contiguous(bias)); - GGML_ASSERT(dst->type == GGML_TYPE_F16 && ggml_is_contiguous(dst)); - GGML_ASSERT(dst->ne[0] == nblk && dst->ne[1] == S && dst->ne[2] == Hd); - GGML_ASSERT(bias->ne[0] == nblk && bias->ne[1] == S); - - const int topk = p->topk_blocks < nblk ? p->topk_blocks : nblk; - - const ggml_fp16_t f16_zero = ggml_fp32_to_fp16(0.0f); - const ggml_fp16_t f16_ninf = ggml_fp32_to_fp16(-INFINITY); - - std::vector rank(nblk); - std::vector valid(nblk); - std::vector ord(nblk); - - ggml_fp16_t * out = (ggml_fp16_t *) dst->data; - - for (int i = ith; i < S; i += nth) { - const float * bias_col = (const float *) bias->data + (size_t) i * nblk; - for (int h = 0; h < Hd; ++h) { - const float * bs_col = (const float *) bs->data + ((size_t) i * Hd + h) * nblk; - - for (int bk = 0; bk < nblk; ++bk) { - // a block is selectable if it has a visible token or is locally forced - valid[bk] = !msa_score_masked(bs_col[bk]) || bias_col[bk] > 0.0f; - rank [bk] = bias_col[bk] > 0.0f ? bias_col[bk] : bs_col[bk]; - ord [bk] = bk; - } - - std::partial_sort(ord.begin(), ord.begin() + topk, ord.end(), - [&](int a, int b) { return rank[a] > rank[b]; }); - - ggml_fp16_t * dst_col = out + ((size_t) h * S + i) * nblk; - for (int bk = 0; bk < nblk; ++bk) { - dst_col[bk] = f16_ninf; - } - for (int t = 0; t < topk; ++t) { - const int bk = ord[t]; - if (!valid[bk]) { - break; // sorted desc: first invalid -> fewer than topk selectable blocks - } - dst_col[bk] = f16_zero; - } - } - } -} - // One FA call for all GQA groups (and at multi-stream decode, all streams) by mapping them onto the FA sequence dim (ne[3]) ggml_tensor * llama_model_minimax_m3::graph::build_attn_msa_fa( ggml_tensor * q_cur, // [D, HQ, T] @@ -334,8 +271,6 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ } else { const int64_t n_idx_dim = hparams.indexer_head_size; // 128 - GGML_ASSERT(!inp_attn->self_k_rot && !inp_attn->self_v_rot && "MSA: attn-rot not supported"); - // Index Branch, project, norm, partial RoPE, cache ggml_tensor * iq = build_lora_mm(model.layers[il].index_q_proj, cur); ggml_tensor * ik = build_lora_mm(model.layers[il].index_k_proj, cur); @@ -352,6 +287,14 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ ggml_build_forward_expand(gf, mctx_cur->cpy_k_idx(ctx0, ik, inp_attn->get_k_idxs(), il)); ggml_tensor * ik_kv = mctx_cur->get_k_idx(ctx0, il); + if (inp_attn->self_k_rot) { + Qcur = llama_mul_mat_hadamard(ctx0, Qcur, inp_attn->self_k_rot); + Kcur = llama_mul_mat_hadamard(ctx0, Kcur, inp_attn->self_k_rot); + } + if (inp_attn->self_v_rot) { + Vcur = llama_mul_mat_hadamard(ctx0, Vcur, inp_attn->self_v_rot); + } + // Main branch: store K/V, take cache views ggml_build_forward_expand(gf, Qcur); ggml_build_forward_expand(gf, Kcur); @@ -433,8 +376,8 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ msa_mf->nb[1], msa_mf->nb[1], st*msa_mf->nb[3]); ggml_tensor * km_s = ggml_view_3d(ctx0, msa_kqm, n_kv, n_tps, 1, msa_kqm->nb[1], msa_kqm->nb[3], st*msa_kqm->nb[3]); - ggml_tensor * bias_s = ggml_view_2d(ctx0, msa_loc->bias, nblk, n_tps, - msa_loc->bias->nb[1], st*n_tps*msa_loc->bias->nb[1]); + ggml_tensor * bias_s = ggml_view_3d(ctx0, msa_loc->bias, nblk, 1, n_tps, + msa_loc->bias->nb[1], msa_loc->bias->nb[1], st*n_tps*msa_loc->bias->nb[1]); ggml_tensor * q_s = ggml_view_3d(ctx0, Qcur, D, n_head, n_tps, Qcur->nb[1], Qcur->nb[2], st*n_tps*Qcur->nb[2]); ggml_tensor * k_s = ggml_view_4d(ctx0, k, D, HKV, n_kv, 1, @@ -453,15 +396,27 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * bs = ggml_pool_2d(ctx0, sc, GGML_OP_POOL_MAX, blk, 1, blk, 1, 0, 0); cb(bs, "msa_bs", il); - // block-level 0/-inf keep mask on the CPU, tiny transfer - ggml_tensor * srcs[2] = { bs, bias_s }; - ggml_tensor * bm = ggml_custom_4d(ctx0, GGML_TYPE_F16, - nblk, n_tps, Hd, 1, - srcs, 2, msa_block_mask_op, GGML_N_TASKS_MAX, - const_cast(&mm.msa_p)); + // bias the scores so locally-forced blocks always rank first + ggml_tensor * bsf = ggml_add(ctx0, bs, bias_s); // [nblk, Hd, n_tps] + cb(bsf, "msa_bsf", il); + + ggml_tensor * idx = ggml_top_k(ctx0, bsf, K); // [K, Hd, n_tps] i32 + + ggml_tensor * ninf = ggml_cast(ctx0, + ggml_scale_bias(ctx0, bias_s, 0.0f, -1e30f), + GGML_TYPE_F16); // [nblk, 1, n_tps] + ninf = ggml_repeat_4d(ctx0, ninf, nblk, Hd, n_tps, 1); + ggml_tensor * zero = ggml_scale(ctx0, + ggml_cast(ctx0, idx, GGML_TYPE_F32), 0.0f); + ggml_tensor * bm = ggml_set_rows(ctx0, + ggml_reshape_3d(ctx0, ninf, 1, nblk, Hd*n_tps), + ggml_reshape_3d(ctx0, zero, 1, K, Hd*n_tps), + ggml_reshape_2d(ctx0, idx, K, Hd*n_tps)); + bm = ggml_reshape_3d(ctx0, bm, nblk, Hd, n_tps); + bm = ggml_cont(ctx0, ggml_permute(ctx0, bm, 0, 2, 1, 3)); // [nblk, n_tps, Hd] cb(bm, "msa_block_mask", il); - // expand block -> token granularity on the GPU (j = bk*blk + t), + // expand block -> token granularity (j = bk*blk + t), // then combine with the causal mask in place ggml_tensor * bmx = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, bm, 1, nblk, n_tps*Hd), @@ -482,7 +437,9 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ cur = ggml_concat(ctx0, cur, outs[st], 1); } } - + if (inp_attn->self_v_rot) { + cur = llama_mul_mat_hadamard(ctx0, cur, inp_attn->self_v_rot); + } cb(cur, "kqv_out", il); if (model.layers[il].wo) { cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s); diff --git a/src/models/models.h b/src/models/models.h index 7c7e092e0e..6cdd92bde1 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -424,6 +424,22 @@ struct llama_model_mellum : public llama_model_base { std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; +struct llama_model_nanbeige : public llama_model_base { + llama_model_nanbeige(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + int n_loops = 1; + int n_layer_phys = 0; + bool skip_loop_final_norm = false; + + struct graph : public llm_graph_context { + graph(const llama_model & model, const llm_graph_params & params); + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + struct llama_model_qwen : public llama_model_base { llama_model_qwen(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; @@ -1234,6 +1250,10 @@ struct llama_model_glm_dsa : public llama_model_base { graph(const llama_model & model, const llm_graph_params & params); }; + struct graph_mtp : public llm_graph_context { + graph_mtp(const llama_model & model, const llm_graph_params & params); + }; + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; @@ -2120,6 +2140,10 @@ struct llama_model_mimo2 : public llama_model_base { graph(const llama_model & model, const llm_graph_params & params); }; + struct graph_mtp : public llm_graph_context { + graph_mtp(const llama_model & model, const llm_graph_params & params); + }; + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; }; diff --git a/src/models/nanbeige.cpp b/src/models/nanbeige.cpp new file mode 100644 index 0000000000..3a546600fa --- /dev/null +++ b/src/models/nanbeige.cpp @@ -0,0 +1,184 @@ +#include "models.h" + +void llama_model_nanbeige::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + uint32_t n_loops_u = 1; + ml.get_key(LLM_KV_NUM_LOOPS, n_loops_u, false); + GGML_ASSERT(n_loops_u >= 1); + + skip_loop_final_norm = false; + ml.get_key(LLM_KV_SKIP_LOOP_FINAL_NORM, skip_loop_final_norm, false); + + n_layer_phys = (int) hparams.n_layer(); + + // Bound-check before casting: signed int mul can overflow and bypass the guard. + GGML_ASSERT((size_t) n_layer_phys * (size_t) n_loops_u <= (size_t) LLAMA_MAX_LAYERS); + n_loops = (int) n_loops_u; + + // Expand logical layer count before load_tensors() allocates layers / KV. + if (n_loops > 1) { + for (int j = 1; j < n_loops; ++j) { + for (int i = 0; i < n_layer_phys; ++i) { + const int dst = i + j * n_layer_phys; + hparams.n_head_arr[dst] = hparams.n_head_arr[i]; + hparams.n_head_kv_arr[dst] = hparams.n_head_kv_arr[i]; + hparams.n_ff_arr[dst] = hparams.n_ff_arr[i]; + hparams.is_swa_impl[dst] = hparams.is_swa_impl[i]; + hparams.is_recr_impl[dst] = hparams.is_recr_impl[i]; + } + } + hparams.n_layer_all = (uint32_t) ((size_t) n_layer_phys * (size_t) n_loops); + } + + type = LLM_TYPE_UNKNOWN; +} + +void llama_model_nanbeige::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + const int n_phys = n_layer_phys > 0 ? n_layer_phys : n_layer; + for (int i = 0; i < n_phys; ++i) { + auto & layer = layers[i]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); + + create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd}, 0); + + layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, + TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0)); + + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); + } + + // Share physical weights across loops; each slot still has its own KV index. + if (n_loops > 1) { + for (int j = 1; j < n_loops; ++j) { + for (int i = 0; i < n_phys; ++i) { + layers[i + j * n_phys] = layers[i]; + } + } + } +} + +std::unique_ptr llama_model_nanbeige::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +llama_model_nanbeige::graph::graph(const llama_model & model, const llm_graph_params & params) : + llm_graph_context(params) { + const auto & nb = static_cast(model); + + const int64_t n_embd_head = hparams.n_embd_head_v(); + GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + + const int n_phys = nb.n_layer_phys > 0 ? nb.n_layer_phys : (int) n_layer; + const int n_loops = nb.n_loops > 0 ? nb.n_loops : 1; + + ggml_tensor * cur; + ggml_tensor * inpL; + + inpL = build_inp_embd(model.tok_embd); + + ggml_tensor * inp_pos = build_inp_pos(); + + auto * inp_attn = build_attn_inp_kv(); + + const float kq_scale = hparams.f_attention_scale == 0.0f + ? 1.0f / sqrtf(float(n_embd_head)) + : hparams.f_attention_scale; + + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + for (int il = 0; il < n_layer; ++il) { + ggml_tensor * inpSA = inpL; + + cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "attn_norm", il); + + { + ggml_tensor * rope_factors = model.get_rope_factors(cparams, il); + + auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, + n_embd_head, n_head, n_head_kv, il); + + Qcur = ggml_rope_ext( + ctx0, Qcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + + Kcur = ggml_rope_ext( + ctx0, Kcur, inp_pos, rope_factors, + n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + + cb(Qcur, "Qcur", il); + cb(Kcur, "Kcur", il); + cb(Vcur, "Vcur", il); + + cur = build_attn(inp_attn, + model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s, + Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); + cb(cur, "attn_out", il); + } + + if (il == n_layer - 1 && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); + } + + ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); + cb(ffn_inp, "ffn_inp", il); + + cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "ffn_norm", il); + + cur = build_ffn(cur, + model.layers[il].ffn_up, model.layers[il].ffn_up_b, model.layers[il].ffn_up_s, + model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, model.layers[il].ffn_gate_s, + model.layers[il].ffn_down, model.layers[il].ffn_down_b, model.layers[il].ffn_down_s, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_inp); + cb(cur, "ffn_out", il); + + cur = build_cvec(cur, il); + cb(cur, "l_out", il); + + inpL = cur; + + if (n_loops > 1 && + ((il + 1) % n_phys) == 0 && + (il + 1) < n_layer && + !nb.skip_loop_final_norm) { + cur = build_norm(inpL, model.output_norm, NULL, LLM_NORM_RMS, il); + cb(cur, "loop_norm", il); + inpL = cur; + } + } + + cur = inpL; + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur, model.output_s); + cb(cur, "result_output", -1); + res->t_logits = cur; + + ggml_build_forward_expand(gf, cur); +} diff --git a/src/models/openai-moe.cpp b/src/models/openai-moe.cpp index 6d74f9c7e6..c91bae1c35 100644 --- a/src/models/openai-moe.cpp +++ b/src/models/openai-moe.cpp @@ -116,7 +116,7 @@ llama_model_openai_moe::graph::graph(const llama_model & model, const llm_graph_ cb(cur, "attn_out", il); } - if (il == n_layer - 1) { + if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked) { // skip computing output for unused tokens cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); @@ -154,6 +154,12 @@ llama_model_openai_moe::graph::graph(const llama_model & model, const llm_graph_ } cur = inpL; + res->t_h_nextn = cur; + + if (!cparams.embeddings_nextn_masked && inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index d8ffe43ae7..309dd43244 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -39,6 +39,7 @@ void llama_model_qwen35::load_arch_tensors(llama_model_loader & ml) { const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr); const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0; tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); @@ -97,25 +98,25 @@ void llama_model_qwen35::load_arch_tensors(llama_model_loader & ml) { auto & layer = layers[il]; // MTP block looks like a full-attention Qwen3.5 decoder block. - layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, 0); - layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, 0); + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, mtp_flags); + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, mtp_flags); - create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, 0); - layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, 0); - layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, 0); - layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, 0); + create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, mtp_flags); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, mtp_flags); + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, mtp_flags); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, mtp_flags); - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", il), {n_embd, n_ff}, 0); - layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", il), { n_ff, n_embd}, 0); - layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", il), {n_embd, n_ff}, 0); + layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", il), {n_embd, n_ff}, mtp_flags); + layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", il), { n_ff, n_embd}, mtp_flags); + layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", il), {n_embd, n_ff}, mtp_flags); // NextN-specific tensors that define the MTP block. - layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, 0); - layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, 0); - layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, 0); - layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); - layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); - layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, TENSOR_NOT_REQUIRED); + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, mtp_flags); + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, mtp_flags); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, mtp_flags); + layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED); + layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED); + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, mtp_flags|TENSOR_NOT_REQUIRED); }; for (int i = 0; i < n_layer; ++i) { diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp index 7b0876cbb0..38f2a57985 100644 --- a/src/models/qwen35moe.cpp +++ b/src/models/qwen35moe.cpp @@ -42,6 +42,7 @@ void llama_model_qwen35moe::load_arch_tensors(llama_model_loader & ml) { const bool mtp_only = (hparams.n_layer_nextn > 0) && (ml.get_weight("blk.0.attn_norm.weight") == nullptr); const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0; tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), { n_embd, n_vocab }, 0); @@ -113,32 +114,32 @@ void llama_model_qwen35moe::load_arch_tensors(llama_model_loader & ml) { const int64_t n_ff_shexp = hparams.n_ff_shexp ? hparams.n_ff_shexp : n_ff; // MTP block looks like a full-attention Qwen3.5 decoder block with MoE FFN. - layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, 0); - layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, 0); + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), { n_embd }, mtp_flags); + layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", il), { n_embd }, mtp_flags); - create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, 0); - layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, 0); - layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, 0); - layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, 0); + create_tensor_qkv(layer, il, n_embd, n_embd_head_k * n_head * 2, n_embd_k_gqa, n_embd_v_gqa, mtp_flags); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), { n_embd_head_k * n_head, n_embd }, mtp_flags); + layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", il), { n_embd_head_k }, mtp_flags); + layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", il), { n_embd_head_k }, mtp_flags); // Routed experts - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { n_ff_exp, n_embd, n_expert }, 0); - create_tensor_gate_up_exps(layer, il, n_embd, n_ff_exp, n_expert, 0); + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), { n_embd, n_expert }, mtp_flags); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), { n_ff_exp, n_embd, n_expert }, mtp_flags); + create_tensor_gate_up_exps(layer, il, n_embd, n_ff_exp, n_expert, mtp_flags); // Shared experts - layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", il), { n_embd }, 0); - layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, n_ff_shexp }, 0); - layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, n_ff_shexp }, 0); - layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { n_ff_shexp, n_embd }, 0); + layer.ffn_gate_inp_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP_SHEXP, "weight", il), { n_embd }, mtp_flags); + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), { n_embd, n_ff_shexp }, mtp_flags); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), { n_embd, n_ff_shexp }, mtp_flags); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), { n_ff_shexp, n_embd }, mtp_flags); // NextN-specific tensors that define the MTP block. - layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, 0); - layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, 0); - layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, 0); - layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); - layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, TENSOR_NOT_REQUIRED); - layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, TENSOR_NOT_REQUIRED); + layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", il), { 2 * n_embd, n_embd }, mtp_flags); + layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", il), { n_embd }, mtp_flags); + layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", il), { n_embd }, mtp_flags); + layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED); + layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED); + layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, mtp_flags|TENSOR_NOT_REQUIRED); }; for (int i = 0; i < n_layer; ++i) { diff --git a/src/models/step35.cpp b/src/models/step35.cpp index 9b7b18a367..5b1d902581 100644 --- a/src/models/step35.cpp +++ b/src/models/step35.cpp @@ -48,7 +48,11 @@ void llama_model_step35::load_arch_tensors(llama_model_loader & ml) { const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight"; const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr); const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0; - const int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0; + + if (!ml.load_mtp) { + mtp_flags |= TENSOR_SKIP; + } tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7a93b19a07..881e55c75a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -87,7 +87,7 @@ function(llama_build_and_test source) set(multiValueArgs ARGS) cmake_parse_arguments(LLAMA_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - set(TEST_SOURCES ${source} ${LLAMA_TEST_UNPARSED_ARGUMENTS} get-model.cpp) + set(TEST_SOURCES ${source} ${LLAMA_TEST_UNPARSED_ARGUMENTS}) if (NOT DEFINED LLAMA_TEST_LABEL) set(LLAMA_TEST_LABEL "main") @@ -148,7 +148,7 @@ if (LLAMA_LLGUIDANCE) llama_build_and_test(test-grammar-llguidance.cpp ARGS ${PROJECT_SOURCE_DIR}/models/ggml-vocab-llama-bpe.gguf) endif () -llama_build(test-recurrent-state-rollback.cpp get-model.cpp) +llama_build(test-recurrent-state-rollback.cpp) if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) # these tests are disabled on Windows because they use internal functions not exported with LLAMA_API (when building with shared libraries) @@ -278,6 +278,10 @@ set_tests_properties(test-state-restore-fragmented PROPERTIES FIXTURES_REQUIRED llama_build_and_test(test-save-load-state.cpp LABEL "model" ARGS -m "${MODEL_DEST}") set_tests_properties(test-save-load-state PROPERTIES FIXTURES_REQUIRED test-download-model) +if (APPLE) + llama_build(test-rset-release.cpp) +endif() + if (NOT GGML_BACKEND_DL) # these tests use the backends directly and cannot be built with dynamic loading llama_build_and_test(test-barrier.cpp) diff --git a/tests/get-model.cpp b/tests/get-model.cpp deleted file mode 100644 index 4edb685f0f..0000000000 --- a/tests/get-model.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include -#include -#include - -#include "get-model.h" - -char * get_model_or_exit(int argc, char *argv[]) { - char * model_path; - if (argc > 1) { - model_path = argv[1]; - - } else { - model_path = getenv("LLAMACPP_TEST_MODELFILE"); - if (!model_path || strlen(model_path) == 0) { - fprintf(stderr, "\033[33mWARNING: No model file provided. Skipping this test. Set LLAMACPP_TEST_MODELFILE= to silence this warning and run this test.\n\033[0m"); - exit(EXIT_SUCCESS); - } - } - - return model_path; -} diff --git a/tests/get-model.h b/tests/get-model.h deleted file mode 100644 index 81a3a0fefd..0000000000 --- a/tests/get-model.h +++ /dev/null @@ -1,2 +0,0 @@ -#pragma once -char * get_model_or_exit(int, char*[]); diff --git a/tests/snapshots/qwen3.5-27b.schema b/tests/snapshots/qwen3.6-27b.schema similarity index 100% rename from tests/snapshots/qwen3.5-27b.schema rename to tests/snapshots/qwen3.6-27b.schema diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cpp index 000ecd9aaa..1d3584f903 100644 --- a/tests/test-arg-parser.cpp +++ b/tests/test-arg-parser.cpp @@ -143,6 +143,10 @@ static void test(void) { assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK); + argv = {"binary_name", "-lm", "mmap+mlock"}; + assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); + assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK); + argv = {"binary_name", "-lm", "dio"}; assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO); @@ -187,6 +191,11 @@ static void test(void) { assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK); + setenv("LLAMA_ARG_LOAD_MODE", "mmap+mlock", true); + argv = {"binary_name"}; + assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); + assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK); + setenv("LLAMA_ARG_LOAD_MODE", "dio", true); argv = {"binary_name"}; assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); diff --git a/tests/test-autorelease.cpp b/tests/test-autorelease.cpp index ca87c56a8f..370428809f 100644 --- a/tests/test-autorelease.cpp +++ b/tests/test-autorelease.cpp @@ -1,15 +1,13 @@ // ref: https://github.com/ggml-org/llama.cpp/issues/4952#issuecomment-1892864763 -#include -#include #include #include "llama.h" -#include "get-model.h" +#include "common.h" // This creates a new context inside a pthread and then tries to exit cleanly. int main(int argc, char ** argv) { - auto * model_path = get_model_or_exit(argc, argv); + auto * model_path = common_get_model_or_exit(argc, argv); std::thread([&model_path]() { llama_backend_init(); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index e7cd6d0cb6..4098acaaf9 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1350,18 +1350,22 @@ struct test_case { // check if the backends support the ops bool supported = true; + std::string unsupported_str; for (ggml_backend_t backend : {backend1, backend2}) { for (ggml_tensor * t = ggml_get_first_tensor(ctx.get()); t != NULL; t = ggml_get_next_tensor(ctx.get(), t)) { if (!ggml_backend_supports_op(backend, t)) { supported = false; - break; + if (unsupported_str.empty()) { + unsupported_str = std::string(ggml_backend_name(backend)); + } else { + unsupported_str += ", " + std::string(ggml_backend_name(backend)); + } } } } if (!supported) { - // Create test result for unsupported operation - test_result result(ggml_backend_name(backend1), current_op_name, vars(), "test", + test_result result(unsupported_str, current_op_name, vars(), "test", false, false, "not supported"); print_test_result_locked(output_printer, result); @@ -4000,7 +4004,7 @@ struct test_ssm_scan : public test_case { test_ssm_scan(ggml_type type = GGML_TYPE_F32, int64_t d_state = 32, - int64_t head_dim = 1, // non-zero for Mamba-2 + int64_t head_dim = 1, // 1 = Mamba-1; > 1 = Mamba-2 (scalar A per head) int64_t n_head = 32, int64_t n_group = 1, int64_t n_seq_tokens = 32, @@ -4008,6 +4012,11 @@ struct test_ssm_scan : public test_case { bool xbc_overlap = false) : type(type), d_state(d_state), head_dim(head_dim), n_head(n_head), n_group(n_group), n_seq_tokens(n_seq_tokens), n_seqs(n_seqs), xbc_overlap(xbc_overlap) {} + double max_nmse_err() override { + // SSD path (head_dim > 1) uses FP16 intermediates (M matrix, X_dt); Mamba-1 is pure FP32. + return (head_dim > 1) ? 2e-7 : 1e-7; + } + ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * s = ggml_new_tensor_4d(ctx, type, d_state, head_dim, n_head, n_seqs); ggml_tensor * dt = ggml_new_tensor_3d(ctx, type, n_head, n_seq_tokens, n_seqs); @@ -4034,14 +4043,14 @@ struct test_ssm_scan : public test_case { return out; } - // similar to test_mul_mat_id + void initialize_tensors(ggml_context * ctx) override { std::random_device rd; std::default_random_engine rng(rd()); for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { if (t->type == GGML_TYPE_I32) { if (ggml_is_view_op(t->op)) { continue; } - // ids + // ids: permutation of [0..n_seqs) for (int64_t r = 0; r < ggml_nrows(t); r++) { std::vector data(t->ne[0]); for (int i = 0; i < t->ne[0]; i++) { @@ -4050,6 +4059,11 @@ struct test_ssm_scan : public test_case { std::shuffle(data.begin(), data.end(), rng); ggml_backend_tensor_set(t, data.data(), r * t->nb[1], t->ne[0] * sizeof(int32_t)); } + } else if (ggml_is_view_op(t->op)) { + continue; + } else if (t->ne[1] == n_head && t->ne[2] == 1) { + // A {1 or d_state, n_head}: negative decay (2-D tensor, ne[2]==1 distinguishes from 3-D/4-D tensors) + init_tensor_uniform(t, -1.0f, -0.5f); } else { init_tensor_uniform(t); } @@ -7999,6 +8013,7 @@ static const ggml_type other_types[] = { GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, GGML_TYPE_Q1_0, + GGML_TYPE_Q2_0, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K, @@ -8177,9 +8192,9 @@ static std::vector> make_test_cases_eval() { for (ggml_type type_input : {GGML_TYPE_F32}) { for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) { - for (int k0 : {1, 3}) { - for (int s0 : {1, 2}) { - for (int p0 : {0, 1}) { + for (int k0 : {1, 2, 3}) { + for (int s0 : {1, 2, 3}) { + for (int p0 : {0, 1, 2, 3}) { test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 10, 3, 2, 1 }, k0, s0, p0)); test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 11, 1, 3, 2 }, k0, s0, p0)); test_cases.emplace_back(new test_pool1d(pool_type, type_input, { 128, 2, 1, 3 }, k0, s0, p0)); @@ -8314,7 +8329,11 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_conv_2d( { act_case[iwh_idx], act_case[iwh_idx], act_case[Cin_idx], act_case[B_idx] }, { act_case[kwh_idx], act_case[kwh_idx], act_case[Cin_idx], act_case[Cout_idx] }, - kernel_type, 1, 1, 0, 0, 1, 1, false)); + kernel_type, 1, 1, 0, 0, 1, 1, false)); // bool cwhn = false + test_cases.emplace_back(new test_conv_2d( + { act_case[iwh_idx], act_case[iwh_idx], act_case[Cin_idx], act_case[B_idx] }, + { act_case[kwh_idx], act_case[kwh_idx], act_case[Cin_idx], act_case[Cout_idx] }, + kernel_type, 1, 1, 0, 0, 1, 1, true)); // bool cwhn = true } } #endif @@ -8343,7 +8362,9 @@ static std::vector> make_test_cases_eval() { calc_conv_output_size(H, KH, s1, p1, d1) > 0) { for (auto kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { test_cases.emplace_back(new test_conv_2d( - { W, H, Cin, 2 }, { KW, KH, Cin, Cout }, kernel_type, s0, s1, p0, p1, d0, d1, false)); + { W, H, Cin, 2 }, { KW, KH, Cin, Cout }, kernel_type, s0, s1, p0, p1, d0, d1, false)); // bool cwhn = false + test_cases.emplace_back(new test_conv_2d( + { W, H, Cin, 2 }, { KW, KH, Cin, Cout }, kernel_type, s0, s1, p0, p1, d0, d1, true)); // bool cwhn = true } } } @@ -8355,7 +8376,8 @@ static std::vector> make_test_cases_eval() { } } for (auto kernel_type : {GGML_TYPE_F32, GGML_TYPE_F16}) { - test_cases.emplace_back(new test_conv_2d({ 256, 256, 192, 1 }, { 3, 3, 192, 96 }, kernel_type, 1, 1, 1, 1, 1, 1, false)); + test_cases.emplace_back(new test_conv_2d({ 256, 256, 192, 1 }, { 3, 3, 192, 96 }, kernel_type, 1, 1, 1, 1, 1, 1, false)); // bool cwhn = false + test_cases.emplace_back(new test_conv_2d({ 256, 256, 192, 1 }, { 3, 3, 192, 96 }, kernel_type, 1, 1, 1, 1, 1, 1, true)); // bool cwhn = true } // sycl backend will limit task global_range < MAX_INT @@ -8770,6 +8792,9 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 16, 2, 32, 4)); // Mamba-2 test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 256, 64, 8, 2, 32, 4)); // Falcon-H1 test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 128, 4, 4, 16, 2, true)); // x/B/C overlap + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 80, 128, 1, 256, 1)); // Nemotron-9B SSD path + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 80, 128, 1, 512, 1)); // Nemotron-9B SSD multi-chunk (2 aligned chunks) + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 80, 8, 300, 2)); // Mamba-2 SSD multi-chunk (partial 2nd chunk, 2 seqs) test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 1, 1)); test_cases.emplace_back(new test_rwkv_wkv6(GGML_TYPE_F32, 32, 64, 32, 1)); @@ -8793,6 +8818,9 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 512, 1, 512)); test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 32, 128)); test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 4, 128, {2, 3})); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 256, 512, 256)); // many rows + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 32, 1, 32)); // too small (N<64) + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 1024, 1, 1024)); // too big (N>512) #if 0 // > 4GB A matrix. Too slow to be enabled by default. @@ -8817,6 +8845,10 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1})); + // m == 1, with n on both sides of MMVF_MAX_BATCH_SIZE (8): mmvf below, operand swap above + for (int64_t n : {1, 7, 8, 9, 16, 128, 512}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1, n, 2048, {1, 1}, {1, 1})); + } #if 0 { @@ -9481,6 +9513,18 @@ static std::vector> make_test_cases_eval() { } } + // prefill-shaped cases with long KV (nb >= 32, kv >= 1024): covers the + // XMX/GEMM-accelerated SYCL FA path which only activates for these shapes. + for (int kv : { 1024, 2048, }) { + for (int hs : { 64, 128, 256, }) { + for (int nb : { 32, 64, }) { + for (ggml_type type_KV : { GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0, }) { + test_cases.emplace_back(new test_flash_attn_ext(hs, hs, 8, {4, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, type_KV, type_KV)); + } + } + } + } + for (int hsk : { 40, 64, 72, 80, 96, 128, 192, 256, 320, 512, 576 }) { for (int hsv : { 40, 64, 72, 80, 96, 128, 192, 256, 512 }) { if (hsk != 192 && hsk != 320 && hsk != 576 && hsk != hsv) continue; @@ -9543,6 +9587,19 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q1_0, GGML_TYPE_Q4_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q1_0)); test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q1_0, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 4, {1, 1}, 96, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_Q2_0)); + test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_Q4_0)); + test_cases.emplace_back(new test_flash_attn_ext(64, 128, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q2_0)); + test_cases.emplace_back(new test_flash_attn_ext(128, 64, 4, {1, 1}, 64, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q2_0, GGML_TYPE_F16)); + + // large-KV F16 cases (Qwen3.6-27B geometry and a llama-class control): the upstream matrix + // stops at kv=1024, blind to long-context FA bugs (e.g. the oneDNN SDPA ordering race on BMG). + for (int64_t kv : { 4096, 16384 }) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, 512, true, false, 0, 0, + GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(128, 128, 8, {4, 1}, kv, 512, true, false, 0, 0, + GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + } test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, { 10, 5, 4, 3})); test_cases.emplace_back(new test_cross_entropy_loss (GGML_TYPE_F32, {30000, 1, 1, 1})); @@ -9719,7 +9776,11 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_conv_2d( { act_case[iwh_idx], act_case[iwh_idx], act_case[Cin_idx], act_case[B_idx] }, { act_case[kwh_idx], act_case[kwh_idx], act_case[Cin_idx], act_case[Cout_idx] }, - kernel_type, 1, 1, 0, 0, 1, 1, false)); + kernel_type, 1, 1, 0, 0, 1, 1, false)); // bool cwhn = false + test_cases.emplace_back(new test_conv_2d( + { act_case[iwh_idx], act_case[iwh_idx], act_case[Cin_idx], act_case[B_idx] }, + { act_case[kwh_idx], act_case[kwh_idx], act_case[Cin_idx], act_case[Cout_idx] }, + kernel_type, 1, 1, 0, 0, 1, 1, true)); // bool cwhn = true } } @@ -9803,6 +9864,10 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 64, 1, 64)); test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 256, 1, 256)); test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 32, 128)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 64, 2048, 64)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 128, 2048, 128)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 256, 2048, 256)); + test_cases.emplace_back(new test_mul_mat_hadamard(GGML_TYPE_F32, GGML_TYPE_F32, 512, 2048, 512)); test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 64, 64, 4, 4 }, { 32, 64, 4, 4 })); test_cases.emplace_back(new test_solve_tri(GGML_TYPE_F32, { 128, 128, 4, 2 }, { 32, 128, 4, 2 })); @@ -9974,6 +10039,8 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_ssm_conv_bias_silu(GGML_TYPE_F32, {4, 3328, 1, 1}, {4, 3328, 1, 1}, true)); // generate test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 48, 1, 512, 1)); // prefill test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 64, 48, 1, 1, 1)); // generate + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 80, 128, 1, 512, 1)); // Nemotron-9B prefill + test_cases.emplace_back(new test_ssm_scan(GGML_TYPE_F32, 128, 80, 128, 1, 1, 1)); // Nemotron-9B generate // acc test_cases.emplace_back(new test_acc(GGML_TYPE_F32, {256, 17, 1, 1}, {256, 16, 1, 1}, -1)); diff --git a/tests/test-backend-sampler.cpp b/tests/test-backend-sampler.cpp index 61ddf91fea..c24076e313 100644 --- a/tests/test-backend-sampler.cpp +++ b/tests/test-backend-sampler.cpp @@ -1,7 +1,6 @@ #include "ggml.h" #include "llama.h" #include "llama-cpp.h" -#include "get-model.h" #include "common.h" #ifdef NDEBUG @@ -1136,7 +1135,7 @@ int main(int argc, char ** argv) { test_args args = parse_cli(argc, argv); if (args.model.empty()) { - args.model = get_model_or_exit(1, argv); + args.model = common_get_model_or_exit(1, argv); } { diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index 4dd00efddf..01b07953a6 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -730,6 +730,71 @@ static common_chat_tool imaginary_number_tool{ })", }; +static common_chat_tool nested_args_tool{ + /* .name = */ "nested_args", + /* .description = */ "Tool with nested array arguments", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "tags": { + "type": "array", + "items": { "type": "string" } + }, + "entries": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { "type": "integer" }, + "label": { "type": "string" } + }, + "required": ["id", "label"] + } + } + }, + "required": ["tags", "entries"] + })", +}; + +static common_chat_tool union_args_tool{ + /* .name = */ "union_args", + /* .description = */ "Tool with union arguments", + /* .parameters = */ R"({ + "type": "object", + "properties": { + "filter": { + "anyOf": [ + { "type": "array", "items": { "type": "string" } }, + { + "type": "object", + "properties": { + "field": { "type": "string" }, + "op": { "type": "string" } + }, + "required": ["field", "op"] + } + ] + }, + "label": { + "oneOf": [ + { "type": "string" }, + { "type": "object", "properties": { "text": { "type": "string" } } } + ] + }, + "limit": { + "oneOf": [ + { "type": "integer" }, + { + "type": "object", + "properties": { "max": { "type": "integer" } }, + "required": ["max"] + } + ] + } + } + })", +}; + static common_chat_tool nullable_string_tool{ /* .name = */ "set_nullable_str", /* .description = */ "Set a nullable string value", @@ -4850,6 +4915,370 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .run(); } + // MiniMax-M3 tests - namespaced XML invoke format, the parameter name is the tag + // Format: + // ]<]minimax[>[ + // ]<]minimax[>[]<]minimax[>[Tokyo]<]minimax[>[]<]minimax[>[ + // ]<]minimax[>[ + // Reasoning uses .... The generation prompt is only "]~b]ai\n", so the model + // opens the thinking block itself; a turn without reasoning is prefixed with a bare . + { + auto tst = peg_tester("models/templates/MiniMax-M3.jinja", detailed_debug); + + // Content only (bare prefix) + tst.test("Hello, world!\nWhat's up?") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .expect(message_assist) + .expect_reconstruction() + .run(); + + // Thinking + content + tst.test("I'm\nthinkingHello, world!\nWhat's up?") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .expect(message_assist_thoughts) + .expect_reconstruction() + .run(); + + // Thinking + tool call (single, string param) + tst.test( + "Let me check the time" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[Tokyo]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ get_time_tool }) + .expect(message_with_tool_calls_and_reasoning("get_time", R"({"city": "Tokyo"})", "Let me check the time")) + .expect_reconstruction() + .run(); + + // Tool call without reasoning, integer param + tst.test( + "" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[1]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ special_function_tool }) + .expect(message_assist_call) + .expect_reconstruction() + .run(); + + // Tool call with no parameters + tst.test( + "Let's call a tool:" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ empty_args_tool }) + .expect(message_with_reasoning_and_tool_call("Let's call a tool:", "empty_args", "{}")) + .expect_reconstruction() + .run(); + + // Multiple parallel tool calls in one block + tst.test( + "Calling both" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[Paris]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[Paris]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .parallel_tool_calls(true) + .tools({ get_time_tool, get_weather_tool }) + .expect(message_with_reasoning_content_and_multiple_tool_calls( + "Calling both", "", + { { "get_time", R"({"city": "Paris"})" }, { "get_weather", R"({"city": "Paris"})" } })) + .expect_reconstruction() + .run(); + + // Content before the tool call block + tst.test( + "Thinking about it" + "Let me call the function." + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[1]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ special_function_tool }) + .expect_reasoning("Thinking about it") + .expect_content("Let me call the function.") + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + }) + .expect_reconstruction() + .run(); + + // Negative number + tst.test( + "Test negative" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[-14]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ magic_int_tool }) + .expect_reasoning("Test negative") + .expect_tool_calls({ + { "magic_int", R"({"ref": -14})", {} }, + }) + .expect_reconstruction() + .run(); + + // Decimal number + tst.test( + "Test decimal" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[3.14]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ amount_tool }) + .expect_reasoning("Test decimal") + .expect_tool_calls({ + { "amount", R"({"orig": 3.14})", {} }, + }) + .expect_reconstruction() + .run(); + + // Boolean + tst.test( + "Test boolean" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[true]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ toggle_tool }) + .expect_reasoning("Test boolean") + .expect_tool_calls({ + { "toggle", R"({"enabled": true})", {} }, + }) + .expect_reconstruction() + .run(); + + // Multiple params of mixed types (required int first, then optional string) + tst.test( + "Multi-arg call" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[42]<]minimax[>[" + "]<]minimax[>[foo bar]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ magic_int_tool }) + .expect_reasoning("Multi-arg call") + .expect_tool_calls({ + { "magic_int", R"({"ref": 42, "name": "foo bar"})", {} }, + }) + .expect_reconstruction() + .run(); + + // Nested object param, expanded into one element per key + tst.test( + "Nested object" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[1.5]<]minimax[>[" + "]<]minimax[>[-2.5]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ imaginary_number_tool }) + .expect_reasoning("Nested object") + .expect_tool_calls({ + { "imaginary_number", R"({"number": {"real": 1.5, "imaginary": -2.5}})", {} }, + }) + .expect_reconstruction() + .run(); + + // Array params, expanded into elements (of scalars and of objects) + tst.test( + "Nested arrays" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[alpha]<]minimax[>[" + "]<]minimax[>[beta]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[1]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ nested_args_tool }) + .expect_reasoning("Nested arrays") + .expect_tool_calls({ + { "nested_args", R"({"tags": ["alpha", "beta"], "entries": [{"id": 1, "label": "one"}]})", {} }, + }) + .expect_reconstruction() + .run(); + + // Union params (anyOf/oneOf), expanded as a choice of the alternatives + tst.test( + "Union array" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[alpha]<]minimax[>[" + "]<]minimax[>[beta]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ union_args_tool }) + .expect_reasoning("Union array") + .expect_tool_calls({ + { "union_args", R"({"filter": ["alpha", "beta"]})", {} }, + }) + .expect_reconstruction() + .run(); + + // oneOf between a scalar and an object + tst.test( + "Union scalar" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[5]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ union_args_tool }) + .expect_reasoning("Union scalar") + .expect_tool_calls({ + { "union_args", R"({"limit": 5})", {} }, + }) + .expect_reconstruction() + .run(); + + tst.test( + "Union nested" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[10]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ union_args_tool }) + .expect_reasoning("Union nested") + .expect_tool_calls({ + { "union_args", R"({"limit": {"max": 10}})", {} }, + }) + .expect_reconstruction() + .run(); + + // A union with a string alternative is a string + tst.test( + "Union string" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ union_args_tool }) + .expect_reasoning("Union string") + .expect_tool_calls({ + { "union_args", R"({"label": "hi"})", {} }, + }) + .expect_reconstruction() + .run(); + + // ... even when the value looks structured + tst.test( + "Union string" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ union_args_tool }) + .expect_reasoning("Union string") + .expect_tool_calls({ + { "union_args", R"({"label": "]<]minimax[>[hi]<]minimax[>["})", {} }, + }) + .expect_reconstruction() + .run(); + + // Edge case: empty reasoning followed by a tool call + tst.test( + "" + "]<]minimax[>[\n" + "]<]minimax[>[" + "]<]minimax[>[XYZCITY]<]minimax[>[" + "]<]minimax[>[\n" + "]<]minimax[>[") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ get_time_tool }) + .expect(message_with_tool_calls("get_time", R"({"city": "XYZCITY"})")) + .run(); + + // Continuation tests + tst.test("world!\nWhat's up?") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .enable_thinking(true) + .messages({ message_user, message_assist_prefill_content }) + .add_generation_prompt(false) + .continue_final_message(COMMON_CHAT_CONTINUATION_CONTENT) + .expect_reasoning("I'm thinking") + .expect_content("Hello, world!\nWhat's up?") + .run(); + + tst.test(" thinkingHello, world!\nWhat's up?") + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .enable_thinking(true) + .messages({ message_user, message_assist_prefill_reasoning }) + .add_generation_prompt(false) + .continue_final_message(COMMON_CHAT_CONTINUATION_REASONING) + .expect_reasoning("I'm thinking") + .expect_content("Hello, world!\nWhat's up?") + .run(); + } + // NVIDIA-Nemotron-Nano-v2 tests - ... format // Format: [{"name": "func", "arguments": {...}}] { diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index d02e65c9ea..a1ed2a76f8 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -428,9 +428,9 @@ static bool arch_supported(const llm_arch arch) { return false; } - // FIXME some models are segfaulting with WebGPU: + // FIXME: these hit scheduler/view-backed-output issues with WebGPU on CI. #ifdef GGML_USE_WEBGPU - if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_KIMI_LINEAR) { + if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA) { return false; } #endif // GGML_USE_WEBGPU @@ -600,9 +600,6 @@ static int test_backends(const llm_arch target_arch, const size_t seed, const gg std::string status_roundtrip = "\033[1;33mSKIP\033[0m"; char nmse_str[12] = {0}; bool skip = !arch_supported(arch) || (dc.split_mode == LLAMA_SPLIT_MODE_TENSOR && dc.devs.empty()); -#if defined(GGML_USE_WEBGPU) - skip = true; // FIXME -#endif // GGML_USE_WEBGPU if (!skip) { if (logits_cpu.empty()) { model_and_ctx_cpu = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {}, LLAMA_SPLIT_MODE_LAYER, encode); diff --git a/tests/test-model-load-cancel.cpp b/tests/test-model-load-cancel.cpp index ecc3022711..f8139b26d7 100644 --- a/tests/test-model-load-cancel.cpp +++ b/tests/test-model-load-cancel.cpp @@ -1,10 +1,10 @@ #include "llama.h" -#include "get-model.h" +#include "common.h" #include int main(int argc, char *argv[] ) { - auto * model_path = get_model_or_exit(argc, argv); + auto * model_path = common_get_model_or_exit(argc, argv); auto * file = fopen(model_path, "r"); if (file == nullptr) { fprintf(stderr, "no model at '%s' found\n", model_path); diff --git a/tests/test-quant-type-selection.cpp b/tests/test-quant-type-selection.cpp index 3c8983360e..9a5f5e53e1 100644 --- a/tests/test-quant-type-selection.cpp +++ b/tests/test-quant-type-selection.cpp @@ -216,18 +216,18 @@ static std::string snapshot_file_from_name(const std::string & name) { } static const remote_model_spec model_specs[] = { - { "ggml-org/Qwen3-0.6B-GGUF", "Q8_0" }, - { "ggml-org/GLM-4.6V-GGUF", "Q8_0" }, - { "ggml-org/Step-3.5-Flash-GGUF", "Q4_K" }, - { "ggml-org/Qwen3-Coder-Next-GGUF", "Q8_0" }, - { "ggml-org/Qwen3-14B-GGUF", "Q8_0" }, - { "ggml-org/Nemotron-Nano-3-30B-A3B-GGUF", "Q8_0" }, - { "ggml-org/gpt-oss-120b-GGUF", "mxfp4" }, - { "ggml-org/gemma-3-4b-it-GGUF", "Q8_0" }, - { "bartowski/Meta-Llama-3.1-70B-Instruct-GGUF", "Q4_K_M" }, - { "bartowski/deepseek-ai_DeepSeek-V3.1-GGUF", "IQ1_M" }, - { "bartowski/Qwen_Qwen3.5-397B-A17B-GGUF", "IQ1_S" }, // TODO: swap with ggml-org if/when it's released - { "bartowski/Qwen_Qwen3.5-27B-GGUF", "Q8_0" }, // TODO: swap with ggml-org if/when it's released + { "ggml-org/Qwen3-0.6B-GGUF", "Q8_0" }, + { "ggml-org/GLM-4.6V-GGUF", "Q8_0" }, + { "ggml-org/Step-3.5-Flash-GGUF", "Q4_K" }, + { "ggml-org/Qwen3-Coder-Next-GGUF", "Q8_0" }, + { "ggml-org/Qwen3-14B-GGUF", "Q8_0" }, + { "ggml-org/NVIDIA-Nemotron-Nano-3-30B-A3B-GGUF", "Q8_0" }, + { "ggml-org/gpt-oss-120b-GGUF", "mxfp4" }, + { "ggml-org/gemma-3-4b-it-GGUF", "Q8_0" }, + { "bartowski/Meta-Llama-3.1-70B-Instruct-GGUF", "Q4_K_M" }, + { "bartowski/deepseek-ai_DeepSeek-V3.1-GGUF", "IQ1_M" }, + //{ "bartowski/Qwen_Qwen3.5-397B-A17B-GGUF", "IQ1_S" }, // TODO: swap with ggml-org if/when it's released + { "ggml-org/Qwen3.6-27B-GGUF", "Q8_0" }, }; static const int n_model_specs = (int) (sizeof(model_specs) / sizeof(model_specs[0])); diff --git a/tests/test-rset-release.cpp b/tests/test-rset-release.cpp new file mode 100644 index 0000000000..c60801c115 --- /dev/null +++ b/tests/test-rset-release.cpp @@ -0,0 +1,53 @@ +// ref: https://github.com/ggml-org/llama.cpp/issues/25937 +// only works reliably when run with a large model that occupies 3GB+ of wired memory +// thus, this test is not run by default +// example model to run with: google/gemma-4-E4B-it-qat-q4_0-gguf + +#include "llama.h" +#include "common.h" + +#include +#include +#include +#include + +static uint64_t wired_memory() { + vm_statistics64_data_t vmstat; + mach_msg_type_number_t count = HOST_VM_INFO64_COUNT; + if (host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t)&vmstat, &count) != KERN_SUCCESS) { + return UINT64_MAX; + } + return static_cast(vmstat.wire_count) * vm_kernel_page_size; +} + +int main(int argc, char ** argv) { + auto * model_path = common_get_model_or_exit(argc, argv); + + llama_backend_init(); + + const uint64_t wired_initial = wired_memory(); + + llama_model_params params = llama_model_default_params(); + params.load_mode = LLAMA_LOAD_MODE_NONE; + struct llama_model* model = llama_model_load_from_file(model_path, params); + + const uint64_t wired_loaded = wired_memory(); + const uint64_t wired_delta = wired_loaded - wired_initial; + // system memory fluctuates, so we need to allocate enough to reliably detect the release + GGML_ASSERT(wired_delta > 2'000'000'000); // 2GB + + llama_model_free(model); + + const uint64_t t_start_ms = ggml_time_ms(); + + // expect most of the allocated memory to be released within 10 seconds + // we allow for some tolerance due to system-wide memory fluctuations + while (wired_memory() > wired_loaded - 0.75 * wired_delta) { + GGML_ASSERT(ggml_time_ms() - t_start_ms < 10'000); + usleep(100'000); // 100ms + } + + llama_backend_free(); + + return 0; +} diff --git a/tests/test-save-load-state.cpp b/tests/test-save-load-state.cpp index bbb025617f..6e93ce6fb8 100644 --- a/tests/test-save-load-state.cpp +++ b/tests/test-save-load-state.cpp @@ -44,8 +44,6 @@ static llama_tokens generate_tokens(llama_context * ctx, llama_sampler * smpl, i n_past++; } - llama_synchronize(ctx); - return result; } diff --git a/tools/cli/README.md b/tools/cli/README.md index 6ee447b073..bcddd05702 100644 --- a/tools/cli/README.md +++ b/tools/cli/README.md @@ -55,10 +55,10 @@ | `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) | | `-np, --parallel N` | number of parallel sequences to decode (default: 1)
(env: LLAMA_ARG_N_PARALLEL) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) | -| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) | +| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) | | `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)
(env: LLAMA_ARG_MMAP) | | `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available
(env: LLAMA_ARG_DIO) | -| `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available

(env: LLAMA_ARG_LOAD_MODE) | +| `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available

(env: LLAMA_ARG_LOAD_MODE) | | `--numa TYPE` | attempt optimizations that help on some NUMA systems
- distribute: spread execution evenly over all nodes
- isolate: only spawn threads on CPUs on the node that execution started on
- numactl: use the CPU map provided by numactl
if run without this previously, it is recommended to drop the system page cache before using this
see https://github.com/ggml-org/llama.cpp/issues/1437
(env: LLAMA_ARG_NUMA) | | `-dev, --device ` | comma-separated list of devices to use for offloading (none = don't offload)
use --list-devices to see a list of available devices
(env: LLAMA_ARG_DEVICE) | | `--list-devices` | print list of available devices and exit | @@ -203,7 +203,7 @@ | `--spec-draft-device, -devd, --device-draft ` | comma-separated list of devices to use for offloading the draft model (none = don't offload)
use --list-devices to see a list of available devices | | `--spec-draft-ngl, -ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)
(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) | | `--spec-draft-model, -md, --model-draft FNAME` | draft model for speculative decoding (default: unused)
(env: LLAMA_ARG_SPEC_DRAFT_MODEL) | -| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)

(env: LLAMA_ARG_SPEC_TYPE) | +| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,draft-dspark,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)

(env: LLAMA_ARG_SPEC_TYPE) | | `--spec-ngram-mod-n-min N` | minimum number of ngram tokens to use for ngram-based speculative decoding (default: 48) | | `--spec-ngram-mod-n-max N` | maximum number of ngram tokens to use for ngram-based speculative decoding (default: 64) | | `--spec-ngram-mod-n-match N` | ngram-mod lookup length (default: 24) | diff --git a/tools/completion/README.md b/tools/completion/README.md index 17f7cd7659..bce71d68d9 100644 --- a/tools/completion/README.md +++ b/tools/completion/README.md @@ -138,10 +138,10 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1 | `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) | | `-np, --parallel N` | number of parallel sequences to decode (default: 1)
(env: LLAMA_ARG_N_PARALLEL) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) | -| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) | +| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) | | `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)
(env: LLAMA_ARG_MMAP) | | `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available
(env: LLAMA_ARG_DIO) | -| `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available

(env: LLAMA_ARG_LOAD_MODE) | +| `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available

(env: LLAMA_ARG_LOAD_MODE) | | `--numa TYPE` | attempt optimizations that help on some NUMA systems
- distribute: spread execution evenly over all nodes
- isolate: only spawn threads on CPUs on the node that execution started on
- numactl: use the CPU map provided by numactl
if run without this previously, it is recommended to drop the system page cache before using this
see https://github.com/ggml-org/llama.cpp/issues/1437
(env: LLAMA_ARG_NUMA) | | `-dev, --device ` | comma-separated list of devices to use for offloading (none = don't offload)
use --list-devices to see a list of available devices
(env: LLAMA_ARG_DEVICE) | | `--list-devices` | print list of available devices and exit | diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp index 29ad352d0c..c17a27b540 100644 --- a/tools/llama-bench/llama-bench.cpp +++ b/tools/llama-bench/llama-bench.cpp @@ -429,45 +429,45 @@ static void print_usage(int /* argc */, char ** argv) { } printf("\n"); printf("test parameters:\n"); - printf(" -m, --model (default: %s)\n", join(cmd_params_defaults.model, ",").c_str()); - printf(" -hf, -hfr, --hf-repo /[:quant] Hugging Face model repository; quant is optional, case-insensitive\n"); - printf(" default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n"); - printf(" example: ggml-org/GLM-4.7-Flash-GGUF:Q4_K_M\n"); - printf(" (default: unused)\n"); - printf(" -hff, --hf-file Hugging Face model file. If specified, it will override the quant in --hf-repo\n"); - printf(" (default: unused)\n"); - printf(" -hft, --hf-token Hugging Face access token\n"); - printf(" (default: value from HF_TOKEN environment variable)\n"); - printf(" --offline Offline mode: forces use of cache, prevents network access\n"); - printf(" (default: disabled)\n"); - printf(" -p, --n-prompt (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str()); - printf(" -n, --n-gen (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str()); - printf(" -pg (default: %s)\n", join(transform_to_str(cmd_params_defaults.n_pg, pair_str), ",").c_str()); - printf(" -d, --n-depth (default: %s)\n", join(cmd_params_defaults.n_depth, ",").c_str()); - printf(" -b, --batch-size (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str()); - printf(" -ub, --ubatch-size (default: %s)\n", join(cmd_params_defaults.n_ubatch, ",").c_str()); - printf(" -ctk, --cache-type-k (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_k, ggml_type_name), ",").c_str()); - printf(" -ctv, --cache-type-v (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_v, ggml_type_name), ",").c_str()); - printf(" -t, --threads (default: %s)\n", join(cmd_params_defaults.n_threads, ",").c_str()); - printf(" -C, --cpu-mask (default: %s)\n", join(cmd_params_defaults.cpu_mask, ",").c_str()); - printf(" --cpu-strict <0|1> (default: %s)\n", join(cmd_params_defaults.cpu_strict, ",").c_str()); - printf(" --poll <0...100> (default: %s)\n", join(cmd_params_defaults.poll, ",").c_str()); - printf(" -ngl, --n-gpu-layers (default: %s)\n", join(cmd_params_defaults.n_gpu_layers, ",").c_str()); - printf(" -ncmoe, --n-cpu-moe (default: %s)\n", join(cmd_params_defaults.n_cpu_moe, ",").c_str()); - printf(" -sm, --split-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str()); - printf(" -mg, --main-gpu (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str()); - printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str()); - printf(" -fa, --flash-attn (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str()); - printf(" -dev, --device (default: auto)\n"); - printf(" -lm, --load-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str()); - printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n"); - printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n"); - printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str()); - printf(" -ts, --tensor-split (default: 0)\n"); + printf(" -m, --model (default: %s)\n", join(cmd_params_defaults.model, ",").c_str()); + printf(" -hf, -hfr, --hf-repo /[:quant] Hugging Face model repository; quant is optional, case-insensitive\n"); + printf(" default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n"); + printf(" example: ggml-org/GLM-4.7-Flash-GGUF:Q4_K_M\n"); + printf(" (default: unused)\n"); + printf(" -hff, --hf-file Hugging Face model file. If specified, it will override the quant in --hf-repo\n"); + printf(" (default: unused)\n"); + printf(" -hft, --hf-token Hugging Face access token\n"); + printf(" (default: value from HF_TOKEN environment variable)\n"); + printf(" --offline Offline mode: forces use of cache, prevents network access\n"); + printf(" (default: disabled)\n"); + printf(" -p, --n-prompt (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str()); + printf(" -n, --n-gen (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str()); + printf(" -pg (default: %s)\n", join(transform_to_str(cmd_params_defaults.n_pg, pair_str), ",").c_str()); + printf(" -d, --n-depth (default: %s)\n", join(cmd_params_defaults.n_depth, ",").c_str()); + printf(" -b, --batch-size (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str()); + printf(" -ub, --ubatch-size (default: %s)\n", join(cmd_params_defaults.n_ubatch, ",").c_str()); + printf(" -ctk, --cache-type-k (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_k, ggml_type_name), ",").c_str()); + printf(" -ctv, --cache-type-v (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_v, ggml_type_name), ",").c_str()); + printf(" -t, --threads (default: %s)\n", join(cmd_params_defaults.n_threads, ",").c_str()); + printf(" -C, --cpu-mask (default: %s)\n", join(cmd_params_defaults.cpu_mask, ",").c_str()); + printf(" --cpu-strict <0|1> (default: %s)\n", join(cmd_params_defaults.cpu_strict, ",").c_str()); + printf(" --poll <0...100> (default: %s)\n", join(cmd_params_defaults.poll, ",").c_str()); + printf(" -ngl, --n-gpu-layers (default: %s)\n", join(cmd_params_defaults.n_gpu_layers, ",").c_str()); + printf(" -ncmoe, --n-cpu-moe (default: %s)\n", join(cmd_params_defaults.n_cpu_moe, ",").c_str()); + printf(" -sm, --split-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str()); + printf(" -mg, --main-gpu (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str()); + printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str()); + printf(" -fa, --flash-attn (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str()); + printf(" -dev, --device (default: auto)\n"); + printf(" -lm, --load-mode (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str()); + printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n"); + printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n"); + printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str()); + printf(" -ts, --tensor-split (default: 0)\n"); printf(" -ot --override-tensor =;...\n"); - printf(" (default: disabled)\n"); - printf(" -nopo, --no-op-offload <0|1> (default: 0)\n"); - printf(" --no-host <0|1> (default: %s)\n", join(cmd_params_defaults.no_host, ",").c_str()); + printf(" (default: disabled)\n"); + printf(" -nopo, --no-op-offload <0|1> (default: 0)\n"); + printf(" --no-host <0|1> (default: %s)\n", join(cmd_params_defaults.no_host, ",").c_str()); printf("\n"); printf( "Multiple values can be given for each parameter by separating them with ','\n" @@ -670,22 +670,7 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { break; } } else if (arg == "--list-devices") { - std::vector devices; - for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { - auto * dev = ggml_backend_dev_get(i); - if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) { - devices.push_back(dev); - } - } - printf("Available devices:\n"); - if (devices.empty()) { - printf(" (none)\n"); - } - for (auto * dev : devices) { - size_t free, total; - ggml_backend_dev_memory(dev, &free, &total); - printf(" %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024); - } + common_print_available_devices(); exit(0); } else if (arg == "-t" || arg == "--threads") { if (++i >= argc) { @@ -785,6 +770,8 @@ static cmd_params parse_cmd_params(int argc, char ** argv) { mode = LLAMA_LOAD_MODE_MMAP; } else if (m == "mlock") { mode = LLAMA_LOAD_MODE_MLOCK; + } else if (m == "mmap+mlock") { + mode = LLAMA_LOAD_MODE_MMAP_MLOCK; } else if (m == "dio") { mode = LLAMA_LOAD_MODE_DIRECT_IO; } else { diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index d0329ca567..15040e4af5 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -47,9 +47,11 @@ add_library(mtmd models/paddleocr.cpp models/pixtral.cpp models/qwen2vl.cpp + models/minimax-m3.cpp models/qwen3vl.cpp models/mimovl.cpp models/qwen3a.cpp + models/mimo-audio.cpp models/step3vl.cpp models/siglip.cpp models/whisper-enc.cpp @@ -58,6 +60,7 @@ add_library(mtmd models/mobilenetv5.cpp models/youtuvl.cpp models/yasa2.cpp + models/parakeet.cpp ) set_target_properties(mtmd PROPERTIES diff --git a/tools/mtmd/clip-graph.h b/tools/mtmd/clip-graph.h index a95de20a31..29352abb4c 100644 --- a/tools/mtmd/clip-graph.h +++ b/tools/mtmd/clip-graph.h @@ -13,6 +13,14 @@ struct build_vit_opts { ggml_tensor * attn_mask = nullptr; + // TODO @ngxson : merge attn_mask and attn_mask_layers into one call + std::vector attn_mask_layers; // one per layer + + // hook at layer output embeddings + std::function callback_layer_out = nullptr; + + // whether to skip the automatic post-layernorm (model.post_ln_w) applied at the end + bool skip_post_ln = false; }; struct clip_graph { diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 5b413681f0..d42b38222c 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -41,6 +41,7 @@ #define KEY_PROJ_DIM "clip.%s.projection_dim" #define KEY_N_HEAD "clip.%s.attention.head_count" #define KEY_N_HEAD_KV "clip.%s.attention.head_count_kv" +#define KEY_N_EMBD_HEAD "clip.%s.attention.head_dim" #define KEY_LAYER_NORM_EPS "clip.%s.attention.layer_norm_epsilon" #define KEY_FEATURE_LAYERS "clip.%s.feature_layer" @@ -82,6 +83,13 @@ #define KEY_A_PROJ_WINDOW_SIZE "clip.audio.projector.window_size" #define KEY_A_PROJ_DOWNSAMPLE_RATE "clip.audio.projector.downsample_rate" #define KEY_A_PROJ_HEAD_COUNT "clip.audio.projector.head_count" +#define KEY_A_RVQ_NUM_QUANTIZERS "clip.audio.rvq.num_quantizers" // mimo-audio-tokenizer +#define KEY_A_RVQ_CODEBOOK_SIZE "clip.audio.rvq.codebook_size" // mimo-audio-tokenizer: per-quantizer bin count +#define KEY_A_WA_PATTERN_MODE "clip.audio.wa_pattern_mode" // mimo-audio-tokenizer, per-layer -1 (full) / 0 (windowed) +#define KEY_A_ATTN_WINDOW_SIZE "clip.audio.window_size" // mimo-audio-tokenizer: sliding-window radius +#define KEY_A_LOCAL_BLOCK_COUNT "clip.audio.local_block_count" // mimo-v2.5: input_local_transformer layer count +#define KEY_A_LOCAL_GROUP_SIZE "clip.audio.local_group_size" // mimo-v2.5: input_local_transformer grouping size +#define KEY_AUDIO_SUBSAMPLING_FACTOR "clip.audio.subsampling_factor" // // tensor name constants @@ -131,6 +139,8 @@ #define TN_MM_SOFT_EMB_N "mm.soft_emb_norm.weight" // gemma3 #define TN_MM_PROJECTOR "mm.model.fc.%s" // idefics3, deepseekocr #define TN_MM_PATCH_MERGER "mm.patch_merger.%s" // mistral small 3.1, glm4v +#define TN_MM_MERGER_FC1 "mm.merger.fc1.%s" // minimax-m3 patch-merge MLP +#define TN_MM_MERGER_FC2 "mm.merger.fc2.%s" #define TN_TOK_IMG_BREAK "v.token_embd.img_break" // pixtral #define TN_TOK_GLM_BOI "adapter.boi" // glm-edge (these embeddings are not in text model) #define TN_TOK_GLM_EOI "adapter.eoi" // glm-edge (these embeddings are not in text model) @@ -173,6 +183,24 @@ #define TN_MM_NORM_PRE "mm.a.norm_pre.%s" #define TN_MM_NORM_MID "mm.a.norm_mid.%s" +// mimo-audio-tokenizer +#define TN_A_DOWNSAMPLE_CONV "a.downsample.conv.%s" +#define TN_A_DOWNSAMPLE_NORM "a.downsample.norm.%s" +#define TN_A_RVQ_CODEBOOK "a.rvq.codebook.%s" +// mimo-v2.5: text-side RVQ code embedding ("text codebook") +#define TN_MM_A_CODE_EMBD "mm.a.code_embd.%s" +// mimo-v2.5: LLM-side connector (input_local_transformer) +#define TN_MM_A_LOCAL_ATTN_Q "mm.a.local_blk.%d.attn_q.%s" +#define TN_MM_A_LOCAL_ATTN_K "mm.a.local_blk.%d.attn_k.%s" +#define TN_MM_A_LOCAL_ATTN_V "mm.a.local_blk.%d.attn_v.%s" +#define TN_MM_A_LOCAL_ATTN_OUT "mm.a.local_blk.%d.attn_out.%s" +#define TN_MM_A_LOCAL_FFN_GATE "mm.a.local_blk.%d.ffn_gate.%s" +#define TN_MM_A_LOCAL_FFN_UP "mm.a.local_blk.%d.ffn_up.%s" +#define TN_MM_A_LOCAL_FFN_DOWN "mm.a.local_blk.%d.ffn_down.%s" +#define TN_MM_A_LOCAL_LN1 "mm.a.local_blk.%d.ln1.%s" +#define TN_MM_A_LOCAL_LN2 "mm.a.local_blk.%d.ln2.%s" +#define TN_MM_A_LOCAL_NORM "mm.a.local_norm.%s" + // cogvlm #define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s" #define TN_MM_H_TO_4H "mm.up.%s" @@ -312,6 +340,12 @@ #define TN_YASA_STAGE_DOWN_CONV "v.stage.%d.down.conv.%s" #define TN_YASA_STAGE_BLK "v.stage.%d.blk.%d.%s.%s" +// parakeet +#define TN_MEL_FILTERS "a.mel_filters" +#define TN_WINDOW "a.window" +#define TN_CONV_NORM_MEAN "%s.blk.%d.conv_norm_mean" +#define TN_CONV_NORM_VAR "%s.blk.%d.conv_norm_var" + // align x to upper multiple of n #define CLIP_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n)) @@ -366,11 +400,14 @@ enum projector_type { PROJECTOR_TYPE_KIMIK25, PROJECTOR_TYPE_NEMOTRON_V2_VL, PROJECTOR_TYPE_HUNYUANVL, + PROJECTOR_TYPE_PARAKEET, PROJECTOR_TYPE_EXAONE4_5, PROJECTOR_TYPE_MINICPMV4_6, PROJECTOR_TYPE_GRANITE_SPEECH, PROJECTOR_TYPE_MIMOVL, + PROJECTOR_TYPE_MINIMAX_M3, PROJECTOR_TYPE_GRANITE4_VISION, + PROJECTOR_TYPE_MIMO_AUDIO, PROJECTOR_TYPE_UNKNOWN, }; @@ -424,7 +461,10 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"}, { PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"}, { PROJECTOR_TYPE_MIMOVL, "mimovl"}, + { PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"}, { PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"}, + { PROJECTOR_TYPE_MIMO_AUDIO, "mimo_audio"}, + { PROJECTOR_TYPE_PARAKEET, "parakeet"}, }; static projector_type clip_projector_type_from_string(const std::string & str) { diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 6d4336c401..8b9db5101d 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -33,7 +33,7 @@ enum resize_algo { RESIZE_ALGO_BILINEAR, // stretch to target resolution RESIZE_ALGO_BICUBIC, // center-crop when aspect ratio doesn't match RESIZE_ALGO_BICUBIC_PILLOW, - // RESIZE_ALGO_LANCZOS, // TODO + RESIZE_ALGO_LANCZOS, }; // Padding style for img_tool::resize @@ -54,6 +54,8 @@ struct clip_hparams { int32_t projection_dim = 0; int32_t n_head = 0; int32_t n_head_kv = 0; + // 0 = derive from n_embd; set when qkv width != n_embd + int32_t n_embd_head = 0; int32_t n_layer = 0; int32_t n_merge = 1; // number of patch merges **per-side** @@ -110,6 +112,8 @@ struct clip_hparams { // audio int32_t n_mel_bins = 0; // whisper preprocessor int32_t proj_stack_factor = 0; // ultravox + int32_t subsampling_factor = 0; // parakeet + int32_t audio_chunk_size = 0; int32_t audio_conv_kernel_size = 0; int32_t audio_max_pos_emb = 0; @@ -124,6 +128,18 @@ struct clip_hparams { int32_t audio_window_len = -1; int32_t audio_hop_len = -1; + // parakeet + std::vector mel_filters; + std::vector window; + + // mimo-audio-tokenizer: residual vector quantizer + int32_t rvq_num_quantizers = 0; + std::vector rvq_codebook_size; // per-quantizer bin count (ragged, e.g. 1024/1024/256/128x17) + + // mimo-v2.5: LLM-side connector (input_local_transformer) + int32_t audio_local_n_layer = 0; + int32_t audio_local_group_size = 0; + // legacy bool has_llava_projector = false; int minicpmv_version = 0; @@ -237,14 +253,16 @@ struct clip_layer { ggml_tensor * norm_conv_b = nullptr; ggml_tensor * linear_pos_w = nullptr; - ggml_tensor * conv_norm_w = nullptr; - ggml_tensor * conv_norm_b = nullptr; - ggml_tensor * conv_dw_w = nullptr; - ggml_tensor * conv_dw_b = nullptr; - ggml_tensor * conv_pw1_w = nullptr; - ggml_tensor * conv_pw1_b = nullptr; - ggml_tensor * conv_pw2_w = nullptr; - ggml_tensor * conv_pw2_b = nullptr; + ggml_tensor * conv_norm_w = nullptr; + ggml_tensor * conv_norm_b = nullptr; + ggml_tensor * conv_norm_mean = nullptr; // parakeet + ggml_tensor * conv_norm_var = nullptr; // parakeet + ggml_tensor * conv_dw_w = nullptr; + ggml_tensor * conv_dw_b = nullptr; + ggml_tensor * conv_pw1_w = nullptr; + ggml_tensor * conv_pw1_b = nullptr; + ggml_tensor * conv_pw2_w = nullptr; + ggml_tensor * conv_pw2_b = nullptr; // gemma4 audio conformer per-layer ggml_tensor * attn_pre_norm_w = nullptr; @@ -397,6 +415,10 @@ struct clip_model { ggml_tensor * mm_0_b = nullptr; ggml_tensor * mm_2_w = nullptr; ggml_tensor * mm_2_b = nullptr; + ggml_tensor * mm_merger_fc1_w = nullptr; // minimax-m3 + ggml_tensor * mm_merger_fc1_b = nullptr; + ggml_tensor * mm_merger_fc2_w = nullptr; + ggml_tensor * mm_merger_fc2_b = nullptr; ggml_tensor * image_newline = nullptr; ggml_tensor * view_seperator = nullptr; @@ -533,6 +555,20 @@ struct clip_model { ggml_tensor * mm_norm_pre_b = nullptr; ggml_tensor * mm_norm_mid_w = nullptr; + // mimo-audio-tokenizer: post-transformer downsample + RVQ codebook + ggml_tensor * downsample_conv_w = nullptr; // no bias + ggml_tensor * downsample_norm_w = nullptr; + ggml_tensor * downsample_norm_b = nullptr; + ggml_tensor * rvq_codebook = nullptr; // merged 3D [n_q, max_bins, dim] + + // mimo-v2.5: text-side RVQ code embedding ("text codebook") + ggml_tensor * mm_a_code_embd = nullptr; // merged 3D [n_channels, vocab, dim] + + // mimo-v2.5: LLM-side connector (input_local_transformer, separate from the + // audio_tokenizer's own encoder `layers`) + std::vector mm_a_local_layers; + ggml_tensor * mm_a_local_norm_w = nullptr; + // qwen3a ggml_tensor * conv2d_1_w = nullptr; ggml_tensor * conv2d_1_b = nullptr; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index b886650649..5f0d00b660 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -253,7 +253,7 @@ clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) : n_embd(hparams.n_embd), n_head(hparams.n_head), n_head_kv(hparams.n_head_kv), - d_head(n_head > 0 ? n_embd / n_head : 0), + d_head(hparams.n_embd_head > 0 ? hparams.n_embd_head : (n_head > 0 ? n_embd / n_head : 0)), n_layer(hparams.n_layer), n_mmproj_embd(clip_n_mmproj_embd(ctx)), eps(hparams.eps), @@ -340,6 +340,11 @@ ggml_tensor * clip_graph::build_vit( auto & layer = model.layers[il]; ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states + ggml_tensor * attn_mask = opts.attn_mask; + if (opts.attn_mask_layers.size() > (size_t) il) { + attn_mask = opts.attn_mask_layers[il]; + } + // layernorm1 cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il); cb(cur, "layer_inp_normed", il); @@ -367,13 +372,13 @@ ggml_tensor * clip_graph::build_vit( /* nb1 */ ggml_row_size(cur->type, d_head), /* nb2 */ cur->nb[1], /* nb3 */ cur->nb[1] * n_pos, - /* offset */ ggml_row_size(cur->type, n_embd)); + /* offset */ ggml_row_size(cur->type, n_head * d_head)); Vcur = ggml_view_4d(ctx0, cur, d_head, n_head, n_pos, B, /* nb1 */ ggml_row_size(cur->type, d_head), /* nb2 */ cur->nb[1], /* nb3 */ cur->nb[1] * n_pos, - /* offset */ ggml_row_size(cur->type, 2 * n_embd)); + /* offset */ ggml_row_size(cur->type, 2 * n_head * d_head)); if (layer.q_norm) { GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]); @@ -452,7 +457,7 @@ ggml_tensor * clip_graph::build_vit( // build_attn returns a flat 2D [n_embd, n_pos*B] cur = build_attn(layer.o_w, layer.o_b, - Qcur, Kcur, Vcur, opts.attn_mask, kq_scale, il); + Qcur, Kcur, Vcur, attn_mask, kq_scale, il); cb(cur, "attn_out", il); } @@ -471,6 +476,10 @@ ggml_tensor * clip_graph::build_vit( inpL = cur; // inpL = residual, cur = hidden_states + if (opts.callback_layer_out) { + opts.callback_layer_out(cur, il); + } + cb(cur, "ffn_inp", il); // layernorm2 (pre-ffn norm) @@ -519,7 +528,7 @@ ggml_tensor * clip_graph::build_vit( } // post-layernorm - if (model.post_ln_w) { + if (model.post_ln_w && !opts.skip_post_ln) { inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, -1); } @@ -915,6 +924,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_STEP3VL: { builder = std::make_unique(ctx, img); @@ -1008,6 +1021,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_MIMO_AUDIO: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_YOUTUVL: { builder = std::make_unique(ctx, img); @@ -1016,6 +1033,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_PARAKEET: + { + builder = std::make_unique(ctx, img); + } break; case PROJECTOR_TYPE_GRANITE4_VISION: { builder = std::make_unique(ctx, img); @@ -1169,6 +1190,7 @@ struct clip_model_loader { const char * prefix = is_vision ? "vision" : "audio"; get_u32(string_format(KEY_N_EMBD, prefix), hparams.n_embd); get_u32(string_format(KEY_N_HEAD, prefix), hparams.n_head); + get_u32(string_format(KEY_N_EMBD_HEAD, prefix), hparams.n_embd_head, false); get_u32(string_format(KEY_N_FF, prefix), hparams.n_ff); get_u32(string_format(KEY_N_BLOCK, prefix), hparams.n_layer); get_u32(string_format(KEY_PROJ_DIM, prefix), hparams.projection_dim); @@ -1339,6 +1361,20 @@ struct clip_model_loader { { get_u32(KEY_PROJ_SCALE_FACTOR, hparams.n_merge, false); } break; + case PROJECTOR_TYPE_PARAKEET: + { + get_u32(KEY_AUDIO_SUBSAMPLING_FACTOR, hparams.subsampling_factor); + GGML_ASSERT(hparams.subsampling_factor == 8 && + "subsampling_factor must match the conv strides in clip_graph_parakeet::build()"); + get_u32(KEY_A_CONV_KERNEL_SIZE, hparams.audio_conv_kernel_size); + GGML_ASSERT(hparams.audio_conv_kernel_size > 0 && hparams.audio_conv_kernel_size % 2 == 1 && + "audio_conv_kernel_size must be a positive odd integer"); + hparams.audio_chunk_len = 0; + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 512; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + } break; case PROJECTOR_TYPE_IDEFICS3: { // use default llava-uhd preprocessing params @@ -1469,6 +1505,17 @@ struct clip_model_loader { LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__); } } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + hparams.n_merge = 2; // spatial_merge_size + hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW; + hparams.image_resize_pad = PAD_NONE; + get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false); + hparams.rope_theta = 10000.0f; // vision_config.rope_theta + // MiniMax-M3: max_pixels 451584 (=672^2) -> 576 merged tokens (image_seq_length) + hparams.set_limit_image_tokens(8, 576); + hparams.set_warmup_n_tokens(16*16); + } break; case PROJECTOR_TYPE_MIMOVL: { hparams.n_merge = 2; // spatial_merge_size @@ -1560,6 +1607,45 @@ struct clip_model_loader { hparams.audio_window_len = 400; hparams.audio_hop_len = 160; } break; + case PROJECTOR_TYPE_MIMO_AUDIO: + { + get_u32(KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers, false); + get_arr_int(KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size, false); + if (hparams.rvq_num_quantizers <= 0) { + throw std::runtime_error(string_format("%s: mimo_audio: missing %s\n", __func__, KEY_A_RVQ_NUM_QUANTIZERS)); + } + if ((int) hparams.rvq_codebook_size.size() != hparams.rvq_num_quantizers) { + throw std::runtime_error(string_format( + "%s: mimo_audio: %s length (%zu) must equal %s (%d)\n", __func__, + KEY_A_RVQ_CODEBOOK_SIZE, hparams.rvq_codebook_size.size(), + KEY_A_RVQ_NUM_QUANTIZERS, hparams.rvq_num_quantizers)); + } + hparams.ffn_op = FFN_GELU_ERF; // PyTorch F.gelu default (approximate="none") + hparams.rope_theta = 10000.0f; + + // audio preprocessing params (mel spectrogram) + hparams.audio_sample_rate = 24000; + hparams.audio_n_fft = 960; + hparams.audio_window_len = 960; + hparams.audio_hop_len = 240; + + get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size); + std::vector wa_pattern; + get_arr_int(KEY_A_WA_PATTERN_MODE, wa_pattern, true); + if ((int) wa_pattern.size() != hparams.n_layer) { + throw std::runtime_error(string_format( + "%s: mimo_audio: %s length (%zu) must equal n_layer (%d)\n", __func__, + KEY_A_WA_PATTERN_MODE, wa_pattern.size(), hparams.n_layer)); + } + hparams.wa_pattern_mode.assign(wa_pattern.begin(), wa_pattern.end()); + + get_u32(KEY_A_LOCAL_BLOCK_COUNT, hparams.audio_local_n_layer); + get_u32(KEY_A_LOCAL_GROUP_SIZE, hparams.audio_local_group_size); + if (hparams.audio_local_group_size <= 0) { + throw std::runtime_error(string_format( + "%s: mimo_audio: %s must be > 0\n", __func__, KEY_A_LOCAL_GROUP_SIZE)); + } + } break; case PROJECTOR_TYPE_PADDLEOCR: { hparams.n_merge = 2; @@ -1826,16 +1912,46 @@ struct clip_model_loader { return cur; }; - auto get_scalar = [&](const std::string & name, float default_val) { + auto get_vector = [&](const std::string & name) { + std::vector result; auto it = tensor_offset.find(name); if (it == tensor_offset.end()) { + return result; + } + + const int64_t idx = gguf_find_tensor(ctx_gguf.get(), name.c_str()); + if (idx < 0) { + throw std::runtime_error(string_format("%s: failed to find tensor %s\n", __func__, name.c_str())); + } + + if (const auto type = gguf_get_tensor_type(ctx_gguf.get(), idx); type != GGML_TYPE_F32) { + throw std::runtime_error(string_format("%s: %s must be %s, was %s\n", __func__, + name.c_str(), ggml_type_name(GGML_TYPE_F32), ggml_type_name(type))); + } + + const size_t n_bytes = gguf_get_tensor_size(ctx_gguf.get(), idx); + if (n_bytes == 0) { + throw std::runtime_error(string_format("%s: tensor %s is empty\n", __func__, name.c_str())); + } + + const size_t n_elems = n_bytes / sizeof(float); + result.resize(n_elems); + fin.seekg(it->second, std::ios::beg); + fin.read(reinterpret_cast(result.data()), n_bytes); + return result; + }; + + auto get_scalar = [&](const std::string & name, float default_val) { + auto v = get_vector(name); + if (v.empty()) { return default_val; } - size_t offset = it->second; - fin.seekg(offset, std::ios::beg); - float value; - fin.read(reinterpret_cast(&value), sizeof(float)); - return value; + if (v.size() != 1) { + throw std::runtime_error(string_format("%s: expected scalar tensor '%s' but got %d elements\n", + __func__, name.c_str(), (int) v.size())); + } + + return v[0]; }; model.class_embedding = get_tensor(TN_CLASS_EMBD, false); @@ -2089,6 +2205,19 @@ struct clip_model_loader { model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false); } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + // per-patch MLP: mm.1 -> gelu -> mm.2 + model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 1, "weight")); + model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 1, "bias")); + model.mm_2_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight")); + model.mm_2_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias")); + // 2x2 merge MLP: mm.merge.fc1 -> gelu -> mm.merge.fc2 + model.mm_merger_fc1_w = get_tensor(string_format(TN_MM_MERGER_FC1, "weight")); + model.mm_merger_fc1_b = get_tensor(string_format(TN_MM_MERGER_FC1, "bias")); + model.mm_merger_fc2_w = get_tensor(string_format(TN_MM_MERGER_FC2, "weight")); + model.mm_merger_fc2_b = get_tensor(string_format(TN_MM_MERGER_FC2, "bias")); + } break; case PROJECTOR_TYPE_STEP3VL: { model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight")); @@ -2416,6 +2545,54 @@ struct clip_model_loader { model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight")); model.mm_2_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "bias")); } break; + case PROJECTOR_TYPE_MIMO_AUDIO: + { + model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); + model.conv1d_1_b = get_tensor(string_format(TN_CONV1D, 1, "bias")); + model.conv1d_2_w = get_tensor(string_format(TN_CONV1D, 2, "weight")); + model.conv1d_2_b = get_tensor(string_format(TN_CONV1D, 2, "bias")); + model.downsample_conv_w = get_tensor(string_format(TN_A_DOWNSAMPLE_CONV, "weight")); + model.downsample_norm_w = get_tensor(string_format(TN_A_DOWNSAMPLE_NORM, "weight")); + model.downsample_norm_b = get_tensor(string_format(TN_A_DOWNSAMPLE_NORM, "bias")); + model.rvq_codebook = get_tensor(string_format(TN_A_RVQ_CODEBOOK, "weight"), false); + model.mm_a_code_embd = get_tensor(string_format(TN_MM_A_CODE_EMBD, "weight"), false); + if (!model.rvq_codebook || !model.mm_a_code_embd) { + throw std::runtime_error(string_format("%s: mimo_audio: missing %s or %s\n", __func__, + TN_A_RVQ_CODEBOOK, TN_MM_A_CODE_EMBD)); + } + // hparams.rvq_codebook_size comes from GGUF metadata and is independent of the + // tensors' actual shapes - bound it so codebook/code_embd views built from it + // (mimo-audio.cpp) can never read past either tensor's allocated bins/vocab. + for (int32_t bins : hparams.rvq_codebook_size) { + if (bins <= 0 || bins > model.rvq_codebook->ne[1] || bins > model.mm_a_code_embd->ne[1]) { + throw std::runtime_error(string_format( + "%s: mimo_audio: %s entry (%d) out of range for codebook/code_embd tensors\n", + __func__, KEY_A_RVQ_CODEBOOK_SIZE, bins)); + } + } + + // LLM-side connector: input_local_transformer + projection + model.mm_a_local_layers.resize(hparams.audio_local_n_layer); + for (int il = 0; il < hparams.audio_local_n_layer; il++) { + auto & layer = model.mm_a_local_layers[il]; + layer.q_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_Q, il, "weight")); + layer.q_b = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_Q, il, "bias")); + layer.k_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_K, il, "weight")); + layer.k_b = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_K, il, "bias")); + layer.v_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_V, il, "weight")); + layer.v_b = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_V, il, "bias")); + layer.o_w = get_tensor(string_format(TN_MM_A_LOCAL_ATTN_OUT, il, "weight")); + layer.ff_gate_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_GATE, il, "weight")); + layer.ff_up_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_UP, il, "weight")); + layer.ff_down_w = get_tensor(string_format(TN_MM_A_LOCAL_FFN_DOWN, il, "weight")); + layer.ln_1_w = get_tensor(string_format(TN_MM_A_LOCAL_LN1, il, "weight")); + layer.ln_2_w = get_tensor(string_format(TN_MM_A_LOCAL_LN2, il, "weight")); + } + model.mm_a_local_norm_w = get_tensor(string_format(TN_MM_A_LOCAL_NORM, "weight")); + + model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight")); + model.mm_2_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight")); + } break; case PROJECTOR_TYPE_VOXTRAL: { model.conv1d_1_w = get_tensor(string_format(TN_CONV1D, 1, "weight")); @@ -2672,6 +2849,68 @@ struct clip_model_loader { layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias")); } } break; + case PROJECTOR_TYPE_PARAKEET: + { + + hparams.mel_filters = get_vector(TN_MEL_FILTERS); + hparams.window = get_vector(TN_WINDOW); + + // Subsampling layers (conv1d) + for (int i : {0, 2, 3, 5, 6}) { + model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight")); + model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias")); + } + model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight")); + model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias")); + + // Projection layers + model.mm_norm_pre_w = get_tensor(string_format(TN_MM_NORM_PRE, "weight"), false); + model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"), false); + model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 2, "weight"), false); + + // Encoder layers + for (int il = 0; il < hparams.n_layer; ++il) { + auto & layer = model.layers[il]; + + // Attention (from shared above) + + // Relative position encoding + layer.linear_pos_w = get_tensor(string_format(TN_LINEAR_POS, prefix, il, "weight")); + layer.pos_bias_u = get_tensor(string_format(TN_POS_BIAS_U, prefix, il)); + layer.pos_bias_v = get_tensor(string_format(TN_POS_BIAS_V, prefix, il)); + + // Convolution module + layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight")); + layer.conv_pw1_b = get_tensor(string_format(TN_CONV_PW1, prefix, il, "bias"), false); + layer.conv_dw_w = get_tensor(string_format(TN_CONV_DW, prefix, il, "weight")); + layer.conv_dw_b = get_tensor(string_format(TN_CONV_DW, prefix, il, "bias"), false); + layer.conv_norm_w = get_tensor(string_format(TN_CONV_NORM, prefix, il, "weight")); + layer.conv_norm_b = get_tensor(string_format(TN_CONV_NORM, prefix, il, "bias")); + layer.conv_norm_mean = get_tensor(string_format(TN_CONV_NORM_MEAN, prefix, il)); + layer.conv_norm_var = get_tensor(string_format(TN_CONV_NORM_VAR, prefix, il)); + layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight")); + layer.conv_pw2_b = get_tensor(string_format(TN_CONV_PW2, prefix, il, "bias"), false); + + // Feed-forward networks + layer.ff_norm_w = get_tensor(string_format(TN_FFN_NORM, prefix, il, "weight")); + layer.ff_norm_b = get_tensor(string_format(TN_FFN_NORM, prefix, il, "bias")); + + layer.ff_norm_1_w = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "weight")); + layer.ff_norm_1_b = get_tensor(string_format(TN_FFN_NORM_1, prefix, il, "bias")); + layer.ff_up_1_w = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "weight")); + layer.ff_up_1_b = get_tensor(string_format(TN_FFN_UP_1, prefix, il, "bias"), false); + layer.ff_down_1_w = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "weight")); + layer.ff_down_1_b = get_tensor(string_format(TN_FFN_DOWN_1, prefix, il, "bias"), false); + + // Layer norms + layer.norm_conv_w = get_tensor(string_format(TN_NORM_CONV, prefix, il, "weight")); + layer.norm_conv_b = get_tensor(string_format(TN_NORM_CONV, prefix, il, "bias")); + } + + model.mm_model_mlp_1_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 0, "weight")); + model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight")); + model.mm_model_mlp_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight")); + } break; case PROJECTOR_TYPE_GRANITE_SPEECH: { model.inp_proj_w = get_tensor(string_format(TN_INP_PROJ, "weight")); @@ -3360,6 +3599,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { case PROJECTOR_TYPE_QWEN3VL: case PROJECTOR_TYPE_EXAONE4_5: case PROJECTOR_TYPE_MIMOVL: + case PROJECTOR_TYPE_MINIMAX_M3: case PROJECTOR_TYPE_GLM4V: case PROJECTOR_TYPE_YOUTUVL: { @@ -3516,10 +3756,23 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { } n_patches = n; } break; + case PROJECTOR_TYPE_PARAKEET: + { + n_patches = (img->nx() + (params.subsampling_factor - 1)) / params.subsampling_factor; + } break; case PROJECTOR_TYPE_GEMMA4UA: { n_patches = img->nx(); // no downsampling: one token per raw waveform frame } break; + case PROJECTOR_TYPE_MIMO_AUDIO: + { + // conv1(s=1) + conv2(s=2) -> RVQ-encoder downsample conv(k=2,s=2) + int n = img->nx(); + n = (n - 1) / 2 + 1; // conv1 + conv2 + n = (n - 2) / 2 + 1; // downsample conv + const int group_size = params.audio_local_group_size; + n_patches = (n + group_size - 1) / group_size; + } break; case PROJECTOR_TYPE_GRANITE_SPEECH: { const int ws = ctx->model.hparams.audio_proj_window_size; @@ -3866,6 +4119,24 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 set_input_i32("positions", positions); } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + const int n_merge = hparams.n_merge; + const int gh = image_size_height / patch_size; + const int gw = image_size_width / patch_size; + std::vector pos_h, pos_w; + pos_h.reserve(gh * gw); + pos_w.reserve(gh * gw); + for (int bh = 0; bh < gh / n_merge; bh++) + for (int bw = 0; bw < gw / n_merge; bw++) + for (int mh = 0; mh < n_merge; mh++) + for (int mw = 0; mw < n_merge; mw++) { + pos_h.push_back(bh * n_merge + mh); + pos_w.push_back(bw * n_merge + mw); + } + set_input_i32("minimax_pos_h", pos_h); + set_input_i32("minimax_pos_w", pos_w); + } break; case PROJECTOR_TYPE_DOTS_OCR: { const int pw = image_size_width / patch_size; @@ -4329,6 +4600,58 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 set_input_f32("pos_emb", pos_emb); } } break; + case PROJECTOR_TYPE_MIMO_AUDIO: + { + GGML_ASSERT(imgs.entries.size() == 1); + const int n_frames = imgs.entries.front().nx(); + const int n_pos = (n_frames - 1) / 2 + 1; // matches conv1(s=1)+conv2(s=2) output length + + std::vector positions(n_pos); + for (int i = 0; i < n_pos; i++) { + positions[i] = i; + } + set_input_i32("mimo_audio_positions", positions); + + const int window = hparams.attn_window_size; + GGML_ASSERT(window > 0); + + const float neg_inf = std::numeric_limits::lowest(); + std::vector full_mask((size_t) n_pos * n_pos); + std::vector window_mask((size_t) n_pos * n_pos); + for (int q = 0; q < n_pos; q++) { + for (int k = 0; k < n_pos; k++) { + const bool causal_ok = k <= q; + full_mask[(size_t) q * n_pos + k] = causal_ok ? 0.0f : neg_inf; + window_mask[(size_t) q * n_pos + k] = (causal_ok && (q - k) <= window) ? 0.0f : neg_inf; + } + } + set_input_f32("mimo_audio_full_mask", full_mask); + set_input_f32("mimo_audio_window_mask", window_mask); + + // input_local_transformer: block-diagonal mask + in-group positions + { + const int n_pos_ds = (n_pos - 2) / 2 + 1; // matches downsample conv (k=2,s=2,p=0) + const int group_size = hparams.audio_local_group_size; + GGML_ASSERT(group_size > 0); + const int n_groups = (n_pos_ds + group_size - 1) / group_size; + const int n_padded = n_groups * group_size; + + std::vector local_positions(n_padded); + for (int i = 0; i < n_padded; i++) { + local_positions[i] = i % group_size; + } + set_input_i32("mimo_audio_local_positions", local_positions); + + std::vector local_mask((size_t) n_padded * n_padded); + for (int q = 0; q < n_padded; q++) { + for (int k = 0; k < n_padded; k++) { + const bool same_group = (q / group_size) == (k / group_size); + local_mask[(size_t) q * n_padded + k] = same_group ? 0.0f : neg_inf; + } + } + set_input_f32("mimo_audio_local_mask", local_mask); + } + } break; case PROJECTOR_TYPE_LFM2A: { GGML_ASSERT(imgs.entries.size() == 1); @@ -4350,6 +4673,88 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 } set_input_f32("pos_emb", pos_emb); } break; + case PROJECTOR_TYPE_PARAKEET: + { + GGML_ASSERT(imgs.entries.size() == 1); + struct ggml_tensor * attn_mask = ggml_graph_get_tensor(gf, "attn_mask"); + const int n_q = attn_mask->ne[1]; + const int n_k = attn_mask->ne[0]; + const int n_frames = imgs.entries.front().nx(); + const int n_tokens_real = (n_frames + hparams.subsampling_factor-1) / hparams.subsampling_factor; + const float mask_value = -1e30f; + + std::vector mask_data(n_q * n_k); + if (n_k == n_q) { + // full attention: mask keys that are padding + for (int q = 0; q < n_q; ++q) { + for (int k = 0; k < n_k; ++k) { + mask_data[q * n_k + k] = (k >= n_tokens_real) ? mask_value : 0.0f; + } + } + } else { + // local attention: mask keys outside the valid window + const int att_left = n_k / 2; + for (int q = 0; q < n_q; ++q) { + for (int k = 0; k < n_k; ++k) { + const int key = q - att_left + k; + mask_data[q * n_k + k] = (key >= 0 && key < n_tokens_real) ? 0.0f : mask_value; + } + } + } + set_input_f32(attn_mask->name, mask_data); + + // local attention skew mask: zeroes out the probs that were + // computed for keys outside the valid sliding window. + if (struct ggml_tensor * local_mask = ggml_graph_get_tensor(gf, "local_mask")) { + const int lm_k = local_mask->ne[0]; + const int lm_q = local_mask->ne[1]; + const int window_size = lm_k - lm_q + 1; + std::vector lm_data(lm_q * lm_k); + for (int q = 0; q < lm_q; ++q) { + for (int k = 0; k < lm_k; ++k) { + const int rel = k - q; + lm_data[q * lm_k + k] = (rel >= 0 && rel < window_size) ? 1.0f : 0.0f; + } + } + set_input_f32(local_mask->name, lm_data); + } + + // Generate rotation frequencies for relative positional encoding. + { + const int n_state = hparams.n_embd; + const int d_half = n_state / 2; + const float log_10000 = logf(10000.0f); + std::vector freqs(d_half); + for (int k = 0; k < d_half; ++k) { + freqs[k] = expf(-(float(k * 2) * log_10000 / float(n_state))); + } + set_input_f32("pos_freqs", freqs); + } + + // Generate relative positional distance values which scaled by + // the frequency to produce the angles for sin/cos. + { + // window_size is only known after graph construction since it depends on + // n_time from the conv output, so we read it back from the graph tensor. + struct ggml_tensor * rel_pos = ggml_graph_get_tensor(gf, "rel_positions"); + const int window_size = rel_pos->ne[1]; + std::vector pos(window_size); + // local attention: window is fixed at [att_left, att_right] + // full attention: window covers the full sequence, centered + if (ggml_graph_get_tensor(gf, "local_mask")) { + const int att_left = window_size / 2; + for (int t = 0; t < window_size; ++t) { + pos[t] = float(att_left - t); + } + } else { + const int n_time = (window_size + 1) / 2; + for (int t = 0; t < window_size; ++t) { + pos[t] = float(n_time - 1 - t); + } + } + set_input_f32(rel_pos->name, pos); + } + } break; case PROJECTOR_TYPE_GRANITE_SPEECH: { const int context_size = ctx->model.hparams.audio_chunk_size; @@ -4569,6 +4974,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.mm_ffn_down_w->ne[1]; case PROJECTOR_TYPE_GLM_EDGE: return ctx->model.mm_model_mlp_3_w->ne[1]; + case PROJECTOR_TYPE_MINIMAX_M3: + return ctx->model.mm_merger_fc2_b->ne[0]; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: case PROJECTOR_TYPE_EXAONE4_5: @@ -4629,6 +5036,10 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.qf_proj_blocks.size() * ctx->model.hparams.projection_dim; case PROJECTOR_TYPE_GLM4V: return ctx->model.mm_ffn_down_w->ne[1]; + case PROJECTOR_TYPE_MIMO_AUDIO: + return ctx->model.mm_2_w->ne[1]; + case PROJECTOR_TYPE_PARAKEET: + return ctx->model.mm_1_w->ne[1]; default: GGML_ABORT("Unknown projector type"); } diff --git a/tools/mtmd/models/mimo-audio.cpp b/tools/mtmd/models/mimo-audio.cpp new file mode 100644 index 0000000000..481b36cc8d --- /dev/null +++ b/tools/mtmd/models/mimo-audio.cpp @@ -0,0 +1,218 @@ +#include "models.h" + +ggml_cgraph * clip_graph_mimo_audio::build() { + ggml_tensor * inp = build_inp_raw(1); // [n_frames, n_mel, 1] + + ggml_tensor * cur = ggml_conv_1d_ph(ctx0, model.conv1d_1_w, inp, 1, 1); + cur = ggml_add(ctx0, cur, model.conv1d_1_b); + cur = ggml_gelu_erf(ctx0, cur); + + cur = ggml_conv_1d_ph(ctx0, model.conv1d_2_w, cur, 2, 1); + cur = ggml_add(ctx0, cur, model.conv1d_2_b); + cur = ggml_gelu_erf(ctx0, cur); + + ggml_tensor * inpL = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [n_embd, n_pos] + const int64_t n_pos = inpL->ne[1]; + cb(inpL, "after_conv1d", -1); + + GGML_ASSERT((int) hparams.wa_pattern_mode.size() == n_layer); + + ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos); + ggml_set_name(inp_pos, "mimo_audio_positions"); + ggml_set_input(inp_pos); + + ggml_tensor * full_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos); + ggml_set_name(full_mask, "mimo_audio_full_mask"); + ggml_set_input(full_mask); + + ggml_tensor * window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos); + ggml_set_name(window_mask, "mimo_audio_window_mask"); + ggml_set_input(window_mask); + + build_vit_opts opts; + opts.attn_mask_layers.resize(n_layer); + for (int il = 0; il < n_layer; il++) { + opts.attn_mask_layers[il] = hparams.wa_pattern_mode[il] == -1 ? full_mask : window_mask; + } + // the skip connection below must be added before the post-transformer norm, + // so build_vit must not apply that norm itself + opts.skip_post_ln = true; + + // encoder_skip_layer_id=3 (1-indexed) -> capture output of layer index 2 + const int skip_capture_il = 2; + GGML_ASSERT(n_layer > skip_capture_il); + ggml_tensor * skip_hidden = nullptr; + opts.callback_layer_out = [&](ggml_tensor * layer_cur, int il) { + if (il == skip_capture_il) { + skip_hidden = layer_cur; + } + }; + + auto add_pos = [&](ggml_tensor * x, const clip_layer &) { + return ggml_rope_ext(ctx0, x, inp_pos, nullptr, d_head, + GGML_ROPE_TYPE_NEOX, 0, hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + }; + + inpL = build_vit(inpL, n_pos, NORM_TYPE_NORMAL, hparams.ffn_op, nullptr, add_pos, opts); + inpL = ggml_reshape_2d(ctx0, inpL, n_embd, n_pos); // build_vit restores a (size-1) batch dim + + GGML_ASSERT(skip_hidden != nullptr); + inpL = ggml_add(ctx0, inpL, skip_hidden); + + inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, NORM_TYPE_NORMAL, eps, -1); + cb(inpL, "after_transformer", -1); + + // downsample: strided conv (no bias) + gelu + layernorm + { + ggml_tensor * ds = ggml_cont(ctx0, ggml_transpose(ctx0, inpL)); // [n_pos, n_embd] + ds = ggml_conv_1d(ctx0, model.downsample_conv_w, ds, 2, 0, 1); + ds = ggml_gelu_erf(ctx0, ds); + ds = ggml_cont(ctx0, ggml_transpose(ctx0, ds)); // [n_embd, n_pos/2] + ds = build_norm(ds, model.downsample_norm_w, model.downsample_norm_b, NORM_TYPE_NORMAL, eps, -1); + inpL = ds; + } + cb(inpL, "after_downsample", -1); + + // RVQ quantize: codebook ne=[dim, max_bins, n_q] + // quantize input vector to codes (type=I32) + std::vector codes; + { + GGML_ASSERT(model.rvq_codebook != nullptr); + const int64_t dim = model.rvq_codebook->ne[0]; + GGML_ASSERT(dim == inpL->ne[0]); + GGML_ASSERT((int64_t) hparams.rvq_codebook_size.size() == model.rvq_codebook->ne[2]); + + ggml_tensor * residual = inpL; // [dim, n_pos_ds] + + for (size_t q = 0; q < hparams.rvq_codebook_size.size(); q++) { + const int64_t bins = hparams.rvq_codebook_size[q]; + ggml_tensor * codebook_q = ggml_view_2d(ctx0, model.rvq_codebook, dim, bins, + model.rvq_codebook->nb[1], q * model.rvq_codebook->nb[2]); + codebook_q = ggml_cont(ctx0, codebook_q); + + ggml_tensor * codebook_norm = ggml_sum_rows(ctx0, ggml_sqr(ctx0, codebook_q)); // [1, bins] + codebook_norm = ggml_cont(ctx0, ggml_transpose(ctx0, codebook_norm)); // [bins, 1] + + ggml_tensor * dot = ggml_mul_mat(ctx0, codebook_q, residual); // [bins, n_pos_ds] + ggml_tensor * scores = ggml_sub(ctx0, ggml_scale(ctx0, dot, 2.0f), codebook_norm); + + ggml_tensor * idx = ggml_argmax(ctx0, scores); // [n_pos_ds] + codes.push_back(idx); + + ggml_tensor * quant = ggml_get_rows(ctx0, codebook_q, idx); // [dim, n_pos_ds] + residual = ggml_sub(ctx0, residual, quant); + cb(idx, "rvq_code", (int) q); + } + } + + // convert codes to LLM embeddings + ggml_tensor * code_embd_sum = nullptr; + { + GGML_ASSERT(model.mm_a_code_embd != nullptr); + const int64_t dim = model.mm_a_code_embd->ne[0]; + const int64_t vocab = model.mm_a_code_embd->ne[1]; + GGML_ASSERT((int64_t) codes.size() == model.mm_a_code_embd->ne[2]); + GGML_ASSERT(dim == inpL->ne[0]); + + for (size_t i = 0; i < codes.size(); i++) { + ggml_tensor * table_i = ggml_view_2d(ctx0, model.mm_a_code_embd, dim, vocab, + model.mm_a_code_embd->nb[1], i * model.mm_a_code_embd->nb[2]); + table_i = ggml_cont(ctx0, table_i); + + ggml_tensor * embd_i = ggml_get_rows(ctx0, table_i, codes[i]); // [dim, n_pos_ds] + code_embd_sum = code_embd_sum ? ggml_add(ctx0, code_embd_sum, embd_i) : embd_i; + } + cb(code_embd_sum, "code_embd_sum", -1); + } + + // input_local_transformer + // groups of `group_size` consecutive downsampled frames are processed together, attending only within their own group. + // Implemented as a block-diagonal mask + in-group-repeating positions + // (rather than a real batch dim) - same technique as the encoder's masks above, and as gemma4a's / deepseekocr2's chunked attention. + + // note: hand-rolled here instead of build_vit() because this is a second, independent layer stack + // (own layer array/count, RMSNorm instead of LN, SiLU FFN, own RoPE theta) + + ggml_tensor * projected; + { + const int group_size = hparams.audio_local_group_size; + GGML_ASSERT(group_size > 0); + const int64_t n_pos_ds = code_embd_sum->ne[1]; + const int64_t n_groups = (n_pos_ds + group_size - 1) / group_size; + const int64_t n_padded = n_groups * group_size; + + ggml_tensor * cur_local = code_embd_sum; + if (n_padded != n_pos_ds) { + cur_local = ggml_pad(ctx0, cur_local, 0, (int) (n_padded - n_pos_ds), 0, 0); + } + + ggml_tensor * local_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_padded); + ggml_set_name(local_pos, "mimo_audio_local_positions"); + ggml_set_input(local_pos); + + ggml_tensor * local_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_padded, n_padded); + ggml_set_name(local_mask, "mimo_audio_local_mask"); + ggml_set_input(local_mask); + + const float local_rope_theta = 640000.0f; // audio_config.rope_theta (differs from the encoder's) + auto apply_local_rope = [&](ggml_tensor * x) { + return ggml_rope_ext(ctx0, x, local_pos, nullptr, d_head, + GGML_ROPE_TYPE_NEOX, 0, local_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + }; + + for (int il = 0; il < hparams.audio_local_n_layer; il++) { + auto & layer = model.mm_a_local_layers[il]; + + ggml_tensor * attn_in = build_norm(cur_local, layer.ln_1_w, nullptr, NORM_TYPE_RMS, eps, il); + + ggml_tensor * Qcur = build_mm(layer.q_w, attn_in); + if (layer.q_b) { + Qcur = ggml_add(ctx0, Qcur, layer.q_b); + } + ggml_tensor * Kcur = build_mm(layer.k_w, attn_in); + if (layer.k_b) { + Kcur = ggml_add(ctx0, Kcur, layer.k_b); + } + ggml_tensor * Vcur = build_mm(layer.v_w, attn_in); + if (layer.v_b) { + Vcur = ggml_add(ctx0, Vcur, layer.v_b); + } + + Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_padded); + Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_padded); + Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_padded); + + Qcur = apply_local_rope(Qcur); + Kcur = apply_local_rope(Kcur); + + ggml_tensor * attn_out = build_attn(layer.o_w, nullptr, Qcur, Kcur, Vcur, local_mask, kq_scale, il); + cur_local = ggml_add(ctx0, cur_local, attn_out); + + ggml_tensor * ffn_in = build_norm(cur_local, layer.ln_2_w, nullptr, NORM_TYPE_RMS, eps, il); + ggml_tensor * ffn_out = build_ffn(ffn_in, + layer.ff_up_w, nullptr, + layer.ff_gate_w, nullptr, + layer.ff_down_w, nullptr, + FFN_SILU, il); + cur_local = ggml_add(ctx0, cur_local, ffn_out); + } + + cur_local = build_norm(cur_local, model.mm_a_local_norm_w, nullptr, NORM_TYPE_RMS, eps, -1); + cb(cur_local, "after_local_transformer", -1); + + // flatten each group of `group_size` frames into one (group_size*n_embd)-dim vector + // (matching AudioProjection's flattened input) + ggml_tensor * grouped = ggml_reshape_2d(ctx0, cur_local, n_embd * group_size, n_groups); + + // AudioProjection: Linear (no bias) -> GELU -> Linear (no bias) + projected = build_ffn(grouped, + model.mm_1_w, nullptr, + nullptr, nullptr, + model.mm_2_w, nullptr, + FFN_GELU_ERF, -1); + cb(projected, "after_projection", -1); + } + + ggml_build_forward_expand(gf, projected); + return gf; +} diff --git a/tools/mtmd/models/minimax-m3.cpp b/tools/mtmd/models/minimax-m3.cpp new file mode 100644 index 0000000000..447621754e --- /dev/null +++ b/tools/mtmd/models/minimax-m3.cpp @@ -0,0 +1,84 @@ +#include "models.h" + +ggml_tensor * clip_graph_minimax_m3::apply_rope( + ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) { + const int64_t Hn = x->ne[1]; + const int64_t P = x->ne[2]; + const size_t es = ggml_element_size(x); + const int dh = (int) x->ne[0]; + const int axd = 2 * ((2 * (dh / 2) / 3) / 2); + + GGML_ASSERT(x->nb[0] == es); + GGML_ASSERT(3 * axd <= dh); + + const float th = hparams.rope_theta; + + // layout of x is [t, h, w, pad] + // t is unrotated, h and w are rotated, pad is unrotated + // note: everything from n_dims onward untouched, so w and pad are rotated in one call. + auto sl = [&](int off, int n) { + return ggml_cont(ctx0, ggml_view_3d(ctx0, x, n, Hn, P, x->nb[1], x->nb[2], (size_t) off * es)); + }; + ggml_tensor * t = sl(0, axd); + ggml_tensor * h = sl(axd, axd); + ggml_tensor * w = sl(2 * axd, dh - 2 * axd); // w + pad + + h = ggml_rope_ext(ctx0, h, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + w = ggml_rope_ext(ctx0, w, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + return ggml_concat(ctx0, ggml_concat(ctx0, t, h, 0), w, 0); +} + +ggml_cgraph * clip_graph_minimax_m3::build() { + GGML_ASSERT(model.patch_bias == nullptr); + GGML_ASSERT(model.class_embedding == nullptr); + GGML_ASSERT(model.patch_embeddings_0 && model.patch_embeddings_1); + GGML_ASSERT(model.mm_1_w && model.mm_2_w); + GGML_ASSERT(model.mm_merger_fc1_w && model.mm_merger_fc2_w); + + const int batch_size = 1; + const int n_pos = n_patches; + const int merge = hparams.n_merge; + + // patch embedding + ggml_tensor * inp_raw = build_inp_raw(); + ggml_tensor * inp = ggml_add(ctx0, + ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1), + ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1)); + + // spatial merge + { + inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); + inp = ggml_cont_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, n_patches_y, batch_size); + inp = ggml_reshape_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, merge, batch_size * (n_patches_y / merge)); + inp = ggml_permute(ctx0, inp, 0, 2, 1, 3); + inp = ggml_cont_3d(ctx0, inp, n_embd, n_patches_x * n_patches_y, batch_size); + } + + // t (time axis) is always 0 for now, so we leave it unrotated + ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos); + ggml_set_name(pos_h, "minimax_pos_h"); ggml_set_input(pos_h); + ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos); + ggml_set_name(pos_w, "minimax_pos_w"); ggml_set_input(pos_w); + + ggml_tensor * inpL = build_vit( + inp, n_pos, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr, + [&](ggml_tensor * c, const clip_layer &) { + return apply_rope(c, pos_h, pos_w); + }); + + // projector + ggml_tensor * emb = inpL; + emb = build_ffn(emb, model.mm_1_w, model.mm_1_b, + nullptr, nullptr, + model.mm_2_w, model.mm_2_b, FFN_GELU_ERF, -1); + + const int64_t proj = emb->ne[0]; + emb = ggml_reshape_2d(ctx0, emb, proj * merge * merge, n_pos / (merge * merge)); + + emb = build_ffn(emb, model.mm_merger_fc1_w, model.mm_merger_fc1_b, + nullptr, nullptr, + model.mm_merger_fc2_w, model.mm_merger_fc2_b, FFN_GELU_ERF, -1); + + ggml_build_forward_expand(gf, emb); + return gf; +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index 5f1493fa60..e54366a086 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -40,6 +40,12 @@ struct clip_graph_qwen3vl : clip_graph_qwen2vl { ggml_cgraph * build() override; }; +struct clip_graph_minimax_m3 : clip_graph { + clip_graph_minimax_m3(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; + ggml_tensor * apply_rope(ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w); +}; + struct clip_graph_mimovl : clip_graph { clip_graph_mimovl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; @@ -204,6 +210,11 @@ struct clip_graph_qwen3a : clip_graph { ggml_cgraph * build() override; }; +struct clip_graph_mimo_audio : clip_graph { + clip_graph_mimo_audio(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; +}; + struct clip_graph_kimik25 : clip_graph { clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; @@ -211,6 +222,11 @@ struct clip_graph_kimik25 : clip_graph { ggml_tensor * resize_position_embeddings_3d(uint32_t interpolation_mode); }; +struct clip_graph_parakeet : clip_graph { + clip_graph_parakeet(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} + ggml_cgraph * build() override; +}; + struct clip_graph_exaone4_5 : clip_graph { clip_graph_exaone4_5(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/models/parakeet.cpp b/tools/mtmd/models/parakeet.cpp new file mode 100644 index 0000000000..8be141d93b --- /dev/null +++ b/tools/mtmd/models/parakeet.cpp @@ -0,0 +1,421 @@ +#include "models.h" + +static constexpr int PARAKEET_LOCAL_ATTN_THRESHOLD = 8192; +static constexpr int PARAKEET_LOCAL_ATTN_WINDOW = 128; + +// conv subsampling + conformer encoder +ggml_cgraph * clip_graph_parakeet::build() { + + // Conv subsampling + ggml_tensor * inp = build_inp_raw(1); + inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); + + // [freq, time, channels, batch] + ggml_tensor * cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[0], inp, 2, 2, 1, 1, 1, 1); + cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[0]); + cb(cur, "pre_conv_0", -1); + + cur = ggml_relu(ctx0, cur); + cb(cur, "pre_conv_0_relu", -1); + + // [freq, time, channels, batch] + cur = ggml_conv_2d_dw_direct(ctx0, model.pre_encode_conv_X_w[2], cur, 2, 2, 1, 1, 1, 1); + cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[2]); + cb(cur, "pre_conv_2", -1); + + // [freq, time, channels, batch] + cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[3], cur, 1, 1, 0, 0, 1, 1); + cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[3]); + cb(cur, "pre_conv_3", -1); + + cur = ggml_relu(ctx0, cur); + cb(cur, "pre_conv_3_relu", -1); + + // [freq, time, channels, batch] + cur = ggml_conv_2d_dw_direct(ctx0, model.pre_encode_conv_X_w[5], cur, 2, 2, 1, 1, 1, 1); + cb(cur, "pre_conv_5_direct", -1); + cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[5]); + cb(cur, "pre_conv_5", -1); + + // [freq, time, channels, batch] + cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[6], cur, 1, 1, 0, 0, 1, 1); + cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[6]); + cb(cur, "pre_conv_6", -1); + + cur = ggml_relu(ctx0, cur); + cb(cur, "pre_conv_6_relu", -1); + + // [freq, time, chan] + cur = ggml_permute(ctx0, cur, 0, 2, 1, 3); + // [freq, chan, time] + cur = ggml_cont(ctx0, cur); + + const int n_freq = cur->ne[0]; + const int n_chan = cur->ne[1]; + const int n_frames = cur->ne[2]; + + // [freq, time, chan, batch] -> [(freq * chan), time] + cur = ggml_reshape_2d(ctx0, cur, n_freq * n_chan, n_frames); + + cur = build_mm(model.pre_encode_out_w, cur); + cur = ggml_add(ctx0, cur, model.pre_encode_out_b); + + ggml_set_name(cur, "pre_enc_out"); + + // Encoder + + const auto & hparams = model.hparams; + const int n_layer = hparams.n_layer; + const int n_state = hparams.n_embd; + const float fc_factor = 0.5f; + + const int n_time = cur->ne[1]; + const bool local_attn = n_time > PARAKEET_LOCAL_ATTN_THRESHOLD; + const int att_left = local_attn ? PARAKEET_LOCAL_ATTN_WINDOW : n_time - 1; + const int att_right = local_attn ? PARAKEET_LOCAL_ATTN_WINDOW : n_time - 1; + const int window_size = local_attn ? att_left + att_right + 1 : 2 * n_time - 1; + const int d_half = n_state / 2; + const int mask_dim = local_attn ? window_size : n_time; + + // mask [key, n_time] + struct ggml_tensor * attn_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, mask_dim, n_time); + ggml_set_name(attn_mask, "attn_mask"); + ggml_set_input(attn_mask); + + struct ggml_tensor * local_mask = nullptr; + if (local_attn) { + const int chunk = att_left + att_right; + local_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, chunk + window_size - 1, chunk); + ggml_set_name(local_mask, "local_mask"); + ggml_set_input(local_mask); + } + + struct ggml_tensor * pos_freqs = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, d_half); + ggml_set_name(pos_freqs, "pos_freqs"); + ggml_set_input(pos_freqs); + + struct ggml_tensor * rel_positions = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, window_size); + ggml_set_name(rel_positions, "rel_positions"); + ggml_set_input(rel_positions); + + struct ggml_tensor * freqs = ggml_repeat_4d(ctx0, pos_freqs, d_half, window_size, 1, 1); + struct ggml_tensor * theta = ggml_mul(ctx0, freqs, rel_positions); + + struct ggml_tensor * sin = ggml_reshape_3d(ctx0, ggml_sin(ctx0, theta), 1, d_half, window_size); + struct ggml_tensor * cos = ggml_reshape_3d(ctx0, ggml_cos(ctx0, theta), 1, d_half, window_size); + struct ggml_tensor * pos_emb = ggml_reshape_2d(ctx0, ggml_cont(ctx0, ggml_concat(ctx0, sin, cos, 0)), n_state, window_size); + ggml_set_name(pos_emb, "pos_emb"); + + for (int il = 0; il < n_layer; ++il) { + const auto & layer = model.layers[il]; + // FFN1 + { + struct ggml_tensor * residual = cur; + ggml_format_name(cur, "enc_%d_res", il); + + // norm + cur = ggml_norm(ctx0, cur, hparams.eps); + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ff_norm_w), layer.ff_norm_b); + ggml_format_name(cur, "enc_%d_ffn_norm_1", il); + + cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, FFN_SILU, il); + ggml_format_name(cur, "enc_%d_ffn_1", il); + + cur = ggml_add(ctx0, residual, ggml_scale(ctx0, cur, fc_factor)); + ggml_format_name(cur, "enc_%d_res_ffn", il); + } + + // self attention block using relative positional encoding from model.position_embedding. + { + // [feat, time_frames, 1, 1] + struct ggml_tensor * residual = cur; + + cur = ggml_norm(ctx0, cur, hparams.eps); + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ln_1_w), layer.ln_1_b); + ggml_format_name(cur, "enc_%d_attn_norm", il); + + const int n_head = hparams.n_head; + const int d_head = n_state / n_head; + + // [feat, time_frames, 1, 1] + struct ggml_tensor * Q_cur = build_mm(layer.q_w, cur); + struct ggml_tensor * K_cur = build_mm(layer.k_w, cur); + struct ggml_tensor * V_cur = build_mm(layer.v_w, cur); + + // [d_head, n_heads, n_time, 1] + Q_cur = ggml_reshape_3d(ctx0, Q_cur, d_head, n_head, n_time); + K_cur = ggml_reshape_3d(ctx0, K_cur, d_head, n_head, n_time); + V_cur = ggml_reshape_3d(ctx0, V_cur, d_head, n_head, n_time); + + // [n_state, window_size] + struct ggml_tensor * pos = build_mm(layer.linear_pos_w, pos_emb); + // [feat, head, window_size, 1] + pos = ggml_reshape_3d(ctx0, pos, d_head, n_head, pos_emb->ne[1]); + // [feat, window_size, head, 1] + pos = ggml_cont(ctx0, ggml_permute(ctx0, pos, 0, 2, 1, 3)); + ggml_format_name(pos, "enc_%d_attn_pos", il); + + if (local_attn) { + const int chunk = att_left + att_right; + const int n_group = (n_time + chunk - 1) / chunk; + const int n_time_padded = n_group * chunk; + const int n_kv_chunk = chunk + window_size - 1; + const int n_kv_dense = n_kv_chunk * n_group; + const bool need_padding = n_time_padded > n_time; + + Q_cur = ggml_cont(ctx0, ggml_permute(ctx0, Q_cur, 0, 2, 1, 3)); + K_cur = ggml_cont(ctx0, ggml_permute(ctx0, K_cur, 0, 2, 1, 3)); + V_cur = ggml_cont(ctx0, ggml_permute(ctx0, V_cur, 0, 2, 1, 3)); + + // content bias + struct ggml_tensor * bias_u = ggml_reshape_3d(ctx0, layer.pos_bias_u, d_head, 1, n_head); + struct ggml_tensor * Q_u = ggml_add(ctx0, Q_cur, bias_u); + + // position bias + struct ggml_tensor * bias_v = ggml_reshape_3d(ctx0, layer.pos_bias_v, d_head, 1, n_head); + struct ggml_tensor * Q_v = ggml_add(ctx0, Q_cur, bias_v); + + // right pad the time dimension + struct ggml_tensor * Q_u_padded = need_padding ? + ggml_pad_ext(ctx0, Q_u, 0, 0, 0, n_time_padded - n_time, 0, 0, 0, 0) : Q_u; + Q_u_padded = ggml_reshape_4d(ctx0, Q_u_padded, d_head, chunk, n_group, n_head); + + // pad front and back for the first and last time frames + struct ggml_tensor * K_padded = ggml_pad_ext(ctx0, K_cur, 0, 0, att_left, att_right, 0, 0, 0, 0); + if (n_kv_dense > K_padded->ne[1]) { + K_padded = ggml_pad_ext(ctx0, K_padded, 0, 0, 0, n_kv_dense - K_padded->ne[1], 0, 0, 0, 0); + } + + // sliding window view: each group spans n_kv_chunk keys but steps by chunk + struct ggml_tensor * K_chunk = ggml_view_4d(ctx0, K_padded, + d_head, n_kv_chunk, n_group, n_head, + K_padded->nb[1], + (size_t) chunk * K_padded->nb[1], + K_padded->nb[2], + 0); + K_chunk = ggml_cont(ctx0, K_chunk); + + struct ggml_tensor * content_scores = ggml_mul_mat(ctx0, K_chunk, Q_u_padded); + + // trim the dense output down to window_size scores per query + content_scores = ggml_view_4d(ctx0, content_scores, + window_size, chunk, n_group, n_head, + (size_t) (chunk + window_size) * content_scores->nb[0], + content_scores->nb[2], + content_scores->nb[3], + 0); + content_scores = ggml_cont(ctx0, content_scores); + + // ungroup: [window_size, n_time_padded, n_head] + content_scores = ggml_reshape_3d(ctx0, content_scores, window_size, n_time_padded, n_head); + if (need_padding) { + content_scores = ggml_view_3d(ctx0, content_scores, + window_size, n_time, n_head, + content_scores->nb[1], + content_scores->nb[2], + 0); + } + + // Q_v: [d_head, time, head] + Q_v = ggml_cont(ctx0, ggml_permute(ctx0, Q_v, 0, 2, 1, 3)); + struct ggml_tensor * rel_pos_scores = ggml_mul_mat(ctx0, pos, Q_v); + + struct ggml_tensor * attn_scores = ggml_add(ctx0, content_scores, rel_pos_scores); + attn_scores = ggml_soft_max_ext(ctx0, attn_scores, attn_mask, 1.0f / std::sqrt(d_head), 0.0f); + ggml_format_name(attn_scores, "enc_%d_attn_probs", il); + + // expand probs back to n_kv_chunk width for the V matmul + struct ggml_tensor * probs_padded = need_padding ? + ggml_pad_ext(ctx0, attn_scores, 0, 0, 0, n_time_padded - n_time, 0, 0, 0, 0) : attn_scores; + + probs_padded = ggml_reshape_4d(ctx0, probs_padded, window_size, chunk, n_group, n_head); + probs_padded = ggml_pad_ext(ctx0, probs_padded, 0, chunk, 0, 0, 0, 0, 0, 0); + probs_padded = ggml_view_4d(ctx0, probs_padded, + n_kv_chunk, chunk, n_group, n_head, + (size_t) n_kv_chunk * probs_padded->nb[0], + probs_padded->nb[2], + probs_padded->nb[3], + 0); + probs_padded = ggml_cont(ctx0, probs_padded); + probs_padded = ggml_mul(ctx0, probs_padded, local_mask); + + struct ggml_tensor * V_padded = ggml_pad_ext(ctx0, V_cur, 0, 0, att_left, att_right, 0, 0, 0, 0); + if (n_kv_dense > V_padded->ne[1]) { + V_padded = ggml_pad_ext(ctx0, V_padded, 0, 0, 0, n_kv_dense - V_padded->ne[1], 0, 0, 0, 0); + } + V_padded = ggml_cont(ctx0, ggml_transpose(ctx0, V_padded)); + + struct ggml_tensor * V_chunk = ggml_view_4d(ctx0, V_padded, + n_kv_chunk, d_head, n_group, n_head, + V_padded->nb[1], + (size_t) chunk * V_padded->nb[0], + V_padded->nb[2], + 0); + V_chunk = ggml_cont(ctx0, V_chunk); + + cur = ggml_mul_mat(ctx0, V_chunk, probs_padded); + cur = ggml_reshape_3d(ctx0, cur, d_head, n_time_padded, n_head); + if (need_padding) { + cur = ggml_view_3d(ctx0, cur, d_head, n_time, n_head, cur->nb[1], cur->nb[2], 0); + } + cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3)); + cur = ggml_reshape_2d(ctx0, cur, n_state, n_time); + cur = build_mm(layer.o_w, cur); + } else { + // full attention + struct ggml_tensor * Q_u = ggml_add(ctx0, Q_cur, layer.pos_bias_u); + ggml_format_name(Q_u, "enc_%d_attn_q_u", il); + + struct ggml_tensor * K_prep = ggml_permute(ctx0, K_cur, 0, 2, 1, 3); + struct ggml_tensor * Q_prep = ggml_permute(ctx0, Q_u, 0, 2, 1, 3); + struct ggml_tensor * content_scores = ggml_mul_mat(ctx0, K_prep, Q_prep); + ggml_format_name(content_scores, "enc_%d_attn_content_scores", il); + + struct ggml_tensor * Q_v = ggml_add(ctx0, Q_cur, layer.pos_bias_v); + ggml_format_name(Q_v, "enc_%d_attn_q_v", il); + + Q_v = ggml_permute(ctx0, Q_v, 0, 2, 1, 3); + Q_v = ggml_cont(ctx0, Q_v); + ggml_format_name(Q_v, "enc_%d_attn_q_v_perm", il); + + struct ggml_tensor * rel_pos_scores = ggml_mul_mat(ctx0, pos, Q_v); + ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos", il); + + // Relative positional shift + { + const auto pos_window = rel_pos_scores->ne[0]; + const auto n_frame = rel_pos_scores->ne[1]; + const auto n_head = rel_pos_scores->ne[2]; + + rel_pos_scores = ggml_pad(ctx0, rel_pos_scores, 1, 0, 0, 0); + rel_pos_scores = ggml_roll(ctx0, rel_pos_scores, 1, 0, 0, 0); + + rel_pos_scores = ggml_reshape_3d(ctx0, rel_pos_scores, n_frame, pos_window + 1, n_head); + rel_pos_scores = ggml_cont(ctx0, rel_pos_scores); + ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos_reshaped", il); + + int center = pos_window / 2; + size_t offset = rel_pos_scores->nb[0] * (center+1); + + rel_pos_scores = ggml_view_3d(ctx0, rel_pos_scores, + n_frame, pos_window, n_head, + (pos_window) * 4, + rel_pos_scores->nb[2], + offset); + rel_pos_scores = ggml_cont(ctx0, rel_pos_scores); + ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos_shifted", il); + + rel_pos_scores = ggml_view_3d(ctx0, rel_pos_scores, + content_scores->ne[0], + content_scores->ne[1], + rel_pos_scores->ne[2], + rel_pos_scores->nb[1], + rel_pos_scores->nb[2], + 0); + rel_pos_scores = ggml_cont(ctx0, rel_pos_scores); + ggml_format_name(rel_pos_scores, "enc_%d_attn_rel_pos_shifted_view", il); + } + + struct ggml_tensor * attn_scores = ggml_add(ctx0, content_scores, rel_pos_scores); + ggml_format_name(attn_scores, "enc_%d_attn_scores", il); + attn_scores = ggml_scale(ctx0, attn_scores, 1.0f / std::sqrt(d_head)); + attn_scores = ggml_add(ctx0, attn_scores, attn_mask); + ggml_format_name(attn_scores, "enc_%d_attn_scores_scaled", il); + + struct ggml_tensor * probs = ggml_soft_max(ctx0, attn_scores); + ggml_format_name(probs, "enc_%d_attn_probs", il); + + V_cur = ggml_cont(ctx0, ggml_permute(ctx0, V_cur, 1, 2, 0, 3)); + ggml_format_name(V_cur, "enc_%d_attn_v_cur", il); + cur = ggml_mul_mat(ctx0, probs, V_cur); + ggml_format_name(cur, "enc_%d_attn_inp", il); + + cur = ggml_permute(ctx0, cur, 2, 0, 1, 3); + cur = ggml_cont_2d(ctx0, cur, n_state, n_time); + cur = build_mm(layer.o_w, cur); + } + ggml_format_name(cur, "enc_%d_attn_out", il); + + cur = ggml_add(ctx0, residual, cur); + ggml_format_name(cur, "enc_%d_attn_res", il); + } + + // Convolution + { + struct ggml_tensor * residual = cur; + ggml_format_name(cur, "enc_%d_residual_conv", il); + + cur = ggml_norm(ctx0, cur, hparams.eps); + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.norm_conv_w), layer.norm_conv_b); + ggml_format_name(cur, "enc_%d_norm_conv", il); + + // pointwise 1d convolution: + cur = build_mm(layer.conv_pw1_w, cur); + ggml_format_name(cur, "enc_%d_conv_pw1", il); + + { + int64_t d = cur->ne[0] / 2; + struct ggml_tensor * signal = ggml_view_2d(ctx0, cur, d, cur->ne[1], cur->nb[1], 0); + struct ggml_tensor * gate = ggml_view_2d(ctx0, cur, d, cur->ne[1], cur->nb[1], d * cur->nb[0]); + + cur = ggml_mul(ctx0, signal, ggml_sigmoid(ctx0, gate)); + ggml_format_name(cur, "enc_%d_conv_glu", il); + } + + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + + // use ggml_ssm_conv for f32 precision + const int dw_pad = (hparams.audio_conv_kernel_size - 1) / 2; + cur = ggml_pad(ctx0, cur, dw_pad, 0, 0, 0); + cur = ggml_roll(ctx0, cur, dw_pad, 0, 0, 0); + cur = ggml_pad(ctx0, cur, dw_pad, 0, 0, 0); + ggml_format_name(cur, "enc_%d_conv_dw_pad", il); + + cur = ggml_ssm_conv(ctx0, cur, layer.conv_dw_w); + ggml_format_name(cur, "enc_%d_conv_1d_dw", il); + + cur = ggml_sub(ctx0, cur, layer.conv_norm_mean); + struct ggml_tensor * std = ggml_sqrt(ctx0, layer.conv_norm_var); + cur = ggml_div(ctx0, cur, std); + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.conv_norm_w), layer.conv_norm_b); + ggml_format_name(cur, "enc_%d_conv_bn", il); + + cur = ggml_silu(ctx0, cur); + ggml_format_name(cur, "enc_%d_conv_silu", il); + + cur = build_mm(layer.conv_pw2_w, cur); + ggml_format_name(cur, "enc_%d_conv_pw2", il); + + cur = ggml_add(ctx0, residual, cur); + ggml_format_name(cur, "enc_%d_conv_res", il); + } + + // FFN2 + { + struct ggml_tensor * residual = cur; + cur = ggml_norm(ctx0, cur, hparams.eps); + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ff_norm_1_w), layer.ff_norm_1_b); + ggml_format_name(cur, "enc_%d_ffn_norm_2", il); + + cur = build_ffn(cur, layer.ff_up_1_w, nullptr, nullptr, nullptr, layer.ff_down_1_w, nullptr, FFN_SILU, il); + cur = ggml_add(ctx0, residual, ggml_scale(ctx0, cur, 0.5)); + ggml_format_name(cur, "enc_%d_ffn_res", il); + } + + cur = ggml_norm(ctx0, cur, hparams.eps); + cur = ggml_add(ctx0, ggml_mul(ctx0, cur, layer.ln_2_w), layer.ln_2_b); + } + + cb(cur, "encoder_out", -1); + + cur = ggml_rms_norm(ctx0, cur, 1e-6); + cur = ggml_mul(ctx0, cur, model.mm_norm_pre_w); + cb(cur, "sound_projection.norm", -1); + + cur = build_ffn(cur, model.mm_0_w, model.mm_0_b, nullptr, nullptr, model.mm_1_w, model.mm_1_b, FFN_RELU_SQR, -1); + cb(cur, "projected", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; +} diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index b72fd067a5..fea03557d0 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -725,6 +725,72 @@ bool mtmd_audio_preprocessor_qwen3a::preprocess(const float * sa return true; } +// +// mtmd_audio_preprocessor_mimo_audio +// +// Matches torchaudio.transforms.MelSpectrogram(power=1.0, center=True) followed by +// log(clip(spec, min=1e-7)): HTK mel scale, no Slaney area norm, magnitude (not power) +// spectrogram, natural log, reflect-padded by n_fft/2 on each side. +// + +void mtmd_audio_preprocessor_mimo_audio::initialize() { + cache.fill_sin_cos_table(hparams.audio_n_fft); + cache.fill_hann_window(hparams.audio_window_len, true); + cache.fill_mel_filterbank_matrix( + hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate, + 0.0f, hparams.audio_sample_rate / 2.0f, + /*slaney_area_norm=*/ false, + /*scale=*/ 1.0f, + /*use_htk=*/ true + ); +} + +bool mtmd_audio_preprocessor_mimo_audio::preprocess(const float * samples, + size_t n_samples, + std::vector & output) { + if (n_samples == 0) { + return false; + } + + GGML_ASSERT(!cache.sin_vals.empty()); + GGML_ASSERT(!cache.cos_vals.empty()); + GGML_ASSERT(!cache.filters.data.empty()); + + const int pad = hparams.audio_n_fft / 2; + + std::vector padded(n_samples + 2 * pad, 0.0f); + for (int i = 0; i < pad; i++) { + int src = pad - i; + padded[i] = (src < (int)n_samples) ? samples[src] : 0.0f; + } + std::copy(samples, samples + n_samples, padded.begin() + pad); + for (int i = 0; i < pad; i++) { + int src = (int)n_samples - 2 - i; + padded[n_samples + pad + i] = (src >= 0) ? samples[src] : 0.0f; + } + + filter_params params; + params.n_mel = hparams.n_mel_bins; + params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + params.hann_window_size = hparams.audio_window_len; + params.hop_length = hparams.audio_hop_len; + params.sample_rate = hparams.audio_sample_rate; + params.no_padding = true; // reflect padding already applied above + params.use_natural_log = true; + params.use_magnitude = true; + params.mel_floor = 1e-7f; + params.norm_per_feature = false; + + mtmd_audio_mel out; + bool ok = log_mel_spectrogram(padded.data(), (int)padded.size(), 4, params, cache, out); + if (!ok) { + return false; + } + + output.push_back(std::move(out)); + return true; +} + // // mtmd_audio_preprocessor_conformer // @@ -956,6 +1022,209 @@ bool mtmd_audio_preprocessor_gemma4a::preprocess(const float * s } // +// mtmd_audio_preprocessor_parakeet implementation +// + +void mtmd_audio_preprocessor_parakeet::worker_thread( + int ith, + const float * window_func, + int window_size, + const std::vector & samples, + int n_samples, + int frame_size, + int frame_step, + int n_threads, + int n_fft_bins, + const mtmd_audio_cache & cache, + mtmd_audio_mel & mel) { + std::vector fft_in(frame_size * 2, 0.0); + std::vector fft_out(frame_size * 2 * 2 * 2); + + int n_fb = n_fft_bins; + int i = ith; + + GGML_ASSERT(n_fb == 1 + (frame_size / 2)); + + const double eps = 5.960464477539063e-08; + + for (; i < std::min(n_samples / frame_step + 1, (int) mel.n_len); i += n_threads) { + const int offset = i * frame_step; + const int window_pad_left = (frame_size - window_size) / 2; + + // Zero-pad left. + std::fill(fft_in.begin(), fft_in.begin() + window_pad_left, 0.0f); + + // Apply windowed samples in the center. + const int n_to_process = std::min({window_size, n_samples - offset}); + for (int j = 0; j < n_to_process; j++) { + fft_in[window_pad_left + j] = window_func[j] * samples[offset + window_pad_left + j]; + } + + // Zero-pad right. + std::fill(fft_in.begin() + window_pad_left + n_to_process, fft_in.begin() + frame_size, 0.0f); + + // FFT. + fft(cache, fft_in.data(), frame_size, fft_out.data()); + + // Calculate modulus^2 of complex numbers. + for (int j = 0; j < n_fb; j++) { + fft_out[j] = (fft_out[2 * j + 0] * fft_out[2 * j + 0] + fft_out[2 * j + 1] * fft_out[2 * j + 1]); + } + + // mel spectrogram. + for (int j = 0; j < mel.n_mel; j++) { + double sum = 0.0; + int k = 0; + for (k = 0; k < n_fb - 3; k += 4) { + sum += + fft_out[k + 0] * cache.filters.data[j * n_fb + k + 0] + + fft_out[k + 1] * cache.filters.data[j * n_fb + k + 1] + + fft_out[k + 2] * cache.filters.data[j * n_fb + k + 2] + + fft_out[k + 3] * cache.filters.data[j * n_fb + k + 3]; + } + for (; k < n_fb; k++) { + sum += fft_out[k] * cache.filters.data[j * n_fb + k]; + } + mel.data[j * mel.n_len + i] = std::log(sum + eps); + } + } + + // Otherwise fft_out are all zero. + const double empty_sum = std::log(eps); + for (; i < mel.n_len; i += n_threads) { + for (int j = 0; j < mel.n_mel; j++) { + mel.data[j * mel.n_len + i] = empty_sum; + } + } +} + +void mtmd_audio_preprocessor_parakeet::initialize() { + cache.fill_sin_cos_table(hparams.audio_n_fft); + + const size_t n_fft = hparams.audio_n_fft / 2 + 1; + GGML_ASSERT(hparams.mel_filters.size() == (size_t)hparams.n_mel_bins * n_fft); + cache.filters.n_mel = hparams.n_mel_bins; + cache.filters.n_fft = n_fft; + cache.filters.data = hparams.mel_filters; + + GGML_ASSERT(hparams.window.size() == (size_t)hparams.audio_window_len); + GGML_ASSERT(hparams.window.size() <= (size_t) hparams.audio_n_fft); + cache.hann_window = hparams.window; +} + +bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples, + size_t n_samples_in, + std::vector & output) { + if (n_samples_in == 0) { + return false; + } + + filter_params params; + params.n_mel = hparams.n_mel_bins; + params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); + params.hann_window_size = hparams.audio_window_len; + params.hop_length = hparams.audio_hop_len; + params.sample_rate = hparams.audio_sample_rate; + + GGML_ASSERT(!cache.sin_vals.empty()); + GGML_ASSERT(!cache.cos_vals.empty()); + GGML_ASSERT(!cache.filters.data.empty()); + + const float * window_func = cache.hann_window.data(); + const int window_size = params.hann_window_size; + const int frame_size = (params.n_fft_bins - 1) * 2; + const int frame_step = params.hop_length; + + // Apply preemphasis filter (high-pass): x[i] = x[i] - 0.97 * x[i-1] + std::vector samples_preprocessed(samples, samples + n_samples_in); + { + const float preemph = 0.97f; + for (int i = n_samples_in - 1; i > 0; i--) { + samples_preprocessed[i] = samples_preprocessed[i] - preemph * samples_preprocessed[i - 1]; + } + } + + // Parakeet uses centered constant padding + const size_t pad = (size_t)(frame_size / 2); + std::vector samples_padded(n_samples_in + 2 * pad, 0.0f); + std::copy(samples_preprocessed.begin(), samples_preprocessed.end(), samples_padded.begin() + pad); + + mtmd_audio_mel out_full; + out_full.n_mel = params.n_mel; + out_full.n_len = (samples_padded.size() - frame_size) / frame_step + 1; + out_full.n_len_org = out_full.n_len; + out_full.data.resize(out_full.n_mel * out_full.n_len); + + const int n_threads = 4; + std::vector workers(n_threads - 1); + for (int iw = 0; iw < n_threads - 1; ++iw) { + workers[iw] = std::thread( + worker_thread, iw + 1, + window_func, + window_size, + std::cref(samples_padded), + samples_padded.size(), + frame_size, + frame_step, + n_threads, + params.n_fft_bins, + std::cref(cache), + std::ref(out_full) + ); + } + + worker_thread(0, + window_func, + window_size, + samples_padded, + samples_padded.size(), + frame_size, + frame_step, + n_threads, + params.n_fft_bins, + cache, + out_full); + + for (int iw = 0; iw < n_threads - 1; ++iw) { + workers[iw].join(); + } + + // Per-feature normalization (only on valid frames) + { + const double eps = 1e-5; + int valid_frames = n_samples_in / frame_step; + + for (int j = 0; j < out_full.n_mel; j++) { + double sum = 0.0; + double sq_diff_sum = 0.0; + + // Calculate Mean ONLY on valid audio frames + for (int i = 0; i < valid_frames; i++) { + sum += (double)out_full.data[j * out_full.n_len + i]; + } + double mean = sum / valid_frames; + + // Calculate Variance ONLY on valid audio frames + for (int i = 0; i < valid_frames; i++) { + double diff = (double)out_full.data[j * out_full.n_len + i] - mean; + sq_diff_sum += diff * diff; + } + + double std_dev = std::sqrt(sq_diff_sum / (valid_frames - 1.0)); + double denominator = std_dev + eps; + + // Apply to ALL frames (including the padded ones) + for (int i = 0; i < out_full.n_len; i++) { + out_full.data[j * out_full.n_len + i] = (float)((out_full.data[j * out_full.n_len + i] - mean) / denominator); + } + } + } + + output.push_back(std::move(out_full)); + return true; +} + + // mtmd_audio_preprocessor_gemma4ua // diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index ad96bd847c..f65f282d96 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -111,6 +111,30 @@ struct mtmd_audio_preprocessor_qwen3a : mtmd_audio_preprocessor { mtmd_audio_cache cache; }; +struct mtmd_audio_preprocessor_mimo_audio : mtmd_audio_preprocessor { + mtmd_audio_preprocessor_mimo_audio(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {} + void initialize() override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + + private: + mtmd_audio_cache cache; +}; + +struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor { + mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { } + void initialize() override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) override; + + private: + mtmd_audio_cache cache; + + static void worker_thread(int ith, const float * window_func, int window_size, + const std::vector & samples, int n_samples, + int frame_size, int frame_step, int n_threads, + int n_fft_bins, + const mtmd_audio_cache & cache, mtmd_audio_mel & mel); +}; + // // streaming ISTFT - converts spectrogram frames back to audio one frame at a time // diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index 84422c89f3..90451d02eb 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -640,6 +640,7 @@ bool mtmd_helper_support_video(mtmd_context * ctx) { #ifdef MTMD_VIDEO return mtmd_support_vision(ctx); #else + GGML_UNUSED(ctx); return false; #endif } @@ -1007,6 +1008,9 @@ mtmd_helper_video * mtmd_helper_video_init( return ctx; #else + GGML_UNUSED(mctx); + GGML_UNUSED(path); + GGML_UNUSED(params); LOG_ERR("%s: video is not supported in this build (MTMD_VIDEO is set to OFF)\n", __func__); return nullptr; #endif @@ -1039,6 +1043,10 @@ mtmd_helper_video * mtmd_helper_video_init_from_buf( return ctx; #else + GGML_UNUSED(mctx); + GGML_UNUSED(buf); + GGML_UNUSED(len); + GGML_UNUSED(params); LOG_ERR("%s: video is not supported in this build (MTMD_VIDEO is set to OFF)\n", __func__); return nullptr; #endif @@ -1050,6 +1058,7 @@ void mtmd_helper_video_free(mtmd_helper_video * ctx) { ctx->stop_ffmpeg(); delete ctx; #else + GGML_UNUSED(ctx); LOG_ERR("%s: video is not supported in this build (MTMD_VIDEO is set to OFF)\n", __func__); #endif } @@ -1058,6 +1067,7 @@ mtmd_helper_video_info mtmd_helper_video_get_info(const mtmd_helper_video * ctx) #ifdef MTMD_VIDEO return ctx->info; #else + GGML_UNUSED(ctx); GGML_ASSERT(false && "video is not supported in this build (MTMD_VIDEO is set to OFF)"); #endif } @@ -1068,6 +1078,9 @@ int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx, if (!ctx) return -2; return ctx->read_next(out_bitmap, out_text); #else + GGML_UNUSED(ctx); + GGML_UNUSED(out_bitmap); + GGML_UNUSED(out_text); GGML_ASSERT(false && "video is not supported in this build (MTMD_VIDEO is set to OFF)"); #endif } diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 36cd463b20..72d35fce69 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -68,6 +68,9 @@ struct img_tool { case RESIZE_ALGO_BICUBIC_PILLOW: resize_bicubic_pillow(src, dst, target_resolution.width, target_resolution.height); break; + case RESIZE_ALGO_LANCZOS: + resize_lanczos_pillow(src, dst, target_resolution.width, target_resolution.height); + break; default: throw std::runtime_error("Unsupported resize algorithm"); } @@ -97,6 +100,9 @@ struct img_tool { case RESIZE_ALGO_BICUBIC_PILLOW: resize_bicubic_pillow(src, resized_image, new_width, new_height); break; + case RESIZE_ALGO_LANCZOS: + resize_lanczos_pillow(src, resized_image, new_width, new_height); + break; default: throw std::runtime_error("Unsupported resize algorithm"); } @@ -337,22 +343,50 @@ private: } } - // Bicubic resize function using Pillow's ImagingResample algorithm + // Pillow-compatible separable resampling (Bicubic and Lanczos) // Adapted from https://github.com/python-pillow/Pillow/blob/main/src/libImaging/Resample.c // - // Key Difference with resize_bicubic: - // 1. Uses separable filtering: horizontal pass followed by vertical pass + // Key properties: + // 1. Separable filtering: horizontal pass followed by vertical pass // 2. Pre-computes normalized filter coefficients for each output pixel - // 3. Applies convolution using fixed-point integer arithmetic for performance + // 3. Fixed-point integer arithmetic (22 fractional bits) for speed and determinism static bool resize_bicubic_pillow(const clip_image_u8 & img, clip_image_u8 & dst, int target_width, int target_height) { + return resize_pillow(img, dst, target_width, target_height, /*use_lanczos=*/false); + } + + // Lanczos-3 (support radius 3), matches Pillow's Image.LANCZOS + static bool resize_lanczos_pillow(const clip_image_u8 & img, clip_image_u8 & dst, int target_width, int target_height) { + return resize_pillow(img, dst, target_width, target_height, /*use_lanczos=*/true); + } + + static bool resize_pillow( + const clip_image_u8 & img, + clip_image_u8 & dst, + int target_width, + int target_height, + bool use_lanczos) { // Fixed-point precision: 22 bits = 32 (int32_t) - 8 (uint8_t pixels) - 2 (headroom for accumulation) // This allows encoding fractional weights as integers: weight * 2^22 const int PRECISION_BITS = 32 - 8 - 2; - // Bicubic filter function with a = -0.5 (Note that GGML/PyTorch takes a = -0.75) + // Resample filter: Lanczos-3 (support [-3, 3]) or bicubic with a = -0.5 (support [-2, 2]) + // Note: GGML/PyTorch bicubic uses a = -0.75, Pillow uses a = -0.5 // Returns filter weight for distance x from pixel center - // Support: [-2, 2], meaning the filter influences pixels within 2 units of distance - auto bicubic_filter = [](double x) -> double { + auto resample_filter = [use_lanczos](double x) -> double { + if (use_lanczos) { + if (-3.0 <= x && x < 3.0) { + auto sinc = [](double v) { + if (v == 0.0) { + return 1.0; + } + const double pi_v = v * 3.141592653589793238462643383279502884; + return std::sin(pi_v) / pi_v; + }; + return sinc(x) * sinc(x / 3.0); + } + return 0.0; + } + constexpr double a = -0.5; if (x < 0.0) { x = -x; @@ -366,8 +400,8 @@ private: return 0.0; // Zero outside [-2, 2] }; - // Filter support radius: bicubic extends 2 pixels in each direction - constexpr double filter_support = 2.0; + // Filter support radius: 2 for bicubic, 3 for lanczos + const double filter_support = use_lanczos ? 3.0 : 2.0; // Clipping function for 8-bit values auto clip8 = [](int val) -> uint8_t { @@ -434,7 +468,7 @@ private: // Compute filter weights for each contributing input pixel for (x = 0; x < xmax; x++) { // Distance from input pixel center to output pixel center in input space - double w = bicubic_filter((x + xmin - center + 0.5) * ss); + double w = resample_filter((x + xmin - center + 0.5) * ss); pre_weights[xx * ksize + x] = w; ww += w; // Accumulate for normalization } @@ -463,6 +497,12 @@ private: const double fxp_scale = std::ldexp(1.0, PRECISION_BITS); // 1.0 * 2^PRECISION_BITS for (int i = 0; i < outSize * ksize; i++) { + if (use_lanczos) { + // Pillow adds +/- 0.5 then truncates toward zero; std::round would round twice + const double rounded = pre_weights[i] * fxp_scale + (pre_weights[i] < 0 ? -0.5 : 0.5); + weights[i] = static_cast(rounded); + continue; + } double tmp_val = pre_weights[i] * fxp_scale; if (pre_weights[i] < 0) { tmp_val -= 0.5; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 5915b4cba9..93ca8cbcf8 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -463,6 +463,13 @@ struct mtmd_context { img_end = "<|vision_end|>"; image_preproc = std::make_unique(ctx_v); } break; + case PROJECTOR_TYPE_MINIMAX_M3: + { + // ]<]start of image[>[ ... (image embeddings) ... ]<]end of image[>[ + img_beg = "]<]start of image[>["; + img_end = "]<]end of image[>["; + image_preproc = std::make_unique(ctx_v); + } break; case PROJECTOR_TYPE_YOUTUVL: { // <|vision_start|> ... (image embeddings) ... <|vision_end|> @@ -717,12 +724,22 @@ struct mtmd_context { aud_end = ""; audio_preproc = std::make_unique(ctx_a); } break; + case PROJECTOR_TYPE_PARAKEET: + { + audio_preproc = std::make_unique(ctx_a); + } break; case PROJECTOR_TYPE_GEMMA4UA: { aud_beg = "<|audio>"; aud_end = ""; audio_preproc = std::make_unique(ctx_a); } break; + case PROJECTOR_TYPE_MIMO_AUDIO: + { + aud_beg = "<|mimo_audio_start|>"; + aud_end = "<|mimo_audio_end|>"; + audio_preproc = std::make_unique(ctx_a); + } break; default: throw std::runtime_error(string_format("%s: unexpected audio projector type %d\n", __func__, proj)); } diff --git a/tools/server/README-dev.md b/tools/server/README-dev.md index b4ec9f17d3..b41d70c63a 100644 --- a/tools/server/README-dev.md +++ b/tools/server/README-dev.md @@ -136,13 +136,13 @@ Producer side: `server_res_generator` extends `server_res_spipe`, which keeps al Lifetime safety: the session holds no back reference to the response, so `spipe` is a plain `unique_ptr` touched only by the http worker. `cancel` raises an atomic the producer polls; the producer finalizes the session from its destructor, which also runs `~server_response_reader::stop()` to cancel the generation at the queue level. A `DELETE` stops work by raising the flag and letting the worker unwind. -Consumer side: `GET /v1/stream/?from=N` opens a `text/event-stream` that replays buffered bytes from offset `N` and blocks for live bytes, so the browser reattaches like a fresh EventSource. An offset below the dropped prefix returns 400. +Consumer side: `GET /v1/stream?conv_id=&from=N` opens a `text/event-stream` that replays buffered bytes from offset `N` and blocks for live bytes, so the browser reattaches like a fresh EventSource. An offset below the dropped prefix returns 400. Routes: -- `GET /v1/stream/:conv_id?from=N`: replay or live reattach. +- `GET /v1/stream?conv_id=&from=N`: replay or live reattach. The id travels in the query string because it can embed a model name containing slashes. - `POST /v1/streams/lookup` with `{"conversation_ids": [...]}`: returns session status only for ids the caller already owns. There is no listing route, so live sessions cannot be enumerated (an earlier `GET /v1/streams` was removed for exactly this reason). -- `DELETE /v1/stream/:conv_id`: explicit Stop, idempotent (`evict_and_cancel`). +- `DELETE /v1/stream?conv_id=`: explicit Stop, idempotent (`evict_and_cancel`). Router mode binds the same paths to proxy handlers. A `conv_id -> child` map (`conv_models`), populated when a POST is routed, resolves the owning child in one lookup with no polling. The lookup groups ids per child; GET and DELETE proxy straight to the owner. This loopback REST hop is expected to move to a websocket IPC later, swapping only the transport. @@ -166,8 +166,8 @@ graph TD GC[GC thread] -- drop after TTL --> Sess end Sess -- read_from offset --> Cons[stream_pipe_consumer] - Cons -- "GET /v1/stream/:id?from=N" --> Client - DEL[DELETE /v1/stream/:id] -- evict_and_cancel --> Sess + Cons -- "GET /v1/stream?conv_id=id&from=N" --> Client + DEL[DELETE /v1/stream?conv_id=id] -- evict_and_cancel --> Sess ``` The diagram shows the buffer touch points. The live wire (chunks streamed to the original client during a normal generation) is the producer's default output, described under "Producer side" above. diff --git a/tools/server/README.md b/tools/server/README.md index d345654554..f45c018972 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -72,10 +72,10 @@ For the full list of features, please refer to [server's changelog](https://gith | `-ctv, --cache-type-v TYPE` | KV cache data type for V
allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1
(default: f16)
(env: LLAMA_ARG_CACHE_TYPE_V) | | `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)
(env: LLAMA_ARG_DEFRAG_THOLD) | | `--rpc SERVERS` | comma-separated list of RPC servers (host:port)
(env: LLAMA_ARG_RPC) | -| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) | +| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing
(env: LLAMA_ARG_MLOCK) | | `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)
(env: LLAMA_ARG_MMAP) | | `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available
(env: LLAMA_ARG_DIO) | -| `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available

(env: LLAMA_ARG_LOAD_MODE) | +| `-lm, --load-mode MODE` | model loading mode (default: mmap)
- none: no special loading mode
- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)
- mlock: force system to keep model in RAM rather than swapping or compressing
- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing
- dio: use DirectIO if available

(env: LLAMA_ARG_LOAD_MODE) | | `--numa TYPE` | attempt optimizations that help on some NUMA systems
- distribute: spread execution evenly over all nodes
- isolate: only spawn threads on CPUs on the node that execution started on
- numactl: use the CPU map provided by numactl
if run without this previously, it is recommended to drop the system page cache before using this
see https://github.com/ggml-org/llama.cpp/issues/1437
(env: LLAMA_ARG_NUMA) | | `-dev, --device ` | comma-separated list of devices to use for offloading (none = don't offload)
use --list-devices to see a list of available devices
(env: LLAMA_ARG_DEVICE) | | `--list-devices` | print list of available devices and exit | @@ -259,7 +259,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--spec-draft-device, -devd, --device-draft ` | comma-separated list of devices to use for offloading the draft model (none = don't offload)
use --list-devices to see a list of available devices | | `--spec-draft-ngl, -ngld, --gpu-layers-draft, --n-gpu-layers-draft N` | max. number of draft model layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)
(env: LLAMA_ARG_N_GPU_LAYERS_DRAFT) | | `--spec-draft-model, -md, --model-draft FNAME` | draft model for speculative decoding (default: unused)
(env: LLAMA_ARG_SPEC_DRAFT_MODEL) | -| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)

(env: LLAMA_ARG_SPEC_TYPE) | +| `--spec-type none,draft-simple,draft-eagle3,draft-mtp,draft-dflash,draft-dspark,ngram-simple,ngram-map-k,ngram-map-k4v,ngram-mod,ngram-cache` | comma-separated list of types of speculative decoding to use (default: none)

(env: LLAMA_ARG_SPEC_TYPE) | | `--spec-ngram-mod-n-min N` | minimum number of ngram tokens to use for ngram-based speculative decoding (default: 48) | | `--spec-ngram-mod-n-max N` | maximum number of ngram tokens to use for ngram-based speculative decoding (default: 64) | | `--spec-ngram-mod-n-match N` | ngram-mod lookup length (default: 24) | diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 744593c760..4655b518e2 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -78,31 +78,41 @@ struct server_batch { }; std::vector tokens; int32_t n_tokens_alloc = 0; + int32_t n_embd = 0; // track if given slot can be batched with slots already in the batch server_slot * slot_batched = nullptr; + // in embd mode, we temporarily swap out the tokens arr and restore it on clear() + bool has_embd = false; + llama_token * tokens_ptr = nullptr; + std::vector embd; + float alora_scale = -1.0f; size_t alora_disabled_id = 0; server_batch() { - batch.token = nullptr; // sentinel: uninitialized batch + batch.pos = nullptr; // sentinel: uninitialized batch } ~server_batch() { - if (batch.token != nullptr) { + if (batch.pos != nullptr) { + clear(); llama_batch_free(batch); } } - void init(int32_t n_tokens_alloc) { + void init(int32_t n_tokens_alloc, int32_t n_embd) { this->n_tokens_alloc = n_tokens_alloc; + this->n_embd = n_embd; batch = llama_batch_init(n_tokens_alloc, 0, 1); + tokens_ptr = batch.token; tokens.reserve(n_tokens_alloc); } bool add(int32_t id_slot, llama_token token, llama_pos pos, bool output) { - GGML_ASSERT(batch.token != nullptr); + GGML_ASSERT(!has_embd); // cannot mix tokens + embd in same batch + GGML_ASSERT(batch.pos != nullptr); if ((int32_t)tokens.size() >= n_tokens_alloc) { return false; } @@ -110,13 +120,30 @@ struct server_batch { return true; } + bool add(int32_t id_slot, const std::vector & embd_in, llama_pos pos, bool output) { + GGML_ASSERT(batch.pos != nullptr); + if ((int32_t)tokens.size() >= n_tokens_alloc) { + return false; + } + tokens.push_back({ id_slot, LLAMA_TOKEN_NULL, pos, output }); + has_embd = true; + embd.insert(embd.end(), embd_in.begin(), embd_in.end()); + return true; + } + void clear() { tokens.clear(); + embd.clear(); common_batch_clear(batch); slot_batched = nullptr; alora_scale = -1.0f; alora_disabled_id = 0; batch_rendered = false; + has_embd = false; + if (batch.token == nullptr) { + batch.token = tokens_ptr; + batch.embd = nullptr; + } } int32_t size() const { @@ -129,25 +156,33 @@ struct server_batch { } void render() { - GGML_ASSERT(batch.token != nullptr); + GGML_ASSERT(!batch_rendered); + GGML_ASSERT(batch.pos != nullptr); common_batch_clear(batch); for (int32_t i = 0; i < size(); i++) { const auto & t = tokens[i]; common_batch_add(batch, t.token, t.pos, { t.id_slot }, t.output); } + if (has_embd) { + batch.token = nullptr; // will be restored on clear() + batch.embd = embd.data(); + } batch_rendered = true; } llama_batch get_view(int32_t off, int32_t n_tokens) const { - GGML_ASSERT(batch.token != nullptr); + GGML_ASSERT(batch.pos != nullptr); GGML_ASSERT(batch_rendered); GGML_ASSERT(off >= 0 && off < size()); GGML_ASSERT(n_tokens > 0 && off + n_tokens <= size()); + auto * token = batch.token ? batch.token + off : nullptr; + auto * embd = batch.embd ? batch.embd + off * n_embd : nullptr; + llama_batch view = { n_tokens, - batch.token + off, - nullptr, + token, + embd, batch.pos + off, batch.n_seq_id + off, batch.seq_id + off, @@ -164,6 +199,8 @@ struct server_slot { llama_context * ctx_tgt = nullptr; llama_context * ctx_dft = nullptr; + common_memory mem; + // multimodal mtmd_context * mctx = nullptr; mtmd::batch_ptr mbatch = nullptr; @@ -175,6 +212,7 @@ struct server_slot { llama_tokens spec_prompt; std::vector spec_i_batch; common_prompt_checkpoint spec_ckpt; + bool spec_is_replay = false; // TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state // see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837 @@ -253,10 +291,7 @@ struct server_slot { void prompt_clear() { SLT_TRC(*this, "clearing prompt with %zu tokens\n", prompt.tokens.size()); - common_context_seq_rm(ctx_tgt, id, -1, -1); - if (ctx_dft) { - common_context_seq_rm(ctx_dft, id, -1, -1); - } + mem.seq_rm(id, -1, -1); prompt.clear(); } @@ -271,6 +306,10 @@ struct server_slot { llama_token sampled; // in speculative mode, this is the last accepted token + // for TTS models, this is the embd generated from prev step, decode this to generate next hidden state + // corresponding to one token position (size = n_embd) + std::vector inp_embd; + // stats size_t n_sent_text = 0; // number of sent text character @@ -294,6 +333,8 @@ struct server_slot { void reset() { SLT_DBG(*this, "%s", "\n"); + spec_is_replay = false; + n_prompt_tokens_cache = 0; last_nl_pos = 0; @@ -379,7 +420,9 @@ struct server_slot { bool can_batch_with(server_slot & other_slot) const { GGML_ASSERT(task); - return task->type == other_slot.task->type && are_lora_equal(lora, other_slot.lora); + return task->type == other_slot.task->type + && inp_embd.size() == other_slot.inp_embd.size() + && are_lora_equal(lora, other_slot.lora); } bool has_budget(const common_params & global_params) { @@ -445,7 +488,11 @@ struct server_slot { // no speculative decoding i_batch = batch.size(); - add_ok &= batch.add(id, sampled, prompt.tokens.pos_next(), true); + if (!inp_embd.empty()) { + add_ok &= batch.add(id, inp_embd, prompt.tokens.pos_next(), true); + } else { + add_ok &= batch.add(id, sampled, prompt.tokens.pos_next(), true); + } SLT_DBG(*this, "slot decode token, id=%d, n_ctx = %d, n_tokens = %d, truncated = %d\n", sampled, n_ctx, prompt.n_tokens(), truncated); @@ -668,13 +715,8 @@ struct server_slot { void copy_state_to(server_slot & other) const { GGML_ASSERT(state == SLOT_STATE_DONE_PROMPT); - common_context_seq_rm(ctx_tgt, other.id, -1, -1); - common_context_seq_cp(ctx_tgt, id, other.id, -1, -1); - - if (ctx_dft) { - common_context_seq_rm(ctx_dft, other.id, -1, -1); - common_context_seq_cp(ctx_dft, id, other.id, -1, -1); - } + mem.seq_rm(other.id, -1, -1); + mem.seq_cp(id, other.id, -1, -1); other.n_decoded = n_decoded; other.n_remaining = n_remaining; @@ -1302,6 +1344,7 @@ private: slot.id = i; slot.ctx_tgt = ctx_tgt; slot.ctx_dft = ctx_dft; + slot.mem.init(ctx_tgt, ctx_dft); slot.spec = spec.get(); slot.n_ctx = n_ctx_slot; @@ -1339,7 +1382,8 @@ private: // note that n_batch can be > n_ctx (e.g. for non-causal attention models such as BERT where the KV cache is not used) { const int32_t n_batch = llama_n_batch(ctx_tgt); - batch.init(std::max(n_batch, params_base.n_parallel)); + const int32_t n_embd = llama_model_n_embd_inp(model_tgt); + batch.init(std::max(n_batch, params_base.n_parallel), n_embd); } if (params_base.cache_ram_mib != 0) { @@ -1542,7 +1586,7 @@ private: // find the slot that has at least n% prompt similarity if (slot_prompt_similarity != 0.0f) { - float sim_best = 0; + float f_sim_best = 0; for (server_slot & slot : slots) { if (task.id_slot != -1 && slot.id != task.id_slot) { @@ -1551,6 +1595,7 @@ private: // skip the slot if it is not available if (slot.is_processing()) { + SLT_TRC(slot, " - skipping, is_processing = %d\n", slot.is_processing()); continue; } @@ -1558,26 +1603,30 @@ private: // skip the slot if it does not contains cached tokens if (tokens.empty()) { + SLT_TRC(slot, "%s", " - skipping, slot is empty\n"); continue; } // fraction of the Longest Common Prefix length with respect to the input prompt length - const float sim_cur = float(tokens.get_common_prefix(task.tokens)) / task.tokens.size(); + const size_t lcp_len = tokens.get_common_prefix(task.tokens); + const float f_sim_cur = float(lcp_len) / task.tokens.size(); + + SLT_TRC(slot, " - checking sim = %.3f (%zu/%zu) > %.3f\n", f_sim_cur, lcp_len, task.tokens.size(), slot_prompt_similarity); // select the current slot if the criteria match - if (sim_cur > sim_best && sim_cur > slot_prompt_similarity) { - sim_best = sim_cur; + if (f_sim_cur > f_sim_best && f_sim_cur > slot_prompt_similarity) { + f_sim_best = f_sim_cur; ret = &slot; } } if (ret != nullptr) { - const float f_keep = (sim_best*task.tokens.size()) / ret->prompt.tokens.size(); + const float f_keep = (f_sim_best*task.tokens.size()) / ret->prompt.tokens.size(); if (task.id_slot == -1) { - SLT_INF(*ret, "selected slot by LCP similarity, sim_best = %.3f (> %.3f thold), f_keep = %.3f\n", - sim_best, slot_prompt_similarity, f_keep); + SLT_INF(*ret, "selected slot by LCP similarity, f_sim_best = %.3f (> %.3f thold), f_keep = %.3f\n", + f_sim_best, slot_prompt_similarity, f_keep); } // if we are about to lose a large portion of the existing context - save it in the prompt cache @@ -2881,13 +2930,8 @@ private: SLT_WRN(slot, "slot context shift, n_keep = %d, n_left = %d, n_discard = %d\n", n_keep, n_left, n_discard); - common_context_seq_rm (ctx_tgt, slot.id, n_keep , n_keep + n_discard); - common_context_seq_add(ctx_tgt, slot.id, n_keep + n_discard, slot.prompt.n_tokens(), -n_discard); - - if (ctx_dft) { - common_context_seq_rm (ctx_dft, slot.id, n_keep , n_keep + n_discard); - common_context_seq_add(ctx_dft, slot.id, n_keep + n_discard, slot.prompt.tokens.pos_next(), -n_discard); - } + slot.mem.seq_rm (slot.id, n_keep , n_keep + n_discard); + slot.mem.seq_add(slot.id, n_keep + n_discard, slot.prompt.tokens.pos_next(), -n_discard); // add generated tokens to cache // ref: https://github.com/ggml-org/llama.cpp/pull/16818#discussion_r2473269481 @@ -2998,7 +3042,9 @@ private: ckpt.load_dft(ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); } - common_context_seq_rm(ctx_dft, slot.id, ckpt.pos_max + 1, -1); + if (!llama_memory_seq_rm(llama_get_memory(ctx_dft), slot.id, ckpt.pos_max + 1, -1)) { + GGML_ABORT("failed to remove sequence %d\n", slot.id); + } } if (!draft.empty()) { @@ -3201,13 +3247,8 @@ private: const int64_t kv_shift = (int64_t) head_p - (int64_t) head_c; - common_context_seq_rm (ctx_tgt, slot.id, head_p, head_c); - common_context_seq_add(ctx_tgt, slot.id, head_c, head_c + n_match, kv_shift); - - if (ctx_dft) { - common_context_seq_rm (ctx_dft, slot.id, head_p, head_c); - common_context_seq_add(ctx_dft, slot.id, head_c, head_c + n_match, kv_shift); - } + slot.mem.seq_rm (slot.id, head_p, head_c); + slot.mem.seq_add(slot.id, head_c, head_c + n_match, kv_shift); for (size_t i = 0; i < n_match; i++) { slot.prompt.tokens.set_token(head_p + i, slot.prompt.tokens[head_c + i]); @@ -3379,10 +3420,7 @@ private: SLT_TRC(slot, "cached n_tokens = %d, memory_seq_rm [%d, end)\n", slot.prompt.n_tokens(), p0); - common_context_seq_rm(ctx_tgt, slot.id, p0, -1); - if (ctx_dft) { - common_context_seq_rm(ctx_dft, slot.id, p0, -1); - } + slot.mem.seq_rm(slot.id, p0, -1); // If using an alora, there may be uncached tokens that come // before the invocation sequence. When this happens, the @@ -3589,6 +3627,15 @@ private: n_empty_consecutive = 0; } + // TODO @ngxson : dft model may have different n_embd than the tgt model, so we check & reject if that's the case + // this case is not currently used by any models, but may need to be supported in the future + if (spec && batch.has_embd) { + if (llama_model_n_embd_inp(model_dft) != llama_model_n_embd_inp(model_tgt)) { + SRV_ERR("%s", "unsupported batch.has_embd + spec case\n"); + throw std::runtime_error("unsupported batch.has_embd + spec case"); + } + } + const int ret = llama_decode(ctx_tgt, batch_view); metrics.on_decoded(slots); @@ -3831,24 +3878,21 @@ private: } // partial acceptance is not supported by the context -> truncate the draft and restore the state + slot.spec_is_replay = true; slot.spec_draft = std::move(accepted); const auto & ckpt = slot.spec_ckpt; SLT_DBG(slot, "restoring speculative checkpoint (pos_min = %d, pos_max = %d, size = %zu)\n", ckpt.pos_min, ckpt.pos_max, ckpt.size()); - { - ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); - - common_context_seq_rm(slot.ctx_tgt, slot.id, ckpt.pos_max + 1, -1); - } + ckpt.load_tgt(slot.ctx_tgt, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); if (slot.ctx_dft) { ckpt.load_dft(slot.ctx_dft, slot.id, LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY); - - common_context_seq_rm(slot.ctx_dft, slot.id, ckpt.pos_max + 1, -1); } + slot.mem.seq_rm(slot.id, ckpt.pos_max + 1, -1); + slot.prompt.tokens.keep_first(ckpt.n_tokens); slot.smpl = std::move(smpl_save); @@ -3869,16 +3913,22 @@ private: const auto ids = std::move(slot.spec_draft); + size_t n_accepted = ids.size() - 1; + if (slot.spec_is_replay && n_accepted > 0) { + n_accepted--; + } + slot.spec_is_replay = false; + slot.t_token_generation = std::max(1, t_now - slot.t_start_generation) / 1e3; // update how many tokens out of those tested were accepted - slot.n_draft_accepted += ids.size() - 1; + slot.n_draft_accepted += n_accepted; slot.n_draft_verif_steps += 1; if (slot.n_accepted_per_pos.empty()) { slot.n_accepted_per_pos.resize(common_speculative_n_max(¶ms_base.speculative), 0); } - for (size_t i = 0; i < ids.size() - 1 && i < slot.n_accepted_per_pos.size(); ++i) { + for (size_t i = 0; i < n_accepted && i < slot.n_accepted_per_pos.size(); ++i) { slot.n_accepted_per_pos[i]++; } @@ -3889,10 +3939,7 @@ private: slot.sampled = ids.back(); // last accepted token SLT_DBG(slot, "add accepted tokens: sampled=%d, ids.size=%zu, n_draft=%zu\n", slot.sampled, ids.size(), n_draft); - common_context_seq_rm(slot.ctx_tgt, slot.id, slot.prompt.tokens.pos_next(), -1); - if (slot.ctx_dft) { - common_context_seq_rm(slot.ctx_dft, slot.id, slot.prompt.tokens.pos_next(), -1); - } + slot.mem.seq_rm(slot.id, slot.prompt.tokens.pos_next(), -1); for (size_t i = 0; i < ids.size(); ++i) { completion_token_output result; @@ -3917,7 +3964,7 @@ private: slot.print_timings_tg(); - SLT_DBG(slot, "accepted %d/%d draft tokens, new n_tokens = %d\n", (int) ids.size() - 1, (int) n_draft, slot.prompt.n_tokens()); + SLT_DBG(slot, "accepted %d/%d draft tokens, new n_tokens = %d\n", (int) n_accepted, (int) n_draft, slot.prompt.n_tokens()); }); } diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 923b3533e9..188a72a374 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -1172,7 +1172,7 @@ bool server_models::ensure_model_ready(const std::string & name) { return true; } -server_http_res_ptr server_models::proxy_request(const server_http_req & req, const std::string & method, const std::string & name, bool update_last_used) { +server_http_res_ptr server_models::proxy_request(const server_http_req & req, const std::string & method, const std::string & name, bool update_last_used, bool detached) { auto meta = get_meta(name); if (!meta.has_value()) { throw std::runtime_error("model name=" + name + " is not found"); @@ -1198,7 +1198,10 @@ server_http_res_ptr server_models::proxy_request(const server_http_req & req, co req.headers, req.body, req.files, - req.should_stop, + // a detached request belongs to a replay session that outlives the client socket: + // it reaches the child even when the downstream died during the load wait, the + // session buffer is the recipient and DELETE remains the stop + detached ? std::function([]() { return false; }) : req.should_stop, base_params.timeout_read, base_params.timeout_write ); @@ -1469,13 +1472,9 @@ static bool router_validate_model(std::string & name, server_models & models, bo } // resolve alias to canonical model name name = meta->name; - if (models_autoload) { - models.ensure_model_ready(name); - } else { - if (!meta->is_running()) { - res_err(res, format_error_response("model is not loaded", ERROR_TYPE_INVALID_REQUEST)); - return false; - } + if (!models_autoload && !meta->is_running()) { + res_err(res, format_error_response("model is not loaded", ERROR_TYPE_INVALID_REQUEST)); + return false; } return true; } @@ -1568,6 +1567,9 @@ void server_models_routes::init_routes() { if (!router_validate_model(name, models, autoload, error_res)) { return error_res; } + if (autoload) { + models.ensure_model_ready(name); + } return models.proxy_request(req, method, name, false); }; @@ -1581,12 +1583,23 @@ void server_models_routes::init_routes() { return error_res; } // remember which child serves this conversation so the stream routes can route straight - // to it without polling, keyed on the exact conv id from the header + // to it without polling, keyed on the exact conv id from the header. registered before + // the load wait so a stop issued while the model loads can erase the entry and cancel + // this request instead of leaving an orphan generation std::string conv_id = server_stream_conv_id_from_headers(req.headers); - if (!conv_id.empty()) { - models.conv_models.remember(conv_id, name); + uint64_t ticket = models.conv_models.remember(conv_id, name); + bool waited = autoload && models.ensure_model_ready(name); + if (ticket != 0 && !models.conv_models.alive(conv_id, ticket)) { + SRV_INF("request for conv_id=%s cancelled while model name=%s was loading\n", + conv_id.c_str(), name.c_str()); + res_err(error_res, format_error_response( + "request cancelled by a stop while the model was loading", ERROR_TYPE_INVALID_REQUEST)); + return error_res; } - return models.proxy_request(req, method, name, true); // update last usage for POST request only + // a session request that waited for a load detaches from the client socket: the + // client may have dropped during the wait (page reload) and the session buffer must + // still receive the generation for a later resume + return models.proxy_request(req, method, name, true, waited && ticket != 0); // update last usage for POST request only }; this->post_router_models_load = [this](const server_http_req & req) { @@ -1779,7 +1792,7 @@ void server_models_routes::init_routes() { }; this->router_stream_get = [this](const server_http_req & req) { - // GET /v1/stream/?from=N. resolve the owning child from the conv_id -> model + // GET /v1/stream?conv_id=&from=N. resolve the owning child from the conv_id -> model // map, 404 when nothing maps auto res = std::make_unique(); std::string conv_id = req.get_param("conv_id"); @@ -1789,13 +1802,24 @@ void server_models_routes::init_routes() { } std::optional owner = resolve_child_for_conv(models, conv_id); if (!owner.has_value()) { - res_err(res, format_error_response("Stream not found or expired", ERROR_TYPE_NOT_FOUND)); + // a registered conv whose model is still loading earns a retry: the session appears + // once the load ends and the pending request reaches the child + auto tracked = models.conv_models.lookup(conv_id); + auto meta = tracked.has_value() ? models.get_meta(*tracked) : std::nullopt; + bool transient = meta.has_value() && (meta->status == SERVER_MODEL_STATUS_LOADING || + meta->status == SERVER_MODEL_STATUS_DOWNLOADING || + meta->status == SERVER_MODEL_STATUS_DOWNLOADED); + if (transient) { + res_err(res, format_error_response("Stream owner model is loading, retry later", ERROR_TYPE_UNAVAILABLE)); + } else { + res_err(res, format_error_response("Stream not found or expired", ERROR_TYPE_NOT_FOUND)); + } return res; } std::string from = req.get_param("from"); - std::string child_path = "/v1/stream/" + encode_qs(conv_id); + std::string child_path = "/v1/stream?conv_id=" + encode_qs(conv_id); if (!from.empty()) { - child_path += "?from=" + from; + child_path += "&from=" + from; } SRV_TRC("proxying stream resume to model %s on port %d, path=%s\n", owner->name.c_str(), owner->port, child_path.c_str()); @@ -1875,7 +1899,7 @@ void server_models_routes::init_routes() { }; this->router_stream_delete = [this](const server_http_req & req) { - // DELETE /v1/stream/. resolve the owning child via the map and forward only to + // DELETE /v1/stream?conv_id=. resolve the owning child via the map and forward only to // it, evict_and_cancel is idempotent on the child auto res = std::make_unique(); std::string conv_id = req.get_param("conv_id"); @@ -1883,7 +1907,7 @@ void server_models_routes::init_routes() { res_err(res, format_error_response("Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST)); return res; } - std::string child_path = "/v1/stream/" + encode_qs(conv_id); + std::string child_path = "/v1/stream?conv_id=" + encode_qs(conv_id); auto owner = resolve_child_for_conv(models, conv_id); if (owner.has_value()) { httplib::Client cli(CHILD_ADDR, owner->port); @@ -1892,6 +1916,11 @@ void server_models_routes::init_routes() { cli.set_write_timeout(0, STREAM_LOOKUP_TIMEOUT_MS * 1000); auto resp = cli.Delete(child_path.c_str()); (void) resp; // the child logs its own miss when the session is unknown there + } else if (auto tracked = models.conv_models.lookup(conv_id); tracked.has_value()) { + // the entry exists but its model is still loading: the forget below erases it, + // which cancels the request parked in proxy_post before the generation starts + SRV_INF("router stop for conv_id=%s while model name=%s is loading, cancelling the pending request\n", + conv_id.c_str(), tracked->c_str()); } else { SRV_WRN("router stop for unknown conv_id=%s, no owning child in the conv map\n", conv_id.c_str()); diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 62bed8725b..614798186c 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -134,12 +134,24 @@ private: // proxy_request forwards a POST carrying an X-Conversation-Id. best effort: a stale entry just // makes the child answer not found and the client recovers. owns its lock, one mutex per struct struct conv_model_tracker { - void remember(const std::string & conv_id, const std::string & model) { + // returns the ticket of this registration, 0 when nothing was registered. erasing or + // replacing the entry invalidates the ticket, which is how a stop cancels a request + // parked in the model load wait + uint64_t remember(const std::string & conv_id, const std::string & model) { if (conv_id.empty() || model.empty()) { - return; + return 0; } std::lock_guard lock(mu); - map[conv_id] = model; + uint64_t ticket = next_ticket++; + map[conv_id] = { model, ticket }; + return ticket; + } + + // false means a stop erased the entry or a newer request replaced it + bool alive(const std::string & conv_id, uint64_t ticket) { + std::lock_guard lock(mu); + auto it = map.find(conv_id); + return it != map.end() && it->second.ticket == ticket; } std::optional lookup(const std::string & conv_id) { @@ -151,7 +163,7 @@ private: if (it == map.end()) { return std::nullopt; } - return it->second; + return it->second.model; } void forget(const std::string & conv_id) { @@ -163,8 +175,13 @@ private: } private: - std::mutex mu; - std::unordered_map map; + struct entry_t { + std::string model; + uint64_t ticket; + }; + std::mutex mu; + uint64_t next_ticket = 1; + std::unordered_map map; }; common_preset_context ctx_preset; @@ -249,7 +266,7 @@ public: bool ensure_model_ready(const std::string & name); // proxy an HTTP request to the model instance - server_http_res_ptr proxy_request(const server_http_req & req, const std::string & method, const std::string & name, bool update_last_used); + server_http_res_ptr proxy_request(const server_http_req & req, const std::string & method, const std::string & name, bool update_last_used, bool detached = false); // handle message sent from server_child::notify_to_router() // raw input must starts with CMD_CHILD_TO_ROUTER_STATE, followed by a JSON string diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp index e880f4ca72..674d3ba337 100644 --- a/tools/server/server-schema.cpp +++ b/tools/server/server-schema.cpp @@ -209,6 +209,7 @@ std::vector> make_llama_cmpl_schema(const common_params & ->set_hard_limits(0.0f, 1.0f) ->set_desc("Minimum speculative decoding probability for draft tokens (0 = greedy)")); + add((new field_str("speculative.type")) ->set_desc("Speculative decoding method (for debugging and research purposes)") ->set_handler([&](field_eval_context & ctx, const json & data) { diff --git a/tools/server/server-stream.cpp b/tools/server/server-stream.cpp index f0a35b18e5..f6b9b8a9f4 100644 --- a/tools/server/server-stream.cpp +++ b/tools/server/server-stream.cpp @@ -453,7 +453,7 @@ static server_http_res_ptr make_error_response(int status, const std::string & m server_http_context::handler_t server_stream_make_get_handler() { return [](const server_http_req & req) -> server_http_res_ptr { - // GET /v1/stream/?from=N replays buffered SSE bytes then blocks for live + // GET /v1/stream?conv_id=&from=N replays buffered SSE bytes then blocks for live // bytes until the session finalizes, streamed as text/event-stream for EventSource std::string conv_id = req.get_param("conv_id"); if (conv_id.empty()) { @@ -560,13 +560,13 @@ server_http_context::handler_t server_stream_make_lookup_handler() { server_http_context::handler_t server_stream_make_delete_handler() { return [](const server_http_req & req) -> server_http_res_ptr { - // DELETE /v1/stream/ is the explicit user Stop, cancels the producer and evicts + // DELETE /v1/stream?conv_id= is the explicit user Stop, cancels the producer and evicts // the buffer. idempotent, returns 204 even if the session was already gone std::string conv_id = req.get_param("conv_id"); if (conv_id.empty()) { return make_error_response(400, "Missing conversation id in path", ERROR_TYPE_INVALID_REQUEST); } - SRV_TRC("DELETE /v1/stream/%s -> evict_and_cancel\n", conv_id.c_str()); + SRV_TRC("DELETE /v1/stream conv_id=%s -> evict_and_cancel\n", conv_id.c_str()); g_stream_sessions.evict_and_cancel(conv_id); auto res = std::make_unique(); res->status = 204; @@ -621,7 +621,7 @@ bool server_res_spipe::conn_alive() { bool server_res_spipe::should_stop() { if (spipe) { - // note: if DELETE /v1/stream/ is called, is_cancelled() will be true + // note: if DELETE /v1/stream is called for this conv, is_cancelled() will be true return spipe->is_cancelled(); } else { return !conn_alive(); diff --git a/tools/server/server-stream.h b/tools/server/server-stream.h index 9753140dd6..1e7461285f 100644 --- a/tools/server/server-stream.h +++ b/tools/server/server-stream.h @@ -45,7 +45,13 @@ void server_stream_session_manager_start(); void server_stream_session_manager_stop(); // route handler factories wired under /v1/stream/* by server.cpp +// child-side handlers for the resumable stream routes. the conv id travels in the conv_id +// query string because it can embed a model name containing slashes (org/repo), which the +// decoded path would split before the param is captured server_http_context::handler_t server_stream_make_get_handler(); +// POST /v1/streams/lookup with body {"conversation_ids": [...]}: only answers for ids the +// caller already owns (the WebUI passes the convs visible in its sidebar), the server never +// lists ids it has not been asked about, so a random caller cannot enumerate live sessions server_http_context::handler_t server_stream_make_lookup_handler(); server_http_context::handler_t server_stream_make_delete_handler(); diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 1fd7cce27b..070f1ade24 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -1742,9 +1742,9 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok const int lcp_best = prompt.tokens.get_common_prefix(tokens_new); float f_keep_best = prompt.tokens.size() > 0 ? float(lcp_best) / prompt.tokens.size() : -1.0f; // empty slot: any cache entry wins - float sim_best = float(lcp_best) / tokens_new.size(); + float f_sim_best = float(lcp_best) / tokens_new.size(); - SRV_TRC(" - looking for better prompt, base f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best); + SRV_TRC(" - looking for better prompt, base f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best); auto it_best = states.end(); @@ -1753,23 +1753,25 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok const int lcp_cur = it->prompt.tokens.get_common_prefix(tokens_new); const float f_keep_cur = float(lcp_cur) / it->prompt.tokens.size(); - const float sim_cur = float(lcp_cur) / tokens_new.size(); + const float f_sim_cur = float(lcp_cur) / tokens_new.size(); + + SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur); // don't trash large prompts if (f_keep_cur < 0.25f) { continue; } - if (f_keep_best < f_keep_cur && sim_best < sim_cur) { + if (f_keep_best < f_keep_cur && f_sim_best < f_sim_cur) { f_keep_best = f_keep_cur; - sim_best = sim_cur; + f_sim_best = f_sim_cur; it_best = it; } } if (it_best != states.end()) { - SRV_TRC(" - found better prompt with f_keep = %.3f, sim = %.3f\n", f_keep_best, sim_best); + SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best); { auto & data = it_best->data.main; diff --git a/tools/server/server-task.h b/tools/server/server-task.h index c3eea2ecb8..411d918079 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -650,7 +650,7 @@ struct server_prompt_cache { server_prompt_cache_state * alloc(const server_prompt & prompt, size_t state_size_main, size_t state_size_drft); - bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_main, llama_context * ctx_drft, int32_t id_slot); + bool load(server_prompt & prompt, const server_tokens & tokens_new, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot); void update(); }; diff --git a/tools/server/server.cpp b/tools/server/server.cpp index b6fef99e87..a3b2a8b0fe 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -272,10 +272,8 @@ int llama_server(common_params & params, int argc, char ** argv) { ctx_http.get ("/slots", ex_wrapper(routes.get_slots)); ctx_http.post("/slots/:id_slot", ex_wrapper(routes.post_slots)); - // resumable streaming, the conversation_id is the session identity end to end. router and - // child wire different handlers under the same paths: a child binds the local session - // factories, the router binds proxies that resolve the owning child through the - // conv_id -> model map + // resumable streaming: a child binds the local session factories, the router binds + // proxies that resolve the owning child, see server-stream.h server_http_context::handler_t stream_get_h; server_http_context::handler_t streams_lookup_h; server_http_context::handler_t stream_delete_h; @@ -288,12 +286,9 @@ int llama_server(common_params & params, int argc, char ** argv) { streams_lookup_h = server_stream_make_lookup_handler(); stream_delete_h = server_stream_make_delete_handler(); } - ctx_http.get ("/v1/stream/:conv_id", ex_wrapper(stream_get_h)); - // POST /v1/streams/lookup with body {"conversation_ids": [...]}. you can only ask for ids - // you already own (the WebUI passes the convs visible in its sidebar). the server never - // lists ids it has not been asked about, so a random caller cannot enumerate live sessions + ctx_http.get ("/v1/stream", ex_wrapper(stream_get_h)); ctx_http.post("/v1/streams/lookup", ex_wrapper(streams_lookup_h)); - ctx_http.del ("/v1/stream/:conv_id", ex_wrapper(stream_delete_h)); + ctx_http.del ("/v1/stream", ex_wrapper(stream_delete_h)); // Google Cloud Platform (Vertex AI) compat ctx_http.register_gcp_compat(); diff --git a/tools/server/tests/unit/test_stream.py b/tools/server/tests/unit/test_stream.py new file mode 100644 index 0000000000..a1ef55567b --- /dev/null +++ b/tools/server/tests/unit/test_stream.py @@ -0,0 +1,153 @@ +import json +import socket +import threading +import time +from urllib.parse import quote +import pytest +from utils import * + +server: ServerProcess + +# a model name with slashes exercises the query string routing of the stream routes: the id +# cannot travel as a path param because the decoded slash would split it before capture +MODEL = "ggml-org/tinygemma3-GGUF:Q8_0" +STREAM_ID = f"conv-stream-test::{MODEL}" +QS = "conv_id=" + quote(STREAM_ID, safe="") + + +@pytest.fixture(autouse=True) +def create_server(): + global server + server = ServerPreset.router() + + +def test_stream_resume_and_stop_with_slashed_model_name(): + global server + server.start() + + content = "" + for data in server.make_stream_request("POST", "/chat/completions", data={ + "model": MODEL, + "stream": True, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hello"}], + }, headers={"X-Conversation-Id": STREAM_ID}): + if data["choices"]: + content += data["choices"][0]["delta"].get("content") or "" + assert len(content) > 0 + + # the finished session replays from the beginning through the router + res = server.make_request("GET", f"/v1/stream?{QS}&from=0") + assert res.status_code == 200 + assert "data: " in str(res.body) + + # the explicit stop reaches the owning child and evicts the session + res = server.make_request("DELETE", f"/v1/stream?{QS}") + assert res.status_code == 204 + res = server.make_request("GET", f"/v1/stream?{QS}&from=0") + assert res.status_code == 404 + + +def test_stream_stop_during_model_load(): + global server + server.start() + + thread_error: list[ServerError] = [] + thread_done = threading.Event() + + def fire_post(): + try: + for _ in server.make_stream_request("POST", "/chat/completions", data={ + "model": MODEL, + "stream": True, + "max_tokens": 512, + "messages": [{"role": "user", "content": "Count from 1 to 1000."}], + }, headers={"X-Conversation-Id": STREAM_ID}): + pass + except ServerError as e: + thread_error.append(e) + finally: + thread_done.set() + + t = threading.Thread(target=fire_post) + t.start() + + # catch the autoload window, tiny models load fast so poll aggressively + saw_loading = False + deadline = time.time() + 5.0 + while time.time() < deadline and not thread_done.is_set(): + res = server.make_request("GET", "/models") + status = next(m["status"]["value"] for m in res.body["data"] if m["id"] == MODEL) + if status == "loading": + saw_loading = True + break + time.sleep(0.002) + if not saw_loading: + t.join() + pytest.skip("load window too short to be observed on this machine") # ty: ignore[too-many-positional-arguments] + + # a stop during the load cancels the parked request instead of leaving an orphan + res = server.make_request("DELETE", f"/v1/stream?{QS}") + assert res.status_code == 204 + assert thread_done.wait(timeout=60) + t.join() + assert len(thread_error) == 1 + assert thread_error[0].code == 400 + assert "cancelled" in json.dumps(thread_error[0].body) + res = server.make_request("GET", f"/v1/stream?{QS}&from=0") + assert res.status_code == 404 + + +def test_stream_resumes_after_reload_during_model_load(): + global server + server.start() + + # raw socket client so the connection can be dropped mid load like a page reload + body = json.dumps({ + "model": MODEL, + "stream": True, + "max_tokens": 16, + "messages": [{"role": "user", "content": "hello"}], + }) + request = ( + f"POST /v1/chat/completions HTTP/1.1\r\n" + f"Host: {server.server_host}:{server.server_port}\r\n" + f"Content-Type: application/json\r\n" + f"X-Conversation-Id: {STREAM_ID}\r\n" + f"Content-Length: {len(body)}\r\n" + f"Connection: close\r\n\r\n{body}" + ) + sock = socket.create_connection((server.server_host, server.server_port)) + sock.sendall(request.encode()) + + # drop the client while the model loads, poll aggressively to catch the window + saw_loading = False + saw_503 = False + deadline = time.time() + 5.0 + while time.time() < deadline: + res = server.make_request("GET", "/models") + status = next(m["status"]["value"] for m in res.body["data"] if m["id"] == MODEL) + if status == "loading": + saw_loading = True + break + if status == "loaded": + break + time.sleep(0.002) + sock.close() + if not saw_loading: + pytest.skip("load window too short to be observed on this machine") # ty: ignore[too-many-positional-arguments] + + # while the model loads the resume route answers retry later, then the session appears, + # receives the whole generation despite the dead client, and replays from the beginning + deadline = time.time() + 60.0 + replay = None + while time.time() < deadline: + res = server.make_request("GET", f"/v1/stream?{QS}&from=0") + if res.status_code == 503: + saw_503 = True + elif res.status_code == 200 and "data: " in str(res.body): + replay = res + break + time.sleep(0.1) + assert saw_503, "resume during the load did not answer 503" + assert replay is not None, "session never became resumable after the client disconnect" diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte index 905c2fe6f0..f81dcf09c0 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte @@ -48,6 +48,9 @@ }: Props = $props(); let dropdownOpen = $state(false); + // The system message action moves focus to the message editor, so the menu + // must not restore focus to the trigger on close + let suppressCloseAutoFocus = false; function handleMcpSettingsClick() { dropdownOpen = false; @@ -96,7 +99,16 @@ - + { + if (suppressCloseAutoFocus) { + suppressCloseAutoFocus = false; + e.preventDefault(); + } + }} + > @@ -148,7 +160,10 @@ { + suppressCloseAutoFocus = true; + onSystemPromptClick?.(); + }} > diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte index 8e8a14ac31..b8068f7907 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte @@ -2,6 +2,7 @@ import { goto } from '$app/navigation'; import { getChatActionsContext, setMessageEditContext } from '$lib/contexts'; import { chatStore, pendingEditMessageId } from '$lib/stores/chat.svelte'; + import { isMobile } from '$lib/stores/viewport.svelte'; import { conversationsStore } from '$lib/stores/conversations.svelte'; import { DatabaseService } from '$lib/services/database.service'; import { SYSTEM_MESSAGE_PLACEHOLDER } from '$lib/constants'; @@ -46,7 +47,14 @@ assistantMessages: number; messageTypes: string[]; } | null>(null); - let editedContent = $derived(message.content); + // The system message placeholder must never surface as editable content; keeping + // it in the derived (not just in handleEdit) guards against prop invalidation + // reverting the override while editing + let editedContent = $derived( + message.role === MessageRole.SYSTEM && message.content === SYSTEM_MESSAGE_PLACEHOLDER + ? '' + : message.content + ); let rawEditContent = $derived.by(() => { if (message.role !== MessageRole.ASSISTANT) return undefined; @@ -265,6 +273,12 @@ chatActions.navigateToSibling(siblingId); } + // After the system message flow ends, hand focus to the main chat form + function focusMainChatForm() { + if (isMobile.current) return; + document.querySelector('.chat-screen-form-wrapper textarea')?.focus(); + } + async function handleSaveEdit() { if (message.role === MessageRole.SYSTEM) { // System messages: update in place without branching @@ -276,6 +290,8 @@ isEditing = false; if (conversationDeleted) { goto(ROUTES.START); + } else { + focusMainChatForm(); } return; } @@ -285,6 +301,7 @@ if (index !== -1) { conversationsStore.updateMessageAtIndex(index, { content: newContent }); } + focusMainChatForm(); } else if (message.role === MessageRole.USER) { const finalExtras = await getMergedExtras(); chatActions.editWithBranching(message, editedContent.trim(), finalExtras); diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte index 00578fcf1a..199d75fcec 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte @@ -10,7 +10,7 @@ } from '$lib/components/app'; import { getMessageEditContext } from '$lib/contexts'; import { useProcessingState } from '$lib/hooks/use-processing-state.svelte'; - import { isLoading, isChatStreaming } from '$lib/stores/chat.svelte'; + import { chatStore, isLoading, isChatStreaming } from '$lib/stores/chat.svelte'; import { modelLoadProgressText } from '$lib/utils'; import { MessageRole } from '$lib/enums'; import { config } from '$lib/stores/settings.svelte'; @@ -82,8 +82,11 @@ let hasNoContent = $derived(!message?.content?.trim()); let isActivelyProcessing = $derived(isCurrentlyLoading || isStreaming); - // during a router auto-load the message has no model yet, so target the selected one - let loadTargetModel = $derived(message.model ?? modelsStore.selectedModelName); + // during a router auto-load the message has no model yet: target the model frozen in the + // persisted stream state (survives a reload), then fall back to the dropdown selection + let loadTargetModel = $derived( + message.model ?? chatStore.getResumeModel(message.convId) ?? modelsStore.selectedModelName + ); let modelLoadProgress = $derived( isRouter && loadTargetModel ? modelsStore.getLoadProgress(loadTargetModel) : null ); diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte index 36798e2283..24b3be4c5f 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte @@ -7,7 +7,7 @@ import { getMessageEditContext } from '$lib/contexts'; import { KeyboardKey, MessageRole } from '$lib/enums'; import { config } from '$lib/stores/settings.svelte'; - import { isIMEComposing } from '$lib/utils'; + import { autoResizeTextarea, isIMEComposing } from '$lib/utils'; interface Props { class?: string; @@ -91,6 +91,11 @@ resizeObserver.disconnect(); }; }); + $effect(() => { + if (editCtx.isEditing && textareaElement) { + autoResizeTextarea(textareaElement); + } + }); function toggleExpand() { isExpanded = !isExpanded; @@ -105,11 +110,15 @@ {#if editCtx.isEditing}
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte index acf2de12ac..92652a0a86 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte @@ -11,8 +11,7 @@ classifyToolResult, formatJsonPretty, parseToolResultWithImages, - type AgenticSection, - type ToolResultLine + type AgenticSection } from '$lib/utils'; import { getBuiltinToolUi } from '$lib/constants/built-in-tools'; import type { DatabaseMessageExtra } from '$lib/types'; @@ -29,11 +28,10 @@ let { section, open, isStreaming, attachments, onToggle }: Props = $props(); const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? ''); - - const parsedLines: ToolResultLine[] = $derived( + const outputKind = $derived(classifyToolResult(section.toolResult)); + const parsedLines = $derived( section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : [] ); - const outputKind = $derived(classifyToolResult(section.toolResult)); diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte index 6f30060f54..b990c3898b 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte @@ -15,7 +15,6 @@ let { section, open, isStreaming, onToggle }: Props = $props(); const editFileMeta = $derived(parseEditFileMeta(section)); - const editDiffs = $derived( (editFileMeta?.edits ?? []).map((edit) => computeLineDiff(edit.oldText, edit.newText)) ); diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockSearchResults.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockSearchResults.svelte index e4b4adf151..60862dd063 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockSearchResults.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockSearchResults.svelte @@ -27,7 +27,7 @@ const isStreamingCall = $derived(section.type === AgenticSectionType.TOOL_CALL_STREAMING); const showSpinner = $derived(isPending || (isStreamingCall && isStreaming)); - const results: SearchResult[] = $derived(extractSearchResults(section.toolResult)); + const results = $derived(extractSearchResults(section.toolResult)); const query = $derived(extractSearchQuery(section.toolArgs)); // Same icon-resolution chain as ChatMessageToolCallBlockDefault so diff --git a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenForm.svelte b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenForm.svelte index 600180742a..8eb17eeae4 100644 --- a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenForm.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenForm.svelte @@ -106,15 +106,23 @@ onFileRemove?.(fileId); } + // Auto-focus must not steal focus already claimed elsewhere (e.g. the system + // message editor opened just before a navigation) + function focusFormUnlessCaptured() { + const active = document.activeElement; + if (active instanceof HTMLTextAreaElement || active instanceof HTMLInputElement) return; + chatFormRef?.focus(); + } + onMount(() => { if (!isMobile.current) { - setTimeout(() => chatFormRef?.focus(), 100); + setTimeout(focusFormUnlessCaptured, 100); } }); afterNavigate((navigation) => { if (navigation?.from != null && !isMobile.current) { - setTimeout(() => chatFormRef?.focus(), 100); + setTimeout(focusFormUnlessCaptured, 100); } }); @@ -127,7 +135,7 @@ $effect(() => { if (previousIsLoading && !isLoading) { - setTimeout(() => chatFormRef?.focus(), 10); + setTimeout(focusFormUnlessCaptured, 10); } previousIsLoading = isLoading; diff --git a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte index 018949aff8..5b44bcf858 100644 --- a/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatScreen/ChatScreenGreeting.svelte @@ -11,7 +11,7 @@