mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-19 09:17:40 -06:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c7468107ef | ||
|
|
ac830a9cbb | ||
|
|
8af68d486a | ||
|
|
f8caa5ea30 | ||
|
|
4db8b7638f | ||
|
|
01592dc58b | ||
|
|
8bf2132c3f | ||
|
|
5812201a81 | ||
|
|
41d8297fe4 | ||
|
|
51a77c4f24 | ||
|
|
76f186df71 | ||
|
|
95b09b62be | ||
|
|
b5dbca1e44 | ||
|
|
80437488a4 | ||
|
|
39039aa27f | ||
|
|
7a4117eef9 | ||
|
|
573aa89de6 | ||
|
|
ecafc67145 | ||
|
|
d4f5c26049 | ||
|
|
08c252e221 | ||
|
|
e4b620af19 | ||
|
|
b001d7bb22 | ||
|
|
0a76757469 | ||
|
|
b044890408 | ||
|
|
78e05298f0 | ||
|
|
785f6bac00 | ||
|
|
edec462cc0 | ||
|
|
4483b2b987 | ||
|
|
bafa0420c1 |
@@ -140,6 +140,11 @@ jobs:
|
||||
source .venv/Scripts/activate
|
||||
pip install -r requirements/requirements-convert_hf_to_gguf.txt
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
shell: cmd
|
||||
|
||||
@@ -187,6 +187,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
|
||||
@@ -61,6 +61,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Install OpenCL Headers and Libs
|
||||
id: install_opencl
|
||||
run: |
|
||||
|
||||
@@ -213,6 +213,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
|
||||
@@ -900,6 +900,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
@@ -1146,6 +1151,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
# TODO: these jobs need to use llvm toolchain in order to utilize the ccache
|
||||
#- name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
|
||||
@@ -175,6 +175,11 @@ jobs:
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
shell: cmd
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<b>LLM inference in C/C++</b>
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v1)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v1)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=b)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
|
||||
@@ -1313,6 +1313,11 @@ common_init_result::common_init_result(common_params & params, bool model_only)
|
||||
LOG_INF("%s", "decision model reads the embeddings output, enabling embedding mode\n");
|
||||
}
|
||||
|
||||
// 0 is n_batch
|
||||
if ((decision_type == COMMON_DECISION_TYPE_LFM3_D1 || decision_type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) && cparams.n_outputs_max != 0) {
|
||||
cparams.n_outputs_max = std::max(cparams.n_outputs_max, (uint32_t) COMMON_DECISION_LFM3_D1_SET_SIZE);
|
||||
}
|
||||
|
||||
if (llama_model_classifier_free_guidance(model)) {
|
||||
params.kv_unified = true;
|
||||
cparams.kv_unified = true;
|
||||
@@ -2334,6 +2339,9 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
|
||||
if (t.decision_order != 0) {
|
||||
llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order);
|
||||
}
|
||||
if (t.decision_group != 0 && !llama_batch_ext_set_decision_group(res, idx, t.decision_group)) {
|
||||
GGML_ABORT("%s: failed to set the decision group of the entry at index %d\n", __func__, i);
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
|
||||
@@ -1002,6 +1002,9 @@ enum common_decision_type {
|
||||
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
|
||||
};
|
||||
|
||||
// lfm3-d1: max questions packed in one prompt
|
||||
#define COMMON_DECISION_LFM3_D1_SET_SIZE 16
|
||||
|
||||
common_decision_type common_get_decision_type(const struct llama_model * model);
|
||||
|
||||
// metadata of a GGUF file, read without loading the model
|
||||
@@ -1117,6 +1120,7 @@ struct common_batch {
|
||||
llama_embd state; // non-owning view of the data passed to set_embd_state(), data == NULL if none
|
||||
std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
|
||||
int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
|
||||
int32_t decision_group = 0; // see llama_batch_ext_set_decision_group()
|
||||
int32_t embd_row = -1; // see add_embd_row(), -1 if none
|
||||
int32_t state_row = -1; // see set_embd_state_row(), -1 if none
|
||||
};
|
||||
|
||||
@@ -287,6 +287,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"TinyVocaModel": "tiny_voca",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"VoxCPM2Model": "voxcpm",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
@@ -367,6 +368,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"FishOmniModel": "drama3",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"VoxCPM2Model": "voxcpm",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Gemma3ForConditionalGeneration": "gemma",
|
||||
"Gemma3nForConditionalGeneration": "gemma",
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, MmprojModel, TextModel, LazyTorchTensor, gguf, logger
|
||||
from .drama3 import fold_snake
|
||||
from .minicpm import MiniCPMModel
|
||||
|
||||
# sample rate of the AudioVAE decoder output, picks the row of the sample-rate conditioning
|
||||
VAE_OUT_SAMPLE_RATE = 48000
|
||||
|
||||
|
||||
def _load_config(dir_model: Path) -> dict[str, Any] | None:
|
||||
path = dir_model / "config.json"
|
||||
if not path.is_file():
|
||||
return None
|
||||
with open(path, encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
return config if config.get("architecture") == "voxcpm2" else None
|
||||
|
||||
|
||||
def _is_voxcpm2_checkpoint(dir_model: Path) -> bool:
|
||||
return _load_config(dir_model) is not None
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_voxcpm2_checkpoint)
|
||||
def _load_voxcpm2_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected VoxCPM2 checkpoint")
|
||||
config = _load_config(dir_model)
|
||||
assert config is not None
|
||||
if config["lm_config"].get("use_mup"):
|
||||
raise NotImplementedError("VoxCPM2: use_mup = true is not supported")
|
||||
if config["dit_config"].get("mean_mode"):
|
||||
raise NotImplementedError("VoxCPM2: dit mean_mode = true is not supported")
|
||||
if not config.get("residual_lm_no_rope"):
|
||||
raise NotImplementedError("VoxCPM2: residual LM with RoPE is not supported")
|
||||
for key in ("hidden_dim", "ffn_dim", "num_heads", "num_layers", "kv_channels"):
|
||||
if config["encoder_config"][key] != config["dit_config"][key]:
|
||||
raise NotImplementedError(f"VoxCPM2: LocEnc and LocDiT differ in {key}")
|
||||
return {**config, "architectures": ["VoxCPM2Model"], "text_config": config["lm_config"], "audio_config": config["dit_config"]}
|
||||
|
||||
|
||||
@ModelBase.register("VoxCPM2Model")
|
||||
@ModelBase.example("openbmb/VoxCPM2")
|
||||
class VoxCPM2TextModel(MiniCPMModel):
|
||||
model_arch = gguf.MODEL_ARCH.MINICPM
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
# the text-semantic LM is a plain MiniCPM, the rest goes to the mmproj
|
||||
if not name.startswith("base_lm."):
|
||||
return None
|
||||
return super().filter_tensors(("model." + name.removeprefix("base_lm."), gen))
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
TextModel.set_gguf_parameters(self)
|
||||
# use_mup = false: no embedding, residual or logit scaling
|
||||
self.gguf_writer.add_embedding_scale(1.0)
|
||||
self.gguf_writer.add_residual_scale(1.0)
|
||||
self.gguf_writer.add_logit_scale(1.0)
|
||||
|
||||
def set_vocab(self):
|
||||
# the checkpoint has no tokenizer.model and its tokenizer class is remote code, so read tokenizer.json
|
||||
# SPM-style BPE: a merged token scores minus its merge rank, so the SPM tokenizer merges in the BPE order
|
||||
with open(self.dir_model / "tokenizer.json", encoding="utf-8") as f:
|
||||
tok = json.load(f)
|
||||
with open(self.dir_model / "tokenizer_config.json", encoding="utf-8") as f:
|
||||
tok_cfg = json.load(f)
|
||||
model = tok["model"]
|
||||
if model["type"] != "BPE" or not model.get("byte_fallback"):
|
||||
raise NotImplementedError("VoxCPM2: unexpected tokenizer type")
|
||||
|
||||
vocab: dict[str, int] = model["vocab"]
|
||||
vocab_size = self.hparams["vocab_size"]
|
||||
tokens = [f"[PAD{i}]" for i in range(vocab_size)]
|
||||
scores = [-1e9] * vocab_size
|
||||
toktypes = [gguf.TokenType.UNUSED] * vocab_size
|
||||
for text, i in vocab.items():
|
||||
tokens[i] = text
|
||||
toktypes[i] = gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-Fa-f]{2}>", text) else gguf.TokenType.NORMAL
|
||||
for rank, merge in enumerate(model["merges"]):
|
||||
a, b = merge.split(" ") if isinstance(merge, str) else merge
|
||||
i = vocab[a + b]
|
||||
scores[i] = max(scores[i], -float(rank))
|
||||
|
||||
# the special tokens (e.g. <|audio_start|>) are only listed in tokenizer_config.json
|
||||
specials = {int(i): t["content"] for i, t in tok_cfg.get("added_tokens_decoder", {}).items() if t.get("special")}
|
||||
specials.update({t["id"]: t["content"] for t in tok["added_tokens"] if t.get("special")})
|
||||
for i, text in specials.items():
|
||||
tokens[i] = text
|
||||
toktypes[i] = gguf.TokenType.CONTROL
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("llama")
|
||||
self.gguf_writer.add_tokenizer_pre("default")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores(scores)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
|
||||
@ModelBase.register("VoxCPM2Model")
|
||||
@ModelBase.example("openbmb/VoxCPM2")
|
||||
class VoxCPM2MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True # AudioVAE encoder, for voice cloning
|
||||
|
||||
_LAYER_MAP = {
|
||||
"input_layernorm.weight": "ln1",
|
||||
"self_attn.q_proj.weight": "attn_q",
|
||||
"self_attn.k_proj.weight": "attn_k",
|
||||
"self_attn.v_proj.weight": "attn_v",
|
||||
"self_attn.o_proj.weight": "attn_out",
|
||||
"post_attention_layernorm.weight": "ln2",
|
||||
"mlp.gate_proj.weight": "ffn_gate",
|
||||
"mlp.up_proj.weight": "ffn_up",
|
||||
"mlp.down_proj.weight": "ffn_down",
|
||||
}
|
||||
|
||||
# libllama names, mtmd loads the residual LM as a text model
|
||||
_RES_LAYER_MAP = {
|
||||
"input_layernorm.weight": "attn_norm",
|
||||
"self_attn.q_proj.weight": "attn_q",
|
||||
"self_attn.k_proj.weight": "attn_k",
|
||||
"self_attn.v_proj.weight": "attn_v",
|
||||
"self_attn.o_proj.weight": "attn_output",
|
||||
"post_attention_layernorm.weight": "ffn_norm",
|
||||
"mlp.gate_proj.weight": "ffn_gate",
|
||||
"mlp.up_proj.weight": "ffn_up",
|
||||
"mlp.down_proj.weight": "ffn_down",
|
||||
}
|
||||
|
||||
_PROJ_MAP = {
|
||||
"fsq_layer.in_proj": "a.gen.fsq.in",
|
||||
"fsq_layer.out_proj": "a.gen.fsq.out",
|
||||
"enc_to_lm_proj": "a.gen.enc_to_lm",
|
||||
"lm_to_dit_proj": "a.gen.lm_to_dit",
|
||||
"res_to_dit_proj": "a.gen.res_to_dit",
|
||||
"fusion_concat_proj": "a.gen.fusion",
|
||||
"stop_proj": "a.gen.stop_proj",
|
||||
"stop_head": "a.gen.stop_head",
|
||||
"feat_encoder.in_proj": "a.gen.enc.in_proj",
|
||||
"feat_decoder.estimator.in_proj": "a.gen.dit.in_proj",
|
||||
"feat_decoder.estimator.cond_proj": "a.gen.dit.cond_proj",
|
||||
"feat_decoder.estimator.out_proj": "a.gen.dit.out_proj",
|
||||
"feat_decoder.estimator.time_mlp.linear_1": "a.gen.dit.time_mlp.0",
|
||||
"feat_decoder.estimator.time_mlp.linear_2": "a.gen.dit.time_mlp.1",
|
||||
}
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
lm = self.global_config["lm_config"]
|
||||
dit = self.global_config["dit_config"]
|
||||
vae = self.global_config["audio_vae_config"]
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
|
||||
# LocEnc + LocDiT + residual LM + AudioVAE decoder, LocEnc has the hparams of LocDiT
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.VOXCPM2_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(dit["hidden_dim"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(dit["ffn_dim"])
|
||||
self.gguf_writer.add_gen_audio_block_count(dit["num_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(dit["num_heads"])
|
||||
self.gguf_writer.add_gen_audio_head_count_kv(lm["num_key_value_heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(lm["rms_norm_eps"])
|
||||
|
||||
# AudioVAE encoder: raw waveform -> latents, LocEnc in the gen model then makes the backbone rows
|
||||
self.gguf_writer.add_clip_has_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.VOXCPM2_SPKENC)
|
||||
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_audio_num_mel_bins(1)
|
||||
self.gguf_writer.add_audio_block_count(0)
|
||||
self.gguf_writer.add_audio_embedding_length(vae["encoder_dim"])
|
||||
self.gguf_writer.add_audio_feed_forward_length(vae["encoder_dim"])
|
||||
self.gguf_writer.add_audio_head_count(1)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(lm["rms_norm_eps"])
|
||||
# the ODE solver, the RoPE factors and the VAE layout are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# the AudioVAE convs and the 1D tensors stay F32
|
||||
if new_name.startswith(("a.vae.", "a.gen.vae.")) or n_dims < 2:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
if item[0].startswith("base_lm."):
|
||||
return None
|
||||
return item
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# already named by generate_extra_tensors()
|
||||
if name.startswith("a."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
# dt is always 0 (mean_mode = false), delta_time_mlp gives a constant that is folded into the time_mlp bias
|
||||
if name.startswith("feat_decoder.estimator.delta_time_mlp."):
|
||||
return
|
||||
if name == "feat_decoder.estimator.time_mlp.linear_2.bias":
|
||||
data_torch = LazyTorchTensor.to_eager(data_torch).float() + self._dtime_const()
|
||||
|
||||
if name == "feat_encoder.special_token":
|
||||
yield ("a.gen.enc.special_token", data_torch.reshape(-1))
|
||||
return
|
||||
|
||||
for src, dst in (("residual_lm.", "a.gen.res"), ("feat_encoder.encoder.", "a.gen.enc"),
|
||||
("feat_decoder.estimator.decoder.", "a.gen.code")):
|
||||
if not name.startswith(src):
|
||||
continue
|
||||
key = name.removeprefix(src)
|
||||
if key == "norm.weight":
|
||||
yield (f"{dst}.output_norm.weight" if dst == "a.gen.res" else f"{dst}.post_ln.weight", data_torch)
|
||||
return
|
||||
assert bid is not None, name
|
||||
key = key.removeprefix(f"layers.{bid}.")
|
||||
layer_map = self._RES_LAYER_MAP if dst == "a.gen.res" else self._LAYER_MAP
|
||||
yield (f"{dst}.blk.{bid}.{layer_map[key]}.weight", data_torch)
|
||||
return
|
||||
|
||||
base, _, suffix = name.rpartition(".")
|
||||
yield (f"{self._PROJ_MAP[base]}.{suffix}", data_torch)
|
||||
|
||||
# delta_time_mlp(SinusoidalPosEmb(0)), see VoxCPMLocDiTV2.forward()
|
||||
def _dtime_const(self) -> Tensor:
|
||||
d = {k: LazyTorchTensor.to_eager(self.model_tensors[f"feat_decoder.estimator.delta_time_mlp.{k}"]()).float()
|
||||
for k in ("linear_1.weight", "linear_1.bias", "linear_2.weight", "linear_2.bias")}
|
||||
dim = d["linear_1.weight"].shape[1]
|
||||
emb = torch.cat([torch.zeros(dim // 2), torch.ones(dim // 2)]) # sin(0), cos(0)
|
||||
h = torch.nn.functional.silu(d["linear_1.weight"] @ emb + d["linear_1.bias"])
|
||||
return d["linear_2.weight"] @ h + d["linear_2.bias"]
|
||||
|
||||
@staticmethod
|
||||
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
|
||||
return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# LongRoPE factors of LocEnc and LocDiT, picked as in MiniCPMLongRoPE._set_cos_sin_cache()
|
||||
lm = self.global_config["lm_config"]
|
||||
rope = lm["rope_scaling"]
|
||||
if rope.get("type") != "longrope":
|
||||
raise NotImplementedError(f"VoxCPM2: unexpected rope_scaling {rope.get('type')}")
|
||||
if lm["max_position_embeddings"] > rope["original_max_position_embeddings"]:
|
||||
raise NotImplementedError("VoxCPM2: the long RoPE factors also scale cos/sin, not supported")
|
||||
yield ("a.gen.rope_factors", torch.tensor(rope["short_factor"], dtype=torch.float32))
|
||||
|
||||
path = self.dir_model / "audiovae.pth"
|
||||
logger.info(f"gguf: loading AudioVAE from '{path}'")
|
||||
sd = torch.load(str(path), map_location="cpu", mmap=True, weights_only=True)
|
||||
sd = sd.get("state_dict", sd)
|
||||
vae = self.global_config["audio_vae_config"]
|
||||
|
||||
def conv(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
yield (f"{dst}.weight", self._wn(sd, src))
|
||||
yield (f"{dst}.bias", sd[src + ".bias"].float())
|
||||
|
||||
def snake(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
alpha, beta = fold_snake(sd[src + ".alpha"])
|
||||
yield (f"{dst}.alpha", alpha)
|
||||
yield (f"{dst}.beta", beta)
|
||||
|
||||
def res_units(dst: str, src: str, first: int) -> Iterable[tuple[str, Tensor]]:
|
||||
# dilation 1, 3, 9: Snake, depthwise conv k = 7, Snake, conv k = 1
|
||||
for r in range(3):
|
||||
s = f"{src}.{first + r}.block"
|
||||
d = f"{dst}.res.{r}"
|
||||
yield from snake(f"{d}.snake1", f"{s}.0")
|
||||
yield from conv(f"{d}.conv1", f"{s}.1")
|
||||
yield from snake(f"{d}.snake2", f"{s}.2")
|
||||
yield from conv(f"{d}.conv2", f"{s}.3")
|
||||
|
||||
# encoder: conv, then per stride (3 residual units, Snake, strided conv), then fc_mu
|
||||
p = "a.vae.enc"
|
||||
yield from conv(f"{p}.conv_in", "encoder.block.0")
|
||||
for b in range(len(vae["encoder_rates"])):
|
||||
s = f"encoder.block.{b + 1}.block"
|
||||
yield from res_units(f"{p}.blk.{b}", s, 0)
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.3")
|
||||
yield from conv(f"{p}.blk.{b}.down", f"{s}.4")
|
||||
yield from conv(f"{p}.fc_mu", "encoder.fc_mu")
|
||||
|
||||
# decoder: depthwise conv, conv k = 1, then per stride (sample-rate scale and bias, Snake, transposed conv, 3 residual units), then Snake, conv
|
||||
bounds = vae["sr_bin_boundaries"]
|
||||
sr_idx = sum(1 for b in bounds if b < VAE_OUT_SAMPLE_RATE) # torch.bucketize
|
||||
if vae["out_sample_rate"] != VAE_OUT_SAMPLE_RATE:
|
||||
raise NotImplementedError(f"VoxCPM2: unexpected out_sample_rate {vae['out_sample_rate']}")
|
||||
p = "a.gen.vae"
|
||||
yield from conv(f"{p}.conv_in_dw", "decoder.model.0")
|
||||
yield from conv(f"{p}.conv_in", "decoder.model.1")
|
||||
n_up = len(vae["decoder_rates"])
|
||||
for b in range(n_up):
|
||||
c = f"decoder.sr_cond_model.{b + 2}"
|
||||
yield (f"{p}.blk.{b}.sr_scale", sd[c + ".scale_embed.weight"][sr_idx].float())
|
||||
yield (f"{p}.blk.{b}.sr_bias", sd[c + ".bias_embed.weight"][sr_idx].float())
|
||||
s = f"decoder.model.{b + 2}.block"
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.0")
|
||||
yield from conv(f"{p}.blk.{b}.up", f"{s}.1")
|
||||
yield from res_units(f"{p}.blk.{b}", s, 2)
|
||||
yield from snake(f"{p}.post_snake", f"decoder.model.{n_up + 2}")
|
||||
yield from conv(f"{p}.conv_out", f"decoder.model.{n_up + 3}")
|
||||
|
||||
assert math.prod(vae["encoder_rates"]) * VAE_OUT_SAMPLE_RATE == math.prod(vae["decoder_rates"]) * vae["sample_rate"]
|
||||
@@ -100,6 +100,12 @@ The OpenCL backend has the following CMake options that control the behavior of
|
||||
| `GGML_OPENCL_USE_ADRENO_KERNELS` | `ON` | Use kernels optimized for Adreno. |
|
||||
| `GGML_OPENCL_USE_ADRENO_BIN_KERNELS` | `OFF` | Allow using binary kernel lib for Adreno. |
|
||||
|
||||
## Decomposed Flash Attention
|
||||
|
||||
The P8 path rounds scratch row pitches for P and transposed V up to a multiple of 32 KiB before adding the existing 64-byte padding. This reduces cache conflicts when the KV length exceeds 16384 and the V head size is at most 256. Larger V heads keep the previous layout. The extra scratch space is less than 32 KiB per row at the default granule.
|
||||
|
||||
`GGML_OPENCL_FA_KV_PITCH_ROUND` sets the rounding granule in bytes to a positive power of two; `0` restores the previous layout. `GGML_OPENCL_FA_KV_PITCH_ROUND_MIN` sets the KV-length threshold (default `16384`), and `GGML_OPENCL_FA_KV_PITCH_ROUND_MAX_DV` sets the largest V head size to round (default `256`). These settings do not change the existing attention dispatch conditions.
|
||||
|
||||
## Program Binary Cache
|
||||
|
||||
Compiled `cl_program` binaries are cached on disk, so subsequent runs skip the expensive
|
||||
|
||||
@@ -17,6 +17,7 @@ Supported models:
|
||||
- [MiniMax Music 3](#minimax-music-3) (songs)
|
||||
- [Kokoro](#kokoro)
|
||||
- [tiny-voca](#tiny-voca)
|
||||
- [VoxCPM2](#voxcpm2)
|
||||
|
||||
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
|
||||
|
||||
@@ -467,3 +468,45 @@ pip install git+https://github.com/n4ze3m/dizione
|
||||
python convert_hf_to_gguf.py path/to/tiny-voca --outfile tiny-voca.gguf
|
||||
python convert_hf_to_gguf.py path/to/tiny-voca --mmproj --outfile mmproj-tiny-voca.gguf
|
||||
```
|
||||
|
||||
## VoxCPM2
|
||||
|
||||
[VoxCPM2](https://huggingface.co/openbmb/VoxCPM2): a MiniCPM4 backbone (2B) makes one hidden state per frame, without an audio token. The mmproj turns it into a patch of 4 continuous latents: FSQ, an 8-layer residual LM (run with libllama, one context per sequence), then the LocDiT flow matching. The causal AudioVAE decodes the latents to 48 kHz audio.
|
||||
|
||||
Available params:
|
||||
- The language is the one of the text, `--tts-lang` is not used. A voice description in parentheses at the start of the text designs the voice, for example `(A young woman, gentle and warm)Hello!`
|
||||
- Voice cloning: `--tts-speaker-file` with the reference clip (resampled to 16 kHz, truncated to the first 30 s). `--tts-speaker-text` is not used, the transcript of the clip is not needed
|
||||
- `--temp` scales the noise of the flow matching (default 1.0), `--top-k` and `--top-p` are not used
|
||||
- The flow matching uses 10 Euler steps and classifier-free guidance with scale 2.0, like the reference. `llama-server` takes `flow_steps` (2 to 32) and `cfg_scale` to change them
|
||||
- Each frame is 4 latents, 7680 samples (160 ms). Generation stops on the stop head of the model, at least 4 frames, at most 6 frames per text token + 10 (the limit of the reference)
|
||||
- Not supported yet: the continuation mode of the reference (prompt audio with its transcript)
|
||||
- Not supported yet: the bad case retry of the reference (`retry_badcase`, a new seed when the audio reaches 6 frames per text token). Short prompts may then run long and repeat some words, try another seed
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m VoxCPM2.gguf \
|
||||
-mm mmproj-VoxCPM2.gguf \
|
||||
-p "(A calm male narrator)The quick brown fox jumps over the lazy dog." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Voice cloning:
|
||||
|
||||
```sh
|
||||
llama-tts -m VoxCPM2.gguf \
|
||||
-mm mmproj-VoxCPM2.gguf \
|
||||
--tts-speaker-file reference.wav \
|
||||
-p "Hello, this is a cloned voice." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The mmproj holds LocEnc, LocDiT and the residual LM layers (loaded with libllama as a Granite text model with RoPE off) in the `--outtype` type, and the AudioVAE (`audiovae.pth`, encoder for voice cloning, decoder for the audio) always in F32:
|
||||
|
||||
```sh
|
||||
hf download openbmb/VoxCPM2 --local-dir VoxCPM2
|
||||
python convert_hf_to_gguf.py VoxCPM2 --outfile VoxCPM2.gguf
|
||||
python convert_hf_to_gguf.py VoxCPM2 --mmproj --outfile mmproj-VoxCPM2.gguf
|
||||
```
|
||||
|
||||
+1
-1
@@ -90,7 +90,7 @@ Legend:
|
||||
| PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ |
|
||||
| POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
|
||||
+128
-128
@@ -1155,134 +1155,134 @@
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[1,8,1,3],nr23=[1,1],r=2,v=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -8,10 +8,10 @@ extern "C" {
|
||||
|
||||
#define RPC_PROTO_MAJOR_VERSION 10
|
||||
#define RPC_PROTO_MINOR_VERSION 0
|
||||
#define RPC_PROTO_PATCH_VERSION 0
|
||||
#define RPC_PROTO_PATCH_VERSION 1
|
||||
|
||||
#ifdef __cplusplus
|
||||
static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
|
||||
static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
|
||||
#endif
|
||||
|
||||
#define GGML_RPC_MAX_SERVERS 16
|
||||
|
||||
@@ -429,6 +429,8 @@ extern "C" {
|
||||
|
||||
GGML_OP_GLU,
|
||||
|
||||
GGML_OP_SLEEP,
|
||||
|
||||
GGML_OP_COUNT,
|
||||
};
|
||||
|
||||
@@ -2599,6 +2601,12 @@ extern "C" {
|
||||
int n_tasks,
|
||||
void * userdata);
|
||||
|
||||
// Busy-wait operator for scheduling and backend testing.
|
||||
GGML_API struct ggml_tensor * ggml_sleep(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a, // wait after this tensor has executed, return value is simply a full view of this tensor
|
||||
int32_t us); // amount of time to wait in microseconds
|
||||
|
||||
// loss function
|
||||
|
||||
GGML_API struct ggml_tensor * ggml_cross_entropy_loss(
|
||||
|
||||
@@ -231,7 +231,7 @@ if (GGML_OPENMP_FETCH)
|
||||
message(FATAL_ERROR "GGML_OPENMP_FETCH currently requires Clang on Windows")
|
||||
endif()
|
||||
|
||||
set(GGML_OPENMP_LLVM_VERSION "20.1.8")
|
||||
set(GGML_OPENMP_LLVM_VERSION "22.1.8")
|
||||
string(REGEX MATCH "^[0-9]+" GGML_OPENMP_LLVM_VERSION_MAJOR "${GGML_OPENMP_LLVM_VERSION}")
|
||||
string(REGEX MATCH "^[0-9]+" GGML_OPENMP_COMPILER_VERSION_MAJOR "${CMAKE_C_COMPILER_VERSION}")
|
||||
if (NOT GGML_OPENMP_COMPILER_VERSION_MAJOR STREQUAL GGML_OPENMP_LLVM_VERSION_MAJOR)
|
||||
@@ -242,11 +242,11 @@ if (GGML_OPENMP_FETCH)
|
||||
if (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(amd64|x86_64)$")
|
||||
set(GGML_OPENMP_ARCH "x64")
|
||||
set(GGML_OPENMP_INSTALLER_SUFFIX "win64")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "3197846a2b19063687dd56e93e34cd941e3548d907f23a6131571321bdf9fe7b")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "16e5709785fef73c854646241c4a92c5cd574318d1b33c63330dd7721903e55c")
|
||||
elseif (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$")
|
||||
set(GGML_OPENMP_ARCH "arm64")
|
||||
set(GGML_OPENMP_INSTALLER_SUFFIX "woa64")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "7c4ac97eb2ae6b960ca5f9caf3ff6124c8d2a18cc07a7840a4d2ea15537bad8e")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "76f44ef1ba6eeb5a65904e9500f042f588fade49952778ce48f0374daa934396")
|
||||
else()
|
||||
message(FATAL_ERROR "GGML_OPENMP_FETCH does not support ${CMAKE_SYSTEM_PROCESSOR}")
|
||||
endif()
|
||||
|
||||
@@ -2584,6 +2584,7 @@ bool ggml_op_alloc_size_may_expand(enum ggml_op op) {
|
||||
case GGML_OP_ARGSORT:
|
||||
case GGML_OP_TOP_K:
|
||||
case GGML_OP_ARGMAX:
|
||||
case GGML_OP_CONV_2D:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -262,13 +262,22 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
v128_t v0_0 = wasm_v128_load(x0->qs);
|
||||
v128_t v0_0l = wasm_v128_and(v0_0, m4b);
|
||||
v128_t v0_0h = wasm_u8x16_shr(v0_0, 4);
|
||||
v128_t v0_0ls = wasm_i8x16_sub(v0_0l, s8b);
|
||||
v128_t v0_0hs = wasm_i8x16_sub(v0_0h, s8b);
|
||||
|
||||
// Load y0 vectors
|
||||
v128_t y0_l = wasm_v128_load(y0->qs);
|
||||
v128_t y0_h = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// unsigned nibbles (0..15) in the 7-bit operand, then subtract 8*sum(y)
|
||||
v128_t dp0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_l, v0_0l, wasm_i32x4_splat(0));
|
||||
dp0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_h, v0_0h, dp0);
|
||||
v128_t ys0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_l, s8b, wasm_i32x4_splat(0));
|
||||
ys0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_h, s8b, ys0);
|
||||
dp0 = wasm_i32x4_sub(dp0, ys0);
|
||||
# else
|
||||
v128_t v0_0ls = wasm_i8x16_sub(v0_0l, s8b);
|
||||
v128_t v0_0hs = wasm_i8x16_sub(v0_0h, s8b);
|
||||
|
||||
// Extend to i16x8 and compute dot products
|
||||
v128_t dx0l = wasm_i16x8_extend_low_i8x16(v0_0ls);
|
||||
v128_t dx0h = wasm_i16x8_extend_high_i8x16(v0_0ls);
|
||||
@@ -290,18 +299,27 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_dot_i16x8(dx0hh, dy0hh)
|
||||
)
|
||||
);
|
||||
# endif
|
||||
|
||||
// Load and process x1
|
||||
v128_t v0_1 = wasm_v128_load(x1->qs);
|
||||
v128_t v0_1l = wasm_v128_and(v0_1, m4b);
|
||||
v128_t v0_1h = wasm_u8x16_shr(v0_1, 4);
|
||||
v128_t v0_1ls = wasm_i8x16_sub(v0_1l, s8b);
|
||||
v128_t v0_1hs = wasm_i8x16_sub(v0_1h, s8b);
|
||||
|
||||
// Load y1 vectors
|
||||
v128_t y1_l = wasm_v128_load(y1->qs);
|
||||
v128_t y1_h = wasm_v128_load(y1->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// unsigned nibbles (0..15) in the 7-bit operand, then subtract 8*sum(y)
|
||||
v128_t dp1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_l, v0_1l, wasm_i32x4_splat(0));
|
||||
dp1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_h, v0_1h, dp1);
|
||||
v128_t ys1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_l, s8b, wasm_i32x4_splat(0));
|
||||
ys1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_h, s8b, ys1);
|
||||
dp1 = wasm_i32x4_sub(dp1, ys1);
|
||||
# else
|
||||
v128_t v0_1ls = wasm_i8x16_sub(v0_1l, s8b);
|
||||
v128_t v0_1hs = wasm_i8x16_sub(v0_1h, s8b);
|
||||
// Extend to i16x8 and compute dot products
|
||||
v128_t dx1l = wasm_i16x8_extend_low_i8x16(v0_1ls);
|
||||
v128_t dx1h = wasm_i16x8_extend_high_i8x16(v0_1ls);
|
||||
@@ -323,6 +341,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_dot_i16x8(dx1hh, dy1hh)
|
||||
)
|
||||
);
|
||||
# endif
|
||||
|
||||
// Accumulate results with scaling
|
||||
float scale0 = GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d);
|
||||
@@ -461,10 +480,18 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
// extract the 5th bit
|
||||
memcpy(&qh_, x0->qh, sizeof(qh_));
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// the high bit as is: unsigned 5-bit quants for the 7-bit operand
|
||||
tmp[0] = table_b2b_0[(qh_ >> 0) & 0xFF];
|
||||
tmp[1] = table_b2b_0[(qh_ >> 8) & 0xFF];
|
||||
tmp[2] = table_b2b_0[(qh_ >> 16) & 0xFF];
|
||||
tmp[3] = table_b2b_0[(qh_ >> 24) ];
|
||||
# else
|
||||
tmp[0] = table_b2b_1[(qh_ >> 0) & 0xFF];
|
||||
tmp[1] = table_b2b_1[(qh_ >> 8) & 0xFF];
|
||||
tmp[2] = table_b2b_1[(qh_ >> 16) & 0xFF];
|
||||
tmp[3] = table_b2b_1[(qh_ >> 24) ];
|
||||
# endif
|
||||
|
||||
const v128_t qhl = wasm_v128_load(tmp + 0);
|
||||
const v128_t qhh = wasm_v128_load(tmp + 2);
|
||||
@@ -475,14 +502,29 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
const v128_t v0l = wasm_v128_and (v0, m4b);
|
||||
const v128_t v0h = wasm_u8x16_shr(v0, 4);
|
||||
|
||||
// add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
|
||||
const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);
|
||||
const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);
|
||||
|
||||
// load y
|
||||
const v128_t v1l = wasm_v128_load(y0->qs);
|
||||
const v128_t v1h = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// unsigned 5-bit quants (0..31) in the 7-bit operand, then subtract 16*sum(y)
|
||||
const v128_t s16b = wasm_i8x16_splat(0x10);
|
||||
const v128_t v0lu = wasm_v128_or(v0l, qhl);
|
||||
const v128_t v0hu = wasm_v128_or(v0h, qhh);
|
||||
|
||||
v128_t dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1l, v0lu, wasm_i32x4_splat(0));
|
||||
dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1h, v0hu, dp);
|
||||
v128_t ys = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1l, s16b, wasm_i32x4_splat(0));
|
||||
ys = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1h, s16b, ys);
|
||||
dp = wasm_i32x4_sub(dp, ys);
|
||||
|
||||
sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(dp),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# else
|
||||
// add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
|
||||
const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);
|
||||
const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);
|
||||
|
||||
// int8x16 -> int16x8
|
||||
const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);
|
||||
const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);
|
||||
@@ -502,6 +544,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),
|
||||
wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# endif
|
||||
}
|
||||
|
||||
sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
|
||||
@@ -578,6 +621,13 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
const v128_t v1l = wasm_v128_load(y0->qs);
|
||||
const v128_t v1h = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1l, v0lf, wasm_i32x4_splat(0));
|
||||
dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1h, v0hf, dp);
|
||||
|
||||
sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(dp),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# else
|
||||
// int8x16 -> int16x8
|
||||
const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);
|
||||
const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);
|
||||
@@ -597,6 +647,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),
|
||||
wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# endif
|
||||
}
|
||||
|
||||
sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
|
||||
@@ -642,6 +693,17 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
const v128_t y0_0 = wasm_v128_load(y0->qs);
|
||||
const v128_t y0_1 = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// |x| (0..127) in the 7-bit operand and y*sign(x) in the other; relies on
|
||||
// q8_0 values being in -127..127, as ggml's quantizers produce
|
||||
const v128_t zero = wasm_i8x16_splat(0);
|
||||
const v128_t sx0 = wasm_i8x16_lt(x0_0, zero);
|
||||
const v128_t sx1 = wasm_i8x16_lt(x0_1, zero);
|
||||
const v128_t ys0 = wasm_i8x16_sub(wasm_v128_xor(y0_0, sx0), sx0);
|
||||
const v128_t ys1 = wasm_i8x16_sub(wasm_v128_xor(y0_1, sx1), sx1);
|
||||
v128_t sum_dots = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(ys0, wasm_i8x16_abs(x0_0), wasm_i32x4_splat(0));
|
||||
sum_dots = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(ys1, wasm_i8x16_abs(x0_1), sum_dots);
|
||||
# else
|
||||
// Extend 8-bit to 16-bit
|
||||
const v128_t x0_0l = wasm_i16x8_extend_low_i8x16(x0_0);
|
||||
const v128_t x0_0h = wasm_i16x8_extend_high_i8x16(x0_0);
|
||||
@@ -661,7 +723,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
|
||||
// Sum all dot products
|
||||
const v128_t sum_dots = wasm_i32x4_add(wasm_i32x4_add(dx0_0, dx0_1), wasm_i32x4_add(dx1_0, dx1_1));
|
||||
|
||||
# endif
|
||||
// Convert to float and accumulate
|
||||
const float scale = GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d);
|
||||
sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(sum_dots), wasm_f32x4_splat(scale)));
|
||||
@@ -750,6 +812,10 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
v128_t q2_shift_1 = wasm_u8x16_shr(q2_1, shift);
|
||||
v128_t q2_bits_1 = wasm_v128_and(q2_shift_1, wasm_i8x16_splat(0x03));
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t p_sum0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_0, q2_bits_0, wasm_i32x4_splat(0));
|
||||
v128_t p_sum1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_1, q2_bits_1, wasm_i32x4_splat(0));
|
||||
# else
|
||||
// Calculate dot products
|
||||
v128_t p0 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q8_0),
|
||||
@@ -768,10 +834,14 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q2_bits_1)
|
||||
);
|
||||
|
||||
v128_t p_sum0 = wasm_i32x4_add(p0, p1);
|
||||
v128_t p_sum1 = wasm_i32x4_add(p2, p3);
|
||||
# endif
|
||||
|
||||
// Accumulate scaled results
|
||||
v128_t scaled = wasm_i32x4_add(
|
||||
wasm_i32x4_mul(wasm_i32x4_add(p0, p1), wasm_i32x4_splat(d0)),
|
||||
wasm_i32x4_mul(wasm_i32x4_add(p2, p3), wasm_i32x4_splat(d1))
|
||||
wasm_i32x4_mul(p_sum0, wasm_i32x4_splat(d0)),
|
||||
wasm_i32x4_mul(p_sum1, wasm_i32x4_splat(d1))
|
||||
);
|
||||
|
||||
isum_vec = wasm_i32x4_add(isum_vec, scaled);
|
||||
@@ -817,7 +887,84 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
|
||||
const int nb = n / QK_K;
|
||||
|
||||
#if defined __wasm_simd128__
|
||||
#if defined(__wasm_relaxed_simd__)
|
||||
// Unsigned 3-bit quants (0..7) in the relaxed dot's 7-bit operand, q8 in
|
||||
// the other; the -4 offset is taken off with the q8 block sums.
|
||||
const v128_t m3 = wasm_i8x16_splat(0x03);
|
||||
const v128_t m1 = wasm_i8x16_splat(0x01);
|
||||
const v128_t zero = wasm_i32x4_splat(0);
|
||||
uint32_t auxs[4];
|
||||
|
||||
float sumf = 0;
|
||||
|
||||
for (int i = 0; i < nb; ++i) {
|
||||
const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
|
||||
|
||||
const uint8_t * GGML_RESTRICT q3 = x[i].qs;
|
||||
const int8_t * GGML_RESTRICT q8 = y[i].qs;
|
||||
|
||||
// Extract scales
|
||||
memcpy(auxs, x[i].scales, 12);
|
||||
uint32_t tmp = auxs[2];
|
||||
auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);
|
||||
auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);
|
||||
auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);
|
||||
auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);
|
||||
|
||||
const v128_t vsc = wasm_i8x16_sub(wasm_v128_load(auxs), wasm_i8x16_splat(32));
|
||||
const v128_t vsc_l = wasm_i16x8_extend_low_i8x16(vsc);
|
||||
const v128_t vsc_h = wasm_i16x8_extend_high_i8x16(vsc);
|
||||
|
||||
// sum(scale * bsum), for the -4 offset
|
||||
const v128_t vmins = wasm_i32x4_add(
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums), vsc_l),
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums + 8), vsc_h));
|
||||
|
||||
const v128_t hm0 = wasm_v128_load(x[i].hmask);
|
||||
const v128_t hm1 = wasm_v128_load(x[i].hmask + 16);
|
||||
|
||||
v128_t acc = zero;
|
||||
v128_t acc2 = zero;
|
||||
|
||||
for (int j = 0; j < QK_K/128; ++j) {
|
||||
const v128_t q3_0 = wasm_v128_load(q3);
|
||||
const v128_t q3_1 = wasm_v128_load(q3 + 16);
|
||||
q3 += 32;
|
||||
|
||||
const v128_t s32a = wasm_i32x4_extend_low_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
const v128_t s32b = wasm_i32x4_extend_high_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
const v128_t s32[2] = { s32a, s32b };
|
||||
|
||||
for (int g = 0; g < 4; ++g) {
|
||||
// 2 low bits at shift 2g, high bit 4j+g of hmask moved to bit 2
|
||||
// (16-bit shifts are safe on masked bytes)
|
||||
const int hb = 4*j + g;
|
||||
const v128_t u0 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(q3_0, 2*g), m3),
|
||||
wasm_i16x8_shl(wasm_v128_and(wasm_u16x8_shr(hm0, hb), m1), 2));
|
||||
const v128_t u1 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(q3_1, 2*g), m3),
|
||||
wasm_i16x8_shl(wasm_v128_and(wasm_u16x8_shr(hm1, hb), m1), 2));
|
||||
|
||||
const v128_t p0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8), u0, zero);
|
||||
const v128_t p1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 16), u1, zero);
|
||||
q8 += 32;
|
||||
|
||||
const v128_t sv = s32[g >> 1];
|
||||
const v128_t sc0 = (g & 1) ? wasm_i32x4_shuffle(sv, sv, 2, 2, 2, 2) : wasm_i32x4_shuffle(sv, sv, 0, 0, 0, 0);
|
||||
const v128_t sc1 = (g & 1) ? wasm_i32x4_shuffle(sv, sv, 3, 3, 3, 3) : wasm_i32x4_shuffle(sv, sv, 1, 1, 1, 1);
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p0, sc0));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p1, sc1));
|
||||
}
|
||||
}
|
||||
|
||||
acc = wasm_i32x4_sub(wasm_i32x4_add(acc, acc2), wasm_i32x4_shl(vmins, 2));
|
||||
const int32_t isum = wasm_i32x4_extract_lane(acc, 0) + wasm_i32x4_extract_lane(acc, 1) +
|
||||
wasm_i32x4_extract_lane(acc, 2) + wasm_i32x4_extract_lane(acc, 3);
|
||||
sumf += d * isum;
|
||||
}
|
||||
|
||||
*s = sumf;
|
||||
|
||||
#elif defined __wasm_simd128__
|
||||
int8_t aux8[QK_K];
|
||||
float sums[8] = {0};
|
||||
uint32_t auxs[4];
|
||||
@@ -986,6 +1133,14 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
q8 += 64;
|
||||
|
||||
// Low nibble products
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t vacc1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x0, q4l0, wasm_i32x4_splat(0)
|
||||
);
|
||||
vacc1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x1, q4l1, vacc1
|
||||
);
|
||||
# else
|
||||
v128_t vacc1 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q4l0),
|
||||
wasm_i16x8_extend_low_i8x16(q8x0)
|
||||
@@ -1002,8 +1157,17 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q4l1),
|
||||
wasm_i16x8_extend_high_i8x16(q8x1)
|
||||
));
|
||||
# endif
|
||||
|
||||
// High nibble products
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t vacc2 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x2, q4h0, wasm_i32x4_splat(0)
|
||||
);
|
||||
vacc2 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x3, q4h1, vacc2
|
||||
);
|
||||
# else
|
||||
v128_t vacc2 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q4h0),
|
||||
wasm_i16x8_extend_low_i8x16(q8x2)
|
||||
@@ -1020,6 +1184,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q4h1),
|
||||
wasm_i16x8_extend_high_i8x16(q8x3)
|
||||
));
|
||||
# endif
|
||||
|
||||
// Accumulate scaled results
|
||||
int32_t vacc1_sum = wasm_i32x4_extract_lane(vacc1, 0) + wasm_i32x4_extract_lane(vacc1, 1) +
|
||||
@@ -1127,6 +1292,11 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
v128_t q8_3 = wasm_v128_load(q8 + 48);
|
||||
q8 += 64;
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// Process low quants using relaxed SIMD
|
||||
v128_t sum_low = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_0, q5l_0, wasm_i32x4_splat(0));
|
||||
sum_low = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_1, q5l_1, sum_low);
|
||||
# else
|
||||
// Process low quants
|
||||
v128_t pl0 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q5l_0),
|
||||
@@ -1145,7 +1315,13 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q8_1)
|
||||
));
|
||||
v128_t sum_low = wasm_i32x4_add(pl0, pl1);
|
||||
# endif
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// Process high quants using relaxed SIMD
|
||||
v128_t sum_high = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_2, q5h_0, wasm_i32x4_splat(0));
|
||||
sum_high = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_3, q5h_1, sum_high);
|
||||
# else
|
||||
// Process high quants
|
||||
v128_t ph0 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q5h_0),
|
||||
@@ -1164,6 +1340,7 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q8_3)
|
||||
));
|
||||
v128_t sum_high = wasm_i32x4_add(ph0, ph1);
|
||||
# endif
|
||||
|
||||
// Accumulate with scale factors
|
||||
int32_t sl = wasm_i32x4_extract_lane(sum_low, 0) + wasm_i32x4_extract_lane(sum_low, 1) +
|
||||
@@ -1204,7 +1381,78 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
|
||||
const int nb = n / QK_K;
|
||||
|
||||
#if defined __wasm_simd128__
|
||||
#if defined(__wasm_relaxed_simd__)
|
||||
const v128_t m4 = wasm_i8x16_splat(0x0F);
|
||||
const v128_t m2 = wasm_i8x16_splat(0x03);
|
||||
const v128_t zero = wasm_i32x4_splat(0);
|
||||
|
||||
float sumf = 0;
|
||||
|
||||
for (int i = 0; i < nb; ++i) {
|
||||
const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
|
||||
const uint8_t * GGML_RESTRICT ql = x[i].ql;
|
||||
const uint8_t * GGML_RESTRICT qh = x[i].qh;
|
||||
const int8_t * GGML_RESTRICT q8 = y[i].qs;
|
||||
|
||||
const v128_t vsc = wasm_v128_load(x[i].scales);
|
||||
const v128_t vsc_l = wasm_i16x8_extend_low_i8x16(vsc);
|
||||
const v128_t vsc_h = wasm_i16x8_extend_high_i8x16(vsc);
|
||||
const v128_t vmins = wasm_i32x4_add(
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums), vsc_l),
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums + 8), vsc_h));
|
||||
|
||||
v128_t acc = zero;
|
||||
v128_t acc2 = zero;
|
||||
for (int j = 0; j < QK_K/128; ++j) {
|
||||
const v128_t ql0 = wasm_v128_load(ql);
|
||||
const v128_t ql1 = wasm_v128_load(ql + 16);
|
||||
const v128_t ql2 = wasm_v128_load(ql + 32);
|
||||
const v128_t ql3 = wasm_v128_load(ql + 48);
|
||||
const v128_t qh0 = wasm_v128_load(qh);
|
||||
const v128_t qh1 = wasm_v128_load(qh + 16);
|
||||
ql += 64;
|
||||
qh += 32;
|
||||
|
||||
const v128_t q6_0 = wasm_v128_or(wasm_v128_and(ql0, m4), wasm_i16x8_shl(wasm_v128_and(qh0, m2), 4));
|
||||
const v128_t q6_1 = wasm_v128_or(wasm_v128_and(ql1, m4), wasm_i16x8_shl(wasm_v128_and(qh1, m2), 4));
|
||||
const v128_t q6_2 = wasm_v128_or(wasm_v128_and(ql2, m4), wasm_i16x8_shl(wasm_v128_and(qh0, wasm_i8x16_splat(0x0C)), 2));
|
||||
const v128_t q6_3 = wasm_v128_or(wasm_v128_and(ql3, m4), wasm_i16x8_shl(wasm_v128_and(qh1, wasm_i8x16_splat(0x0C)), 2));
|
||||
const v128_t q6_4 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql0, 4), m4), wasm_v128_and(qh0, wasm_i8x16_splat(0x30)));
|
||||
const v128_t q6_5 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql1, 4), m4), wasm_v128_and(qh1, wasm_i8x16_splat(0x30)));
|
||||
const v128_t q6_6 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql2, 4), m4), wasm_u16x8_shr(wasm_v128_and(qh0, wasm_i8x16_splat((int8_t)0xC0)), 2));
|
||||
const v128_t q6_7 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql3, 4), m4), wasm_u16x8_shr(wasm_v128_and(qh1, wasm_i8x16_splat((int8_t)0xC0)), 2));
|
||||
|
||||
const v128_t p0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 0), q6_0, zero);
|
||||
const v128_t p1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 16), q6_1, zero);
|
||||
const v128_t p2 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 32), q6_2, zero);
|
||||
const v128_t p3 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 48), q6_3, zero);
|
||||
const v128_t p4 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 64), q6_4, zero);
|
||||
const v128_t p5 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 80), q6_5, zero);
|
||||
const v128_t p6 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 96), q6_6, zero);
|
||||
const v128_t p7 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 112), q6_7, zero);
|
||||
q8 += 128;
|
||||
|
||||
const v128_t s32a = wasm_i32x4_extend_low_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
const v128_t s32b = wasm_i32x4_extend_high_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p0, wasm_i32x4_shuffle(s32a, s32a, 0, 0, 0, 0)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p1, wasm_i32x4_shuffle(s32a, s32a, 1, 1, 1, 1)));
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p2, wasm_i32x4_shuffle(s32a, s32a, 2, 2, 2, 2)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p3, wasm_i32x4_shuffle(s32a, s32a, 3, 3, 3, 3)));
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p4, wasm_i32x4_shuffle(s32b, s32b, 0, 0, 0, 0)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p5, wasm_i32x4_shuffle(s32b, s32b, 1, 1, 1, 1)));
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p6, wasm_i32x4_shuffle(s32b, s32b, 2, 2, 2, 2)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p7, wasm_i32x4_shuffle(s32b, s32b, 3, 3, 3, 3)));
|
||||
}
|
||||
|
||||
acc = wasm_i32x4_sub(wasm_i32x4_add(acc, acc2), wasm_i32x4_shl(vmins, 5));
|
||||
const int32_t isum = wasm_i32x4_extract_lane(acc, 0) + wasm_i32x4_extract_lane(acc, 1) +
|
||||
wasm_i32x4_extract_lane(acc, 2) + wasm_i32x4_extract_lane(acc, 3);
|
||||
sumf += d * isum;
|
||||
}
|
||||
|
||||
*s = sumf;
|
||||
|
||||
#elif defined __wasm_simd128__
|
||||
int8_t aux8[QK_K] __attribute__((aligned(16)));
|
||||
int32_t aux32[8] __attribute__((aligned(16))) = {0};
|
||||
float sums[8] __attribute__((aligned(16))) = {0};
|
||||
|
||||
@@ -861,8 +861,8 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
__m128 acc_3 = _mm_setzero_ps();
|
||||
|
||||
for (; ib + 1 < nb; ib += 2) {
|
||||
_mm_prefetch(&x[ib] + sizeof(block_q4_0), _MM_HINT_T0);
|
||||
_mm_prefetch(&y[ib] + sizeof(block_q8_0), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&x[ib] + sizeof(block_q4_0)), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&y[ib] + sizeof(block_q8_0)), _MM_HINT_T0);
|
||||
|
||||
// Compute combined scale for the block 0 and 1
|
||||
const __m128 d_0_1 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d) );
|
||||
@@ -879,8 +879,8 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
bx_1 = _mm_sub_epi8(bx_1, off);
|
||||
const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);
|
||||
|
||||
_mm_prefetch(&x[ib] + 2 * sizeof(block_q4_0), _MM_HINT_T0);
|
||||
_mm_prefetch(&y[ib] + 2 * sizeof(block_q8_0), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&x[ib] + 2 * sizeof(block_q4_0)), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&y[ib] + 2 * sizeof(block_q8_0)), _MM_HINT_T0);
|
||||
|
||||
// Compute combined scale for the block 2 and 3
|
||||
const __m128 d_2_3 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) );
|
||||
|
||||
@@ -2034,6 +2034,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
|
||||
{
|
||||
ggml_compute_forward_fill(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_SLEEP:
|
||||
{
|
||||
ggml_compute_forward_sleep(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_FLASH_ATTN_EXT:
|
||||
{
|
||||
ggml_compute_forward_flash_attn_ext(params, tensor);
|
||||
@@ -2306,6 +2310,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
|
||||
case GGML_OP_REPEAT:
|
||||
case GGML_OP_REPEAT_BACK:
|
||||
case GGML_OP_LEAKY_RELU:
|
||||
case GGML_OP_SLEEP:
|
||||
{
|
||||
n_tasks = 1;
|
||||
} break;
|
||||
|
||||
@@ -2238,6 +2238,19 @@ void ggml_compute_forward_fill(const ggml_compute_params * params, ggml_tensor *
|
||||
}
|
||||
}
|
||||
|
||||
// ggml_compute_forward_sleep
|
||||
|
||||
void ggml_compute_forward_sleep(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
if (params->ith != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// same clock as ggml_time_us() callers; on macOS steady_clock is CLOCK_MONOTONIC_RAW, which can run faster
|
||||
const int64_t t_end = ggml_time_us() + ggml_get_op_params_i32(dst, 0);
|
||||
|
||||
while (ggml_time_us() < t_end) {}
|
||||
}
|
||||
|
||||
// ggml_compute_tri
|
||||
|
||||
template <typename T>
|
||||
|
||||
@@ -84,6 +84,7 @@ void ggml_compute_forward_top_k(const struct ggml_compute_params * params, struc
|
||||
void ggml_compute_forward_leaky_relu(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_fill(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_sleep(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_flash_attn_ext(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_flash_attn_back(
|
||||
const struct ggml_compute_params * params,
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
#include "batch-get.cuh"
|
||||
#include "batch-copy.cuh"
|
||||
#include "common.cuh"
|
||||
#include "ggml-cpp.h"
|
||||
|
||||
#include <climits>
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
struct ggml_cuda_batch_get_descriptor {
|
||||
struct ggml_cuda_batch_copy_descriptor {
|
||||
const unsigned char * src;
|
||||
unsigned char * dst;
|
||||
size_t size;
|
||||
};
|
||||
|
||||
static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_descriptor * copies, size_t count) {
|
||||
static __global__ void ggml_cuda_batch_copy_kernel(const ggml_cuda_batch_copy_descriptor * copies, size_t count) {
|
||||
// Blocks along y cooperate on each descriptor.
|
||||
const size_t index = blockIdx.x;
|
||||
if (index >= count) {
|
||||
@@ -50,11 +50,11 @@ static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_desc
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_cuda_batch_get_mapping {
|
||||
struct ggml_cuda_batch_copy_mapping {
|
||||
void * host = nullptr;
|
||||
unsigned char * device = nullptr;
|
||||
|
||||
ggml_cuda_batch_get_mapping(void * host, size_t size) {
|
||||
ggml_cuda_batch_copy_mapping(void * host, size_t size) {
|
||||
const cudaError_t err = cudaHostRegister(host, size, cudaHostRegisterMapped | cudaHostRegisterPortable);
|
||||
if (err != cudaSuccess) {
|
||||
(void) cudaGetLastError();
|
||||
@@ -70,10 +70,10 @@ struct ggml_cuda_batch_get_mapping {
|
||||
}
|
||||
}
|
||||
|
||||
ggml_cuda_batch_get_mapping(const ggml_cuda_batch_get_mapping &) = delete;
|
||||
ggml_cuda_batch_get_mapping & operator=(const ggml_cuda_batch_get_mapping &) = delete;
|
||||
ggml_cuda_batch_copy_mapping(const ggml_cuda_batch_copy_mapping &) = delete;
|
||||
ggml_cuda_batch_copy_mapping & operator=(const ggml_cuda_batch_copy_mapping &) = delete;
|
||||
|
||||
~ggml_cuda_batch_get_mapping() {
|
||||
~ggml_cuda_batch_copy_mapping() {
|
||||
if (host) {
|
||||
// Release only the registration acquired by this object.
|
||||
CUDA_CHECK(cudaHostUnregister(host));
|
||||
@@ -82,11 +82,12 @@ struct ggml_cuda_batch_get_mapping {
|
||||
};
|
||||
#endif
|
||||
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
static bool ggml_cuda_copy_tensor_n(int device, cudaMemcpyKind direction,
|
||||
const ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
GGML_UNUSED(device);
|
||||
GGML_UNUSED(direction);
|
||||
GGML_UNUSED(tensors);
|
||||
GGML_UNUSED(data);
|
||||
GGML_UNUSED(offsets);
|
||||
@@ -94,14 +95,17 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
GGML_UNUSED(n_tensors);
|
||||
return false;
|
||||
#else
|
||||
const bool to_host = direction == cudaMemcpyDeviceToHost;
|
||||
// Provisional dispatch values based on experiments.
|
||||
constexpr size_t min_batch_reads = 128;
|
||||
constexpr size_t max_gather_read_bytes = 128 * 1024;
|
||||
// Collect non-empty CPU destination ranges.
|
||||
constexpr size_t max_kernel_read_bytes = 128 * 1024;
|
||||
// Collect destination ranges; repeated sources are allowed.
|
||||
std::vector<std::pair<uintptr_t, uintptr_t>> ranges;
|
||||
ranges.reserve(n_tensors);
|
||||
size_t payload_bytes = 0;
|
||||
// Number of non-empty reads whose size is at most max_gather_read_bytes.
|
||||
uintptr_t begin = UINTPTR_MAX;
|
||||
uintptr_t end = 0;
|
||||
// Number of non-empty reads whose size is at most max_kernel_read_bytes.
|
||||
size_t n_reads_small = 0;
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
if (sizes[i] == 0) {
|
||||
@@ -111,15 +115,19 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
GGML_ASSERT(address <= UINTPTR_MAX - sizes[i]);
|
||||
GGML_ASSERT(payload_bytes <= SIZE_MAX - sizes[i]);
|
||||
payload_bytes += sizes[i];
|
||||
n_reads_small += sizes[i] <= max_gather_read_bytes;
|
||||
ranges.emplace_back(address, address + sizes[i]);
|
||||
n_reads_small += sizes[i] <= max_kernel_read_bytes;
|
||||
begin = std::min(begin, address);
|
||||
end = std::max(end, address + sizes[i]);
|
||||
const uintptr_t destination = to_host ? address : (uintptr_t) tensors[i]->data + offsets[i];
|
||||
GGML_ASSERT(destination <= UINTPTR_MAX - sizes[i]);
|
||||
ranges.emplace_back(destination, destination + sizes[i]);
|
||||
}
|
||||
if (ranges.empty()) {
|
||||
return true;
|
||||
}
|
||||
// Total number of non-empty reads.
|
||||
const size_t n_reads = ranges.size();
|
||||
if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_get_descriptor) || n_reads > INT_MAX) {
|
||||
if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_copy_descriptor) || n_reads > INT_MAX) {
|
||||
return false;
|
||||
}
|
||||
if (n_reads < 2 || (n_reads < min_batch_reads && n_reads_small == n_reads)) {
|
||||
@@ -127,72 +135,85 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
}
|
||||
std::sort(ranges.begin(), ranges.end());
|
||||
for (size_t i = 1; i < ranges.size(); ++i) {
|
||||
// Preserve the ordered copy path when CPU destinations overlap.
|
||||
// Preserve the ordered copy path when destinations overlap.
|
||||
if (ranges[i].first < ranges[i - 1].second) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Find the CPU span to register, including gaps between destinations.
|
||||
const uintptr_t begin = ranges.front().first;
|
||||
const uintptr_t end = ranges.back().second;
|
||||
if (end - begin - payload_bytes > payload_bytes) {
|
||||
// Limit gaps in the CPU span without underflow when sources overlap.
|
||||
if (end - begin > payload_bytes && end - begin - payload_bytes > payload_bytes) {
|
||||
return false;
|
||||
}
|
||||
// Register and map the existing CPU destination so the kernel can write to it directly.
|
||||
ggml_cuda_batch_get_mapping mapping((void *) begin, end - begin);
|
||||
// Register and map the CPU span for direct kernel access.
|
||||
ggml_cuda_batch_copy_mapping mapping((void *) begin, end - begin);
|
||||
if (!mapping.host) {
|
||||
return false;
|
||||
}
|
||||
// Reads selected for gather; zero means all reads use DMA.
|
||||
size_t n_reads_gather = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0;
|
||||
// Reads selected for the kernel; zero means all reads use DMA.
|
||||
size_t n_reads_kernel = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0;
|
||||
// Keep descriptor storage alive until the final stream synchronization.
|
||||
constexpr size_t max_batch_descriptors = 128 * 1024;
|
||||
const size_t bytes = std::min(n_reads_gather, max_batch_descriptors) * sizeof(ggml_cuda_batch_get_descriptor);
|
||||
ggml_backend_buffer_ptr descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr);
|
||||
const size_t bytes = std::min(n_reads_kernel, max_batch_descriptors) * sizeof(ggml_cuda_batch_copy_descriptor);
|
||||
ggml_backend_buffer_ptr descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr);
|
||||
if (!descriptors) {
|
||||
n_reads_gather = 0;
|
||||
n_reads_kernel = 0;
|
||||
}
|
||||
const auto host_type = ggml_backend_cuda_host_buffer_type();
|
||||
ggml_backend_buffer_ptr host_descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr);
|
||||
ggml_backend_buffer_ptr host_descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr);
|
||||
if (!host_descriptors || ggml_backend_buffer_get_type(host_descriptors.get()) != host_type) {
|
||||
n_reads_gather = 0;
|
||||
n_reads_kernel = 0;
|
||||
}
|
||||
if (n_reads_gather) {
|
||||
auto * copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get()));
|
||||
auto * device_copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(descriptors.get()));
|
||||
if (n_reads_kernel) {
|
||||
auto * copies = static_cast<ggml_cuda_batch_copy_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get()));
|
||||
auto * device_copies = static_cast<ggml_cuda_batch_copy_descriptor *>(ggml_backend_buffer_get_base(descriptors.get()));
|
||||
const int threads = 256;
|
||||
const size_t nsm = ggml_cuda_info().devices[device].nsm;
|
||||
size_t i = 0;
|
||||
for (size_t first = 0; first < n_reads_gather; first += max_batch_descriptors) {
|
||||
const size_t n_batch = std::min(n_reads_gather - first, max_batch_descriptors);
|
||||
for (size_t first = 0; first < n_reads_kernel; first += max_batch_descriptors) {
|
||||
const size_t n_batch = std::min(n_reads_kernel - first, max_batch_descriptors);
|
||||
size_t count = 0;
|
||||
for (; i < n_tensors && count < n_batch; ++i) {
|
||||
if (sizes[i] == 0 || sizes[i] > max_gather_read_bytes) {
|
||||
if (sizes[i] == 0 || sizes[i] > max_kernel_read_bytes) {
|
||||
continue;
|
||||
}
|
||||
copies[count++] = {(const unsigned char *) tensors[i]->data + offsets[i],
|
||||
mapping.device + ((uintptr_t) data[i] - begin), sizes[i]};
|
||||
auto * host = mapping.device + ((uintptr_t) data[i] - begin);
|
||||
auto * tensor = (unsigned char *) tensors[i]->data + offsets[i];
|
||||
copies[count++] = {to_host ? tensor : host, to_host ? host : tensor, sizes[i]};
|
||||
}
|
||||
const size_t blocks_per_copy = std::min<size_t>(UINT16_MAX, std::max<size_t>(1, (nsm + n_batch - 1) / n_batch));
|
||||
const dim3 blocks((unsigned int) n_batch, (unsigned int) blocks_per_copy);
|
||||
// Stream ordering keeps each kernel ahead of the next descriptor overwrite.
|
||||
CUDA_CHECK(cudaMemcpyAsync(device_copies, copies, n_batch * sizeof(copies[0]), cudaMemcpyHostToDevice, cudaStreamPerThread));
|
||||
ggml_cuda_batch_get_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch);
|
||||
ggml_cuda_batch_copy_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch);
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
if (first + n_batch < n_reads_gather) {
|
||||
if (first + n_batch < n_reads_kernel) {
|
||||
// Wait before refilling the pinned host descriptor buffer.
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Keep the destination pinned for DMA too; overlapping destinations already declined above.
|
||||
// Keep the CPU span pinned for DMA too; overlapping destinations already declined above.
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
if (sizes[i] && (!n_reads_gather || sizes[i] > max_gather_read_bytes)) {
|
||||
CUDA_CHECK(cudaMemcpyAsync(data[i], (const char *) tensors[i]->data + offsets[i], sizes[i], cudaMemcpyDeviceToHost, cudaStreamPerThread));
|
||||
if (sizes[i] && (!n_reads_kernel || sizes[i] > max_kernel_read_bytes)) {
|
||||
void * host = const_cast<void *>(data[i]);
|
||||
void * tensor = (char *) tensors[i]->data + offsets[i];
|
||||
CUDA_CHECK(cudaMemcpyAsync(to_host ? host : tensor, to_host ? tensor : host, sizes[i], direction, cudaStreamPerThread));
|
||||
}
|
||||
}
|
||||
// Wait for GPU writes before releasing the CPU mapping and descriptor buffers.
|
||||
// Wait for transfers before releasing the CPU mapping and descriptor buffers.
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
return ggml_cuda_copy_tensor_n(device, cudaMemcpyDeviceToHost, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
|
||||
bool ggml_cuda_set_tensor_n(int device,
|
||||
ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
return ggml_cuda_copy_tensor_n(device, cudaMemcpyHostToDevice, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
@@ -5,3 +5,7 @@
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors);
|
||||
|
||||
bool ggml_cuda_set_tensor_n(int device,
|
||||
ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors);
|
||||
@@ -10,7 +10,7 @@
|
||||
#include "ggml-cuda/arange.cuh"
|
||||
#include "ggml-cuda/argmax.cuh"
|
||||
#include "ggml-cuda/argsort.cuh"
|
||||
#include "ggml-cuda/batch-get.cuh"
|
||||
#include "ggml-cuda/batch-copy.cuh"
|
||||
#include "ggml-cuda/binbcast.cuh"
|
||||
#include "ggml-cuda/clamp.cuh"
|
||||
#include "ggml-cuda/col2im-1d.cuh"
|
||||
@@ -48,6 +48,7 @@
|
||||
#include "ggml-cuda/rope.cuh"
|
||||
#include "ggml-cuda/roll.cuh"
|
||||
#include "ggml-cuda/scale.cuh"
|
||||
#include "ggml-cuda/sleep.cuh"
|
||||
#include "ggml-cuda/snake.cuh"
|
||||
#include "ggml-cuda/softcap.cuh"
|
||||
#include "ggml-cuda/softmax.cuh"
|
||||
@@ -1116,6 +1117,11 @@ static void ggml_backend_cuda_buffer_set_tensor_n(ggml_backend_buffer_t buffer,
|
||||
|
||||
ggml_cuda_set_device(ctx->device);
|
||||
|
||||
if (!ggml_cuda_pinned_memory_disabled() &&
|
||||
ggml_cuda_set_tensor_n(ctx->device, tensors, data, offsets, sizes, n_tensors)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
@@ -3145,6 +3151,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
|
||||
case GGML_OP_FILL:
|
||||
ggml_cuda_op_fill(ctx, dst);
|
||||
break;
|
||||
case GGML_OP_SLEEP:
|
||||
ggml_cuda_op_sleep(ctx, dst);
|
||||
break;
|
||||
case GGML_OP_LIGHTNING_INDEXER:
|
||||
ggml_cuda_lightning_indexer(ctx, dst);
|
||||
break;
|
||||
@@ -6809,6 +6818,12 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
return ggml_cuda_lightning_indexer_supported(dev_ctx->device, op);
|
||||
case GGML_OP_TOP_K_MASK:
|
||||
return op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32;
|
||||
case GGML_OP_SLEEP:
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
return false;
|
||||
#else
|
||||
return true;
|
||||
#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -1094,7 +1094,7 @@ static __global__ void mul_mat_q(
|
||||
int col_high = min(col_low + J, ncols_dst);
|
||||
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
||||
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
||||
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
||||
int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
|
||||
|
||||
if (ids_dst) {
|
||||
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
|
||||
@@ -1176,7 +1176,7 @@ static __global__ void mul_mat_q(
|
||||
int col_high = min(col_low + J, ncols_dst);
|
||||
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
||||
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
||||
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
||||
int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
|
||||
|
||||
if (ids_dst) {
|
||||
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
|
||||
@@ -1257,7 +1257,7 @@ static __global__ void mul_mat_q(
|
||||
int col_high = min(col_low + J, ncols_dst);
|
||||
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
||||
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
||||
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
||||
int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
|
||||
|
||||
if (ids_dst) {
|
||||
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
|
||||
@@ -1403,7 +1403,7 @@ static __global__ void mul_mat_q_stream_k_fixup(
|
||||
const int it = tmp2.x;
|
||||
|
||||
if (!ids_dst) {
|
||||
const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst + it*I;
|
||||
const int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst + it*I;
|
||||
dst += offset_dst;
|
||||
|
||||
const int i_max = nrows_x - it*I - 1;
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#include "sleep.cuh"
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
// %globaltimer is a nanosecond wall clock, unlike clock64() it is unaffected by the SM clock and by frequency scaling
|
||||
static __device__ __forceinline__ uint64_t globaltimer_ns() {
|
||||
uint64_t t;
|
||||
asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
|
||||
return t;
|
||||
}
|
||||
|
||||
// a single thread is enough, the next kernel on the same stream cannot start before this one retires
|
||||
static __global__ void sleep_ns(const uint64_t ns) {
|
||||
const uint64_t t0 = globaltimer_ns();
|
||||
|
||||
while (globaltimer_ns() - t0 < ns) {}
|
||||
}
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
sleep_ns<<<1, 1, 0, ctx.stream()>>>(1000*(uint64_t) ggml_get_op_params_i32(dst, 0));
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
GGML_UNUSED(ctx);
|
||||
GGML_UNUSED(dst);
|
||||
GGML_ABORT("GGML_OP_SLEEP requires the %%globaltimer register, which is only available on CUDA");
|
||||
}
|
||||
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
@@ -0,0 +1,3 @@
|
||||
#include "common.cuh"
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
@@ -2551,6 +2551,26 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_
|
||||
char base[256];
|
||||
char name[256];
|
||||
|
||||
const bool has_tensor = ggml_metal_device_get_props(ggml_metal_library_get_device(lib))->has_tensor;
|
||||
|
||||
if (has_tensor) {
|
||||
// nr0 output positions per threadgroup, nr1 output channels, same tiles as kernel_mul_mm
|
||||
snprintf(base, 256, "kernel_conv_2d_mm_tensor_%s_%s", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type));
|
||||
snprintf(name, 256, "%s", base);
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
|
||||
}
|
||||
|
||||
res.nr0 = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y;
|
||||
res.nr1 = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X;
|
||||
res.nsg = N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y;
|
||||
res.smem = res.nr0 * N_MM_NK_TOTAL * sizeof(ggml_fp16_t);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
const int nr0 = op->src[0]->ne[3] <= 32 ? 32 : 64;
|
||||
const int nr1 = 2048/nr0;
|
||||
|
||||
@@ -2564,13 +2584,27 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_
|
||||
|
||||
res.nr0 = nr0;
|
||||
res.nr1 = nr1;
|
||||
res.nsg = 4;
|
||||
res.nsg = N_SG_CONV_2D;
|
||||
|
||||
res.smem = nr0*nr1*sizeof(float);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm_reduce(ggml_metal_library_t lib, const ggml_tensor * op) {
|
||||
assert(op->op == GGML_OP_CONV_2D);
|
||||
GGML_UNUSED(op);
|
||||
|
||||
const char * name = "kernel_conv_2d_mm_reduce";
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw(ggml_metal_library_t lib, const ggml_tensor * op, bool tiled) {
|
||||
assert(op->op == GGML_OP_CONV_2D_DW);
|
||||
|
||||
|
||||
@@ -170,6 +170,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake (ggml_metal_library_t lib, enum ggml_type type);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm_reduce (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tiled);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
|
||||
@@ -2375,7 +2375,7 @@ void ggml_metal_buffer_memset_tensor(ggml_metal_buffer_t buf, struct ggml_tensor
|
||||
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
||||
|
||||
[encoder fillBuffer:bid_dst.metal
|
||||
range:NSMakeRange(bid_dst.offs, bid_dst.offs + size)
|
||||
range:NSMakeRange(bid_dst.offs, size)
|
||||
value:value];
|
||||
|
||||
[encoder endEncoding];
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
#define N_ARGMAX_NPART_MAX 256
|
||||
#define N_ARGMAX_CHUNK 4096
|
||||
|
||||
#define N_SG_CONV_2D 4
|
||||
|
||||
// kernel parameters for mat-vec threadgroups
|
||||
//
|
||||
// N_R0: number of src0 rows to process per simdgroup
|
||||
@@ -752,8 +754,14 @@ typedef struct {
|
||||
int32_t p1;
|
||||
int32_t d0;
|
||||
int32_t d1;
|
||||
int32_t nk; // reduction length per split
|
||||
} ggml_metal_kargs_conv_2d;
|
||||
|
||||
typedef struct {
|
||||
int32_t ne; // output elements
|
||||
int32_t nsplit;
|
||||
} ggml_metal_kargs_conv_2d_reduce;
|
||||
|
||||
typedef struct {
|
||||
uint64_t nb00; // kernel strides
|
||||
uint64_t nb01;
|
||||
|
||||
@@ -4959,6 +4959,33 @@ int ggml_metal_op_im2col(ggml_metal_op_t ctx, int idx) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// number of reduction splits of the conv_2d matmul kernels
|
||||
// A small output with a long reduction leaves the GPU idle, so the reduction is split until there are about 1M partial sums.
|
||||
static int ggml_metal_op_conv_2d_nsplit(const ggml_tensor * op, int32_t * nk) {
|
||||
const int64_t K = op->src[0]->ne[0]*op->src[0]->ne[1]*op->src[0]->ne[2];
|
||||
const int64_t nsteps = (K + 31)/32;
|
||||
const int64_t ne = ggml_nelements(op);
|
||||
|
||||
// at least 3 reduction steps of 32 per split, at most 32 splits
|
||||
const int64_t nsplit = std::max<int64_t>(1, std::min<int64_t>({ (1024*1024 + ne - 1)/ne, nsteps/3, 32 }));
|
||||
const int64_t steps = (nsteps + nsplit - 1)/nsplit;
|
||||
|
||||
if (nk) {
|
||||
*nk = (int32_t) (steps*32);
|
||||
}
|
||||
|
||||
return (int) ((nsteps + steps - 1)/steps);
|
||||
}
|
||||
|
||||
size_t ggml_metal_op_conv_2d_extra_tmp(const ggml_tensor * op) {
|
||||
assert(op->op == GGML_OP_CONV_2D);
|
||||
|
||||
const int nsplit = ggml_metal_op_conv_2d_nsplit(op, nullptr);
|
||||
|
||||
// partial sums of the reduction splits
|
||||
return nsplit > 1 ? nsplit*ggml_nelements(op)*sizeof(float) : 0;
|
||||
}
|
||||
|
||||
int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_tensor * op = ctx->node(idx);
|
||||
|
||||
@@ -5012,11 +5039,17 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
/*.p1 =*/ p1,
|
||||
/*.d0 =*/ d0,
|
||||
/*.d1 =*/ d1,
|
||||
/*.nk =*/ 0,
|
||||
};
|
||||
|
||||
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev);
|
||||
|
||||
if (props_dev->has_simdgroup_mm) {
|
||||
// the matmul kernels walk the patches with 32-bit byte offsets inside an image
|
||||
// larger images, possibly a strided view, take the generic kernel
|
||||
const uint64_t max_off = (ne12 - 1)*nb12 + (ne11 - 1)*nb11 + (ne10 - 1)*nb10 + sizeof(float);
|
||||
const bool fits_32 = max_off <= UINT32_MAX;
|
||||
|
||||
if (props_dev->has_simdgroup_mm && fits_32) {
|
||||
auto pipeline = ggml_metal_library_get_pipeline_conv_2d_mm(lib, op);
|
||||
|
||||
const int nr0 = pipeline.nr0;
|
||||
@@ -5026,15 +5059,48 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
const int64_t NP = ne3*ne1*ne0;
|
||||
const int64_t OC = ne03;
|
||||
|
||||
const int nsplit = ggml_metal_op_conv_2d_nsplit(op, &args.nk);
|
||||
|
||||
ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);
|
||||
ggml_metal_buffer_id bid_tmp = bid_dst;
|
||||
bid_tmp.offs += ggml_nbytes(op);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline);
|
||||
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
|
||||
ggml_metal_encoder_set_buffer (enc, nsplit > 1 ? bid_tmp : bid_dst, 3);
|
||||
|
||||
ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, 1, 32, nsg, 1);
|
||||
if (props_dev->has_tensor) {
|
||||
// per image: nr0 positions times nr1 channels per threadgroup
|
||||
const int64_t OHW = ne1*ne0;
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (OC + nr1 - 1)/nr1, (OHW + nr0 - 1)/nr0, ne3*nsplit, 32, nsg, 1);
|
||||
} else {
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, nsplit, 32, nsg, 1);
|
||||
}
|
||||
|
||||
if (nsplit > 1) {
|
||||
GGML_ASSERT(ggml_is_contiguous(op));
|
||||
|
||||
// sync the 2 kernels
|
||||
ggml_metal_op_concurrency_reset(ctx);
|
||||
|
||||
ggml_metal_kargs_conv_2d_reduce args0 = {
|
||||
/*.ne =*/ (int32_t) ggml_nelements(op),
|
||||
/*.nsplit =*/ nsplit,
|
||||
};
|
||||
|
||||
auto pipeline0 = ggml_metal_library_get_pipeline_conv_2d_mm_reduce(lib, op);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline0);
|
||||
ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_tmp, 1);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_dst, 2);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (args0.ne + 255)/256, 1, 1, 256, 1, 1);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -49,6 +49,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_rows(const struct ggml_tensor * op);
|
||||
|
||||
size_t ggml_metal_op_argmax_extra(const struct ggml_tensor * op);
|
||||
|
||||
size_t ggml_metal_op_conv_2d_extra_tmp(const struct ggml_tensor * op);
|
||||
|
||||
int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_acc (ggml_metal_op_t ctx, int idx);
|
||||
|
||||
@@ -311,6 +311,10 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
|
||||
{
|
||||
res = 2*sizeof(int32_t)*ggml_nelements(tensor->src[0]);
|
||||
} break;
|
||||
case GGML_OP_CONV_2D:
|
||||
{
|
||||
res += ggml_metal_op_conv_2d_extra_tmp(tensor);
|
||||
} break;
|
||||
case GGML_OP_ARGMAX:
|
||||
{
|
||||
res += ggml_metal_op_argmax_extra(tensor);
|
||||
|
||||
@@ -285,7 +285,7 @@ kernel void kernel_conv_2d_mm(
|
||||
ushort tiitg[[thread_index_in_threadgroup]],
|
||||
ushort tiisg[[thread_index_in_simdgroup]],
|
||||
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
|
||||
constexpr short NSG = 4;
|
||||
constexpr short NSG = N_SG_CONV_2D;
|
||||
constexpr short NTH = 32*NSG;
|
||||
constexpr short NK = 32;
|
||||
constexpr short NKB = NK/8;
|
||||
@@ -300,6 +300,10 @@ kernel void kernel_conv_2d_mm(
|
||||
const int K = args.IC*args.KH*args.KW;
|
||||
const int NP = args.N*args.OH*args.OW;
|
||||
|
||||
// this threadgroup reduces [k_begin, k_end), the splits are summed by kernel_conv_2d_mm_reduce
|
||||
const int k_begin = tgpig.z*args.nk;
|
||||
const int k_end = min(K, k_begin + args.nk);
|
||||
|
||||
const int r0 = tgpig.y*NR0;
|
||||
const int r1 = tgpig.x*NR1;
|
||||
|
||||
@@ -308,8 +312,14 @@ kernel void kernel_conv_2d_mm(
|
||||
const short lb_n = 8*sgitg + tiisg%8;
|
||||
const short lb_k = 8*(tiisg/8);
|
||||
|
||||
// the patches are walked with 32-bit byte offsets inside the image, the host checks that an image fits
|
||||
const uint nb10 = args.nb10;
|
||||
const uint nb11 = args.nb11;
|
||||
const uint nb12 = args.nb12;
|
||||
|
||||
int ix0[NPL];
|
||||
int iy0[NPL];
|
||||
uint off0[NPL];
|
||||
bool ip_ok[NPL];
|
||||
|
||||
device const char * srcb[NPL];
|
||||
@@ -324,6 +334,8 @@ kernel void kernel_conv_2d_mm(
|
||||
ix0[t] = iow*args.s0 - args.p0;
|
||||
iy0[t] = ioh*args.s1 - args.p1;
|
||||
|
||||
off0[t] = iy0[t]*nb11 + ix0[t]*nb10;
|
||||
|
||||
ip_ok[t] = ip < NP;
|
||||
|
||||
srcb[t] = src + in*args.nb13;
|
||||
@@ -337,7 +349,7 @@ kernel void kernel_conv_2d_mm(
|
||||
mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
|
||||
}
|
||||
|
||||
for (int k0 = 0; k0 < K; k0 += NK) {
|
||||
for (int k0 = k_begin; k0 < k_end; k0 += NK) {
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
{
|
||||
@@ -349,7 +361,7 @@ kernel void kernel_conv_2d_mm(
|
||||
const int oc = r0 + row;
|
||||
|
||||
half v = 0;
|
||||
if (kk < K && oc < args.OC) {
|
||||
if (kk < k_end && oc < args.OC) {
|
||||
v = *((device const TK *)(weights + oc*args.nb03) + kk);
|
||||
}
|
||||
|
||||
@@ -364,7 +376,11 @@ kernel void kernel_conv_2d_mm(
|
||||
|
||||
int kw = kk % args.KW;
|
||||
int kh = (kk / args.KW) % args.KH;
|
||||
int ic = kk / (args.KW*args.KH);
|
||||
int dx = kw*args.d0;
|
||||
int dy = kh*args.d1;
|
||||
|
||||
// byte offset of the reduction index kk, relative to the patch origin
|
||||
uint offk = (kk / (args.KW*args.KH))*nb12 + dy*nb11 + dx*nb10;
|
||||
|
||||
FOR_UNROLL (short i = 0; i < 8; i++) {
|
||||
const short k = lb_k + i;
|
||||
@@ -372,12 +388,12 @@ kernel void kernel_conv_2d_mm(
|
||||
FOR_UNROLL (short t = 0; t < NPL; t++) {
|
||||
const short n = 32*t + lb_n;
|
||||
|
||||
const int ix = ix0[t] + kw*args.d0;
|
||||
const int iy = iy0[t] + kh*args.d1;
|
||||
const int ix = ix0[t] + dx;
|
||||
const int iy = iy0[t] + dy;
|
||||
|
||||
half v = 0;
|
||||
if (ip_ok[t] && kk < K && ix >= 0 && ix < args.IW && iy >= 0 && iy < args.IH) {
|
||||
v = *(device const float *)(srcb[t] + ic*args.nb12 + iy*args.nb11 + ix*args.nb10);
|
||||
if (ip_ok[t] && kk < k_end && (uint) ix < (uint) args.IW && (uint) iy < (uint) args.IH) {
|
||||
v = *(device const float *)(srcb[t] + (off0[t] + offk));
|
||||
}
|
||||
|
||||
const short ib = NKB*(n/8) + k/8;
|
||||
@@ -386,11 +402,17 @@ kernel void kernel_conv_2d_mm(
|
||||
}
|
||||
|
||||
kk++;
|
||||
dx += args.d0;
|
||||
offk += args.d0*nb10;
|
||||
if (++kw == args.KW) {
|
||||
kw = 0;
|
||||
kw = 0;
|
||||
dx = 0;
|
||||
dy += args.d1;
|
||||
offk += args.d1*nb11 - args.KW*args.d0*nb10;
|
||||
if (++kh == args.KH) {
|
||||
kh = 0;
|
||||
ic++;
|
||||
kh = 0;
|
||||
dy = 0;
|
||||
offk += nb12 - args.KH*args.d1*nb11;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -398,7 +420,7 @@ kernel void kernel_conv_2d_mm(
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
const short nkb = min((K - k0 + 7)/8, (int) NKB);
|
||||
const short nkb = min((k_end - k0 + 7)/8, (int) NKB);
|
||||
|
||||
for (short ik = 0; ik < nkb; ik++) {
|
||||
FOR_UNROLL (short i = 0; i < 4; i++) {
|
||||
@@ -436,7 +458,8 @@ kernel void kernel_conv_2d_mm(
|
||||
const int oh = (p / args.OW) % args.OH;
|
||||
const int n = p / (args.OW*args.OH);
|
||||
|
||||
device char * dstp = dst + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
|
||||
// with several splits, dst is the partial sum buffer, one contiguous output per split
|
||||
device char * dstp = dst + (uint64_t) tgpig.z*args.N*args.nb3 + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
|
||||
|
||||
for (short j = 0; j < NR0*NR1/NTH; j++) {
|
||||
const short row = tiitg/NR1 + (NTH/NR1)*j;
|
||||
@@ -448,6 +471,158 @@ kernel void kernel_conv_2d_mm(
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef GGML_METAL_HAS_TENSOR
|
||||
// conv_2d as a matmul on the tensor cores, like kernel_mul_mm
|
||||
// The patches of NRA output positions are gathered into threadgroup memory, the kernel rows are read from device memory.
|
||||
// One threadgroup computes NRA positions of one image times NRB output channels, over the reduction range of its split.
|
||||
template <typename TK>
|
||||
kernel void kernel_conv_2d_mm_tensor(
|
||||
constant ggml_metal_kargs_conv_2d & args,
|
||||
device const char * weights,
|
||||
device const char * src,
|
||||
device char * dst,
|
||||
threadgroup char * shmem [[threadgroup(0)]],
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
uint3 tgpg[[threadgroups_per_grid]],
|
||||
ushort tiitg[[thread_index_in_threadgroup]]) {
|
||||
constexpr int NRB = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X; // output channels
|
||||
constexpr int NRA = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y; // output positions
|
||||
constexpr int NK = N_MM_NK_TOTAL;
|
||||
constexpr int NTH = N_SIMDWIDTH * N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y;
|
||||
|
||||
// each thread gathers NK/(NTH/NRA) consecutive reduction indices of one position
|
||||
constexpr int KPT = NK/(NTH/NRA);
|
||||
|
||||
const int K = args.IC*args.KH*args.KW;
|
||||
const int OHW = args.OH*args.OW;
|
||||
|
||||
const int nsplit = tgpg.z/args.N;
|
||||
const int img = tgpig.z/nsplit;
|
||||
const int split = tgpig.z%nsplit;
|
||||
|
||||
const int k_begin = split*args.nk;
|
||||
const int k_end = min(K, k_begin + args.nk);
|
||||
|
||||
const int ra = tgpig.y*NRA;
|
||||
const int rb = tgpig.x*NRB;
|
||||
|
||||
threadgroup half * sa = (threadgroup half *) shmem;
|
||||
|
||||
// the position this thread gathers
|
||||
const int row = tiitg%NRA;
|
||||
const int kc = (tiitg/NRA)*KPT;
|
||||
const int p = ra + row;
|
||||
const bool pok = p < OHW;
|
||||
const int ix0 = (p%args.OW)*args.s0 - args.p0;
|
||||
const int iy0 = (p/args.OW)*args.s1 - args.p1;
|
||||
|
||||
device const char * srcb = src + img*args.nb13;
|
||||
|
||||
// the patch is walked with 32-bit byte offsets inside the image, the host checks that an image fits
|
||||
const uint nb10 = args.nb10;
|
||||
const uint nb11 = args.nb11;
|
||||
const uint nb12 = args.nb12;
|
||||
|
||||
const uint off0 = iy0*nb11 + ix0*nb10;
|
||||
|
||||
device TK * ptrB = (device TK *) weights;
|
||||
const int strideB = args.nb03/sizeof(TK);
|
||||
|
||||
mpp::tensor_ops::matmul2d<
|
||||
mpp::tensor_ops::matmul2d_descriptor(
|
||||
NRB, NRA, static_cast<int>(dynamic_extent), false, true, true,
|
||||
mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
|
||||
execution_simdgroups<N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y>> mm;
|
||||
|
||||
auto tB0 = tensor(ptrB, dextents<int32_t, 2>(K, args.OC), array<int, 2>({1, strideB}));
|
||||
auto tA0 = tensor(sa, dextents<int32_t, 2>(NK, NRA));
|
||||
auto cT = mm.get_destination_cooperative_tensor<decltype(tB0), decltype(tA0), float>();
|
||||
|
||||
for (int k0 = k_begin; k0 < k_end; k0 += NK) {
|
||||
// the reduction indices past k_end are beyond the extent given to the matmul, they are not gathered
|
||||
if (k0 + kc < k_end) {
|
||||
// gather KPT patch values of this position, walking the reduction index incrementally
|
||||
int kk = k0 + kc;
|
||||
int kw = kk % args.KW;
|
||||
int kh = (kk / args.KW) % args.KH;
|
||||
int dx = kw*args.d0;
|
||||
int dy = kh*args.d1;
|
||||
|
||||
// byte offset of the reduction index kk, relative to the patch origin
|
||||
uint offk = (kk / (args.KW*args.KH))*nb12 + dy*nb11 + dx*nb10;
|
||||
|
||||
FOR_UNROLL (short i = 0; i < KPT; i++) {
|
||||
const int ix = ix0 + dx;
|
||||
const int iy = iy0 + dy;
|
||||
|
||||
half v = 0;
|
||||
if (pok && kk < k_end && (uint) ix < (uint) args.IW && (uint) iy < (uint) args.IH) {
|
||||
v = *(device const float *)(srcb + (off0 + offk));
|
||||
}
|
||||
sa[row*NK + kc + i] = v;
|
||||
|
||||
kk++;
|
||||
dx += args.d0;
|
||||
offk += args.d0*nb10;
|
||||
if (++kw == args.KW) {
|
||||
kw = 0;
|
||||
dx = 0;
|
||||
dy += args.d1;
|
||||
offk += args.d1*nb11 - args.KW*args.d0*nb10;
|
||||
if (++kh == args.KH) {
|
||||
kh = 0;
|
||||
dy = 0;
|
||||
offk += nb12 - args.KH*args.d1*nb11;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
// clamp the reduction extent so neither operand is read past the end of the split
|
||||
const int kExt = min(NK, k_end - k0);
|
||||
|
||||
auto tAv = tensor(sa, dextents<int32_t, 2>(kExt, NRA), array<int, 2>({1, NK}));
|
||||
auto tBv = tensor(ptrB + k0 + rb*strideB, dextents<int32_t, 2>(kExt, args.OC - rb), array<int, 2>({1, strideB}));
|
||||
|
||||
mm.run(tBv, tAv, cT);
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
}
|
||||
|
||||
// dst holds [OHW, OC] per image, with several splits it is the partial sum buffer, one contiguous output per split
|
||||
const int tileM = min(NRA, OHW - ra);
|
||||
const int tileN = min(NRB, args.OC - rb);
|
||||
|
||||
device float * dstTile = (device float *) (dst + (uint64_t) split*args.N*args.nb3 + img*args.nb3) + (uint64_t) rb*OHW + ra;
|
||||
|
||||
auto tD = tensor(dstTile, dextents<int32_t, 2>(tileM, tileN), array<int, 2>({1, OHW}));
|
||||
cT.store(tD);
|
||||
}
|
||||
|
||||
typedef decltype(kernel_conv_2d_mm_tensor<float>) kernel_conv_2d_mm_tensor_t;
|
||||
|
||||
template [[host_name("kernel_conv_2d_mm_tensor_f32_f32")]] kernel kernel_conv_2d_mm_tensor_t kernel_conv_2d_mm_tensor<float>;
|
||||
template [[host_name("kernel_conv_2d_mm_tensor_f16_f32")]] kernel kernel_conv_2d_mm_tensor_t kernel_conv_2d_mm_tensor<half>;
|
||||
#endif
|
||||
|
||||
kernel void kernel_conv_2d_mm_reduce(
|
||||
constant ggml_metal_kargs_conv_2d_reduce & args,
|
||||
device const float * tmp,
|
||||
device float * dst,
|
||||
uint tpig[[thread_position_in_grid]]) {
|
||||
if ((int) tpig >= args.ne) {
|
||||
return;
|
||||
}
|
||||
|
||||
float sum = 0.0f;
|
||||
for (int s = 0; s < args.nsplit; ++s) {
|
||||
sum += tmp[(uint64_t) s*args.ne + tpig];
|
||||
}
|
||||
dst[tpig] = sum;
|
||||
}
|
||||
|
||||
typedef decltype(kernel_conv_2d_mm<float, 64, 32>) kernel_conv_2d_mm_t;
|
||||
|
||||
template [[host_name("kernel_conv_2d_mm_f32_f32_64x32")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<float, 64, 32>;
|
||||
|
||||
@@ -23277,8 +23277,31 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
|
||||
const int64_t kqv_m = pad_up(dv, 64);
|
||||
const int64_t tile_n = pad_up(n_q_chunk, 32);
|
||||
|
||||
// Pad the int8 row pitch by one cache line, as in the P8 KQV kernel.
|
||||
const int kv_pitch = (int) n_kv + 64;
|
||||
static const int64_t kv_pitch_round = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND");
|
||||
const int64_t v = (e && e[0]) ? strtoll(e, nullptr, 10) : 32768;
|
||||
return v > 0 && v <= INT_MAX && (v & (v - 1)) == 0 ? v : (int64_t) 0;
|
||||
}();
|
||||
static const int64_t kv_pitch_round_min = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND_MIN");
|
||||
return (e && e[0]) ? strtoll(e, nullptr, 10) : (int64_t) 16384;
|
||||
}();
|
||||
static const int64_t kv_pitch_round_max_dv = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND_MAX_DV");
|
||||
return (e && e[0]) ? strtoll(e, nullptr, 10) : (int64_t) 256;
|
||||
}();
|
||||
|
||||
// Round long P and V^T rows to reduce cache conflicts, then add one cache line.
|
||||
int64_t kv_pitch_value = n_kv + 64;
|
||||
if (kv_pitch_round > 0 && n_kv > kv_pitch_round_min && dv <= kv_pitch_round_max_dv) {
|
||||
const int64_t rounded_pitch = pad_up(n_kv, kv_pitch_round) + 64;
|
||||
if (rounded_pitch <= INT_MAX &&
|
||||
(size_t) rounded_pitch <= SIZE_MAX / n_q_chunk / n_head &&
|
||||
(size_t) rounded_pitch <= SIZE_MAX / dv / n_head_kv) {
|
||||
kv_pitch_value = rounded_pitch;
|
||||
}
|
||||
}
|
||||
const int kv_pitch = (int) kv_pitch_value;
|
||||
const size_t nblk = (size_t) kv_pitch / 32;
|
||||
|
||||
backend_ctx->prealloc_fa_pq.allocate(backend_ctx->context, (size_t) kv_pitch * n_q_chunk * n_head);
|
||||
|
||||
+113
-6
@@ -3307,6 +3307,85 @@ static int iq2_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, co
|
||||
return grid_index;
|
||||
}
|
||||
|
||||
// same result as iq2_find_best_neighbour: score = s*(s*quad_coef - lin_coef) + const, and the coefs do not depend on the scale s, so cache them per pattern
|
||||
// take the cached winner only if all other neighbours are worse by more than the float rounding bound of the exact search, else run the exact search
|
||||
#define IQ2_NEIGHBOUR_CACHE_MAX 512
|
||||
|
||||
typedef struct {
|
||||
int pattern; // pattern whose coefs are cached, -1 if none
|
||||
int last_best; // last winner for this pattern
|
||||
// sums for the rounding bound: |w|, |w|*|x|, |w|*x^2, and the same with 1 + |w|
|
||||
double sum_w, sum_wx, sum_wxx, sum_w1, sum_w1x, sum_w1xx;
|
||||
double lin_term[8][8], quad_term[8][8]; // 2*w*x*q and w*q*q per element, indexed by the grid level q
|
||||
double lin_coef[IQ2_NEIGHBOUR_CACHE_MAX];
|
||||
double quad_coef[IQ2_NEIGHBOUR_CACHE_MAX];
|
||||
} iq2_neighbour_cache;
|
||||
|
||||
static void iq2_neighbour_cache_init(iq2_neighbour_cache * cache, const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight) {
|
||||
cache->pattern = -1;
|
||||
cache->sum_w = cache->sum_wx = cache->sum_wxx = cache->sum_w1 = cache->sum_w1x = cache->sum_w1xx = 0;
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
const double w = (double)weight[i], x = (double)xval[i], abs_w = fabs(w), abs_x = fabs(x);
|
||||
cache->sum_w += abs_w; cache->sum_wx += abs_w*abs_x; cache->sum_wxx += abs_w*abs_x*abs_x;
|
||||
cache->sum_w1 += 1 + abs_w; cache->sum_w1x += (1 + abs_w)*abs_x; cache->sum_w1xx += (1 + abs_w)*abs_x*abs_x;
|
||||
for (int q = 0; q < 8; ++q) {
|
||||
cache->lin_term[i][q] = 2*w*x*q;
|
||||
cache->quad_term[i][q] = w*q*q;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int iq2_count_at_most(const double * GGML_RESTRICT values, int n, double limit) {
|
||||
int count = 0;
|
||||
for (int j = 0; j < n; ++j) count += values[j] <= limit;
|
||||
return count;
|
||||
}
|
||||
|
||||
static int iq2_find_best_neighbour_cached(iq2_neighbour_cache * cache, int pattern, const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid,
|
||||
const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, int8_t * GGML_RESTRICT L) {
|
||||
const int num_neighbors = neighbours[0];
|
||||
const double scale_d = (double)scale;
|
||||
const double max_level = 7*scale_d;
|
||||
// size of the numbers in the exact search, its rounding error is a multiple of 2^-24*size (+ size_subnormal for underflow)
|
||||
const double size = max_level*max_level*cache->sum_w + 2*max_level*cache->sum_wx + cache->sum_wxx;
|
||||
const double size_subnormal = (1 + max_level)*(1 + max_level)*cache->sum_w1 + 2*(1 + max_level)*cache->sum_w1x + cache->sum_w1xx;
|
||||
// the cached score is (exact score - const)/s, so the 2^-18*size margin on the exact score is divided by s
|
||||
const double tolerance = (0x1p-18*size + 0x1p-100*size_subnormal)/scale_d;
|
||||
if (num_neighbors <= 0 || num_neighbors > IQ2_NEIGHBOUR_CACHE_MAX || !(scale_d > 0 && size < 0x1p100 && size_subnormal < 0x1p100 && tolerance > 0)) {
|
||||
return iq2_find_best_neighbour(neighbours, grid, xval, weight, scale, L);
|
||||
}
|
||||
if (cache->pattern != pattern) {
|
||||
for (int j = 0; j < num_neighbors; ++j) {
|
||||
const int8_t * levels = (const int8_t *)(grid + neighbours[j+1]);
|
||||
double lin = 0, quad = 0;
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
lin += cache->lin_term[i][levels[i]];
|
||||
quad += cache->quad_term[i][levels[i]];
|
||||
}
|
||||
cache->lin_coef[j] = lin; cache->quad_coef[j] = quad;
|
||||
}
|
||||
cache->pattern = pattern;
|
||||
cache->last_best = 0;
|
||||
}
|
||||
double score[IQ2_NEIGHBOUR_CACHE_MAX];
|
||||
for (int j = 0; j < num_neighbors; ++j) score[j] = scale_d*cache->quad_coef[j] - cache->lin_coef[j];
|
||||
// the winner must be the only neighbour within tolerance, try the previous winner for this pattern first
|
||||
int best = cache->last_best;
|
||||
if (iq2_count_at_most(score, num_neighbors, score[best] + tolerance) != 1) {
|
||||
for (int j = 0; j < num_neighbors; ++j) {
|
||||
if (score[j] < score[best]) best = j;
|
||||
}
|
||||
if (iq2_count_at_most(score, num_neighbors, score[best] + tolerance) != 1) {
|
||||
return iq2_find_best_neighbour(neighbours, grid, xval, weight, scale, L);
|
||||
}
|
||||
cache->last_best = best;
|
||||
}
|
||||
const int grid_index = neighbours[best+1];
|
||||
const int8_t * levels = (const int8_t *)(grid + grid_index);
|
||||
for (int i = 0; i < 8; ++i) L[i] = (levels[i] - 1)/2;
|
||||
return grid_index;
|
||||
}
|
||||
|
||||
static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights) {
|
||||
|
||||
const int gindex = iq2_data_index(GGML_TYPE_IQ2_XXS);
|
||||
@@ -3335,6 +3414,8 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
|
||||
float waux[32];
|
||||
uint8_t block_signs[4];
|
||||
uint32_t q2[2*(QK_K/32)];
|
||||
iq2_neighbour_cache neighbour_cache[4];
|
||||
int8_t Laux_prev[32];
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
@@ -3390,6 +3471,7 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
|
||||
continue;
|
||||
}
|
||||
float best = 0;
|
||||
for (int k = 0; k < 4; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
|
||||
for (int is = -6; is <= 6; ++is) {
|
||||
float id = (2*kMaxQ-1+is*0.1f)/eff_max;
|
||||
float this_scale = 1/id;
|
||||
@@ -3403,9 +3485,14 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
|
||||
int grid_index = kmap_q2xs[u];
|
||||
if (grid_index < 0) {
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
}
|
||||
}
|
||||
// same candidate as for the previous scale: it cannot change the result
|
||||
if (is > -6 && memcmp(Laux, Laux_prev, 32) == 0) {
|
||||
continue;
|
||||
}
|
||||
memcpy(Laux_prev, Laux, 32);
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
float w = weight[i];
|
||||
@@ -3430,7 +3517,7 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
|
||||
int grid_index = kmap_q2xs[u];
|
||||
if (grid_index < 0) {
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
}
|
||||
const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index);
|
||||
for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2;
|
||||
@@ -3515,6 +3602,9 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
|
||||
bool is_on_grid_aux[2];
|
||||
uint8_t block_signs[2];
|
||||
uint16_t q2[2*(QK_K/16)];
|
||||
iq2_neighbour_cache neighbour_cache[2];
|
||||
int8_t Laux_prev[16];
|
||||
bool is_on_grid_prev[2];
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
@@ -3566,6 +3656,7 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
|
||||
float best = 0;
|
||||
float scale = max/(2*kMaxQ-1);
|
||||
is_on_grid[0] = is_on_grid[1] = true;
|
||||
for (int k = 0; k < 2; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
|
||||
for (int is = -9; is <= 9; ++is) {
|
||||
float id = (2*kMaxQ-1+is*0.1f)/max;
|
||||
float this_scale = 1/id;
|
||||
@@ -3581,9 +3672,15 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
|
||||
if (grid_index < 0) {
|
||||
is_on_grid_aux[k] = false;
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
}
|
||||
}
|
||||
// same candidate as for the previous scale: it cannot change the result
|
||||
if (is > -9 && memcmp(Laux, Laux_prev, 16) == 0 && is_on_grid_aux[0] == is_on_grid_prev[0] && is_on_grid_aux[1] == is_on_grid_prev[1]) {
|
||||
continue;
|
||||
}
|
||||
memcpy(Laux_prev, Laux, 16);
|
||||
is_on_grid_prev[0] = is_on_grid_aux[0]; is_on_grid_prev[1] = is_on_grid_aux[1];
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
for (int i = 0; i < 16; ++i) {
|
||||
float w = weight[i];
|
||||
@@ -3613,7 +3710,7 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
|
||||
int grid_index = kmap_q2xs[u];
|
||||
if (grid_index < 0) {
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
}
|
||||
}
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
@@ -5155,6 +5252,9 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
|
||||
bool is_on_grid[2];
|
||||
bool is_on_grid_aux[2];
|
||||
uint8_t block_signs[2];
|
||||
iq2_neighbour_cache neighbour_cache[2];
|
||||
int8_t Laux_prev[16];
|
||||
bool is_on_grid_prev[2];
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
@@ -5197,6 +5297,7 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
|
||||
float best = 0;
|
||||
float scale = max/(2*kMaxQ-1);
|
||||
is_on_grid[0] = is_on_grid[1] = true;
|
||||
for (int k = 0; k < 2; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
|
||||
for (int is = -9; is <= 9; ++is) {
|
||||
float id = (2*kMaxQ-1+is*0.1f)/max;
|
||||
float this_scale = 1/id;
|
||||
@@ -5212,9 +5313,15 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
|
||||
if (grid_index < 0) {
|
||||
is_on_grid_aux[k] = false;
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
}
|
||||
}
|
||||
// same candidate as for the previous scale: it cannot change the result
|
||||
if (is > -9 && memcmp(Laux, Laux_prev, 16) == 0 && is_on_grid_aux[0] == is_on_grid_prev[0] && is_on_grid_aux[1] == is_on_grid_prev[1]) {
|
||||
continue;
|
||||
}
|
||||
memcpy(Laux_prev, Laux, 16);
|
||||
is_on_grid_prev[0] = is_on_grid_aux[0]; is_on_grid_prev[1] = is_on_grid_aux[1];
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
for (int i = 0; i < 16; ++i) {
|
||||
float w = weight[i];
|
||||
@@ -5244,7 +5351,7 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
|
||||
int grid_index = kmap_q2xs[u];
|
||||
if (grid_index < 0) {
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
}
|
||||
}
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
|
||||
@@ -101,7 +101,7 @@ void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<vk_stagi
|
||||
|
||||
// matmul
|
||||
vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type);
|
||||
void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t padded_n, uint64_t b_col_bytes, uint64_t d_col_bytes);
|
||||
void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t split_k, uint32_t stream_k_groups, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t padded_n, uint64_t b_col_bytes, uint64_t d_col_bytes);
|
||||
bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor);
|
||||
vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to);
|
||||
vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx, ggml_type type);
|
||||
|
||||
@@ -225,7 +225,7 @@ void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size
|
||||
ctx, subctx, p, ggml_vk_subbuffer(ctx, d_X), ggml_vk_subbuffer(ctx, d_Y), ggml_vk_subbuffer(ctx, d_D), ggml_vk_subbuffer(ctx, ctx->prealloc_split_k),
|
||||
m, n, k,
|
||||
k, k, m, k*m, k*n, m*n,
|
||||
split_k, batch, batch, batch, 1, 1, n,
|
||||
split_k, 0, batch, batch, batch, 1, 1, n,
|
||||
k * sizeof(Y_TYPE), m * sizeof(float)
|
||||
);
|
||||
}
|
||||
@@ -693,7 +693,7 @@ void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t
|
||||
ctx, subctx, p, { qx_buf, 0, qx_sz }, { qy_buf, 0, qy_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },
|
||||
m, n, k,
|
||||
k, k, m, k*m, k*n, m*n,
|
||||
split_k, batch, batch, batch, 1, 1, n,
|
||||
split_k, 0, batch, batch, batch, 1, 1, n,
|
||||
(uint64_t)(k / ggml_blck_size(GGML_TYPE_Q8_1)) * ggml_type_size(GGML_TYPE_Q8_1), m * sizeof(float)
|
||||
);
|
||||
}
|
||||
@@ -703,7 +703,7 @@ void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t
|
||||
ctx, subctx, p, { qx_buf, 0, qx_sz }, { y_buf, 0, y_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },
|
||||
m, n, k,
|
||||
k, k, m, k*m, k*n, m*n,
|
||||
split_k, batch, batch, batch, 1, 1, n,
|
||||
split_k, 0, batch, batch, batch, 1, 1, n,
|
||||
k * sizeof(float), m * sizeof(float)
|
||||
);
|
||||
}
|
||||
@@ -1043,6 +1043,8 @@ void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph
|
||||
} else if (tensor->op == GGML_OP_FILL) {
|
||||
const float value = ggml_get_op_params_f32(tensor, 0);
|
||||
tensor_clone = ggml_fill(ggml_ctx, src_clone[0], value);
|
||||
} else if (tensor->op == GGML_OP_SLEEP) {
|
||||
tensor_clone = ggml_sleep(ggml_ctx, src_clone[0], ggml_get_op_params_i32(tensor, 0));
|
||||
} else if (tensor->op == GGML_OP_SQR) {
|
||||
tensor_clone = ggml_sqr(ggml_ctx, src_clone[0]);
|
||||
} else if (tensor->op == GGML_OP_SQRT) {
|
||||
|
||||
@@ -13,8 +13,38 @@ struct vk_mat_mat_push_constants {
|
||||
uint32_t k_split;
|
||||
uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3;
|
||||
uint32_t padded_N;
|
||||
uint32_t k_units_per_tile;
|
||||
uint32_t work_units_per_group;
|
||||
uint32_t groups_with_extra_unit;
|
||||
uint32_t k_units_mp;
|
||||
uint32_t blocks_m_mp;
|
||||
uint32_t blocks_n_mp;
|
||||
uint32_t ne12_mp;
|
||||
uint32_t broadcast2_mp;
|
||||
uint32_t broadcast3_mp;
|
||||
uint32_t div_Ls0;
|
||||
uint32_t div_Ls1;
|
||||
};
|
||||
|
||||
struct vk_mat_mat_stream_k_fixup_push_constants {
|
||||
uint32_t M; uint32_t N;
|
||||
uint32_t BM; uint32_t BN;
|
||||
uint32_t k_units_per_tile;
|
||||
uint32_t blocks_m;
|
||||
uint32_t blocks_n;
|
||||
uint32_t work_units_per_group;
|
||||
uint32_t groups_with_extra_unit;
|
||||
uint32_t k_units_mp;
|
||||
uint32_t blocks_m_mp;
|
||||
uint32_t blocks_n_mp;
|
||||
uint32_t work_units_mp;
|
||||
uint32_t large_group_work_mp;
|
||||
uint32_t div_Ls0;
|
||||
uint32_t div_Ls1;
|
||||
};
|
||||
|
||||
static_assert(sizeof(vk_mat_mat_push_constants) <= 128);
|
||||
|
||||
struct vk_mat_vec_push_constants {
|
||||
uint32_t ncols;
|
||||
uint32_t stride_a;
|
||||
@@ -159,6 +189,10 @@ struct vk_op_push_constants {
|
||||
float param4;
|
||||
};
|
||||
|
||||
struct vk_op_sleep_push_constants {
|
||||
uint32_t ticks;
|
||||
};
|
||||
|
||||
struct vk_op_fwht_push_constants {
|
||||
uint32_t n_rows;
|
||||
uint32_t src_offset;
|
||||
@@ -403,6 +437,34 @@ template <typename T> void init_pushconst_fastdiv(T &p) {
|
||||
static_assert(!std::is_const<T>::value, "unexpected type");
|
||||
}
|
||||
|
||||
inline void init_pushconst_fastdiv(vk_mat_mat_push_constants &p) {
|
||||
uint32_t ne12_L, broadcast2_L, broadcast3_L;
|
||||
init_fastdiv_values(p.ne12, p.ne12_mp, ne12_L);
|
||||
init_fastdiv_values(p.broadcast2, p.broadcast2_mp, broadcast2_L);
|
||||
init_fastdiv_values(p.broadcast3, p.broadcast3_mp, broadcast3_L);
|
||||
p.div_Ls1 = pack_fastdiv_L(ne12_L, broadcast2_L, broadcast3_L);
|
||||
}
|
||||
|
||||
inline void init_pushconst_fastdiv(vk_mat_mat_push_constants &p, uint32_t blocks_m, uint32_t blocks_n) {
|
||||
uint32_t k_units_L, blocks_m_L, blocks_n_L;
|
||||
init_fastdiv_values(p.k_units_per_tile, p.k_units_mp, k_units_L);
|
||||
init_fastdiv_values(blocks_m, p.blocks_m_mp, blocks_m_L);
|
||||
init_fastdiv_values(blocks_n, p.blocks_n_mp, blocks_n_L);
|
||||
p.div_Ls0 = pack_fastdiv_L(k_units_L, blocks_m_L, blocks_n_L);
|
||||
init_pushconst_fastdiv(p);
|
||||
}
|
||||
|
||||
template <> inline void init_pushconst_fastdiv(vk_mat_mat_stream_k_fixup_push_constants &p) {
|
||||
uint32_t k_units_L, blocks_m_L, blocks_n_L, work_units_L, large_group_work_L;
|
||||
init_fastdiv_values(p.k_units_per_tile, p.k_units_mp, k_units_L);
|
||||
init_fastdiv_values(p.blocks_m, p.blocks_m_mp, blocks_m_L);
|
||||
init_fastdiv_values(p.blocks_n, p.blocks_n_mp, blocks_n_L);
|
||||
init_fastdiv_values(p.work_units_per_group, p.work_units_mp, work_units_L);
|
||||
init_fastdiv_values(p.work_units_per_group + 1, p.large_group_work_mp, large_group_work_L);
|
||||
p.div_Ls0 = pack_fastdiv_L(k_units_L, blocks_m_L, blocks_n_L);
|
||||
p.div_Ls1 = pack_fastdiv_L(work_units_L, large_group_work_L, 0);
|
||||
}
|
||||
|
||||
template <> inline void init_pushconst_fastdiv(vk_op_unary_push_constants &p) {
|
||||
// Compute magic values to divide by these six numbers.
|
||||
uint32_t ne0_012L;
|
||||
|
||||
@@ -259,10 +259,11 @@ struct vk_matmul_pipeline_key {
|
||||
ggml_type type_b;
|
||||
bool mul_mat_id;
|
||||
bool f16acc;
|
||||
bool stream_k = false;
|
||||
|
||||
bool operator<(const vk_matmul_pipeline_key & o) const {
|
||||
return std::tie(type_a, type_b, mul_mat_id, f16acc)
|
||||
< std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc);
|
||||
return std::tie(type_a, type_b, mul_mat_id, f16acc, stream_k)
|
||||
< std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc, o.stream_k);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -738,6 +739,7 @@ struct vk_device_struct {
|
||||
bool shader_int64;
|
||||
bool buffer_device_address;
|
||||
bool vulkan_memory_model;
|
||||
bool shader_clock;
|
||||
|
||||
bool add_rms_fusion;
|
||||
uint32_t partials_binding_alignment;
|
||||
@@ -746,6 +748,10 @@ struct vk_device_struct {
|
||||
bool shader_64b_indexing;
|
||||
|
||||
bool integer_dot_product;
|
||||
// Which 4x8 bit packed integer dot products are accelerated
|
||||
bool integer_dot_signed {};
|
||||
bool integer_dot_mixed {};
|
||||
bool integer_dot_unsigned {};
|
||||
// 0: default, 1: force mmvq, -1: disable mmvq
|
||||
int32_t mmvq_mode;
|
||||
|
||||
@@ -819,6 +825,7 @@ struct vk_device_struct {
|
||||
matmul_tile_selector_t matmul_id_tile_selector;
|
||||
|
||||
vk_pipeline pipeline_matmul_split_k_reduce;
|
||||
vk_pipeline pipeline_matmul_stream_k_fixup;
|
||||
vk_pipeline pipeline_quantize_q8_1_x4;
|
||||
|
||||
vk_pipeline pipeline_dequant[GGML_TYPE_COUNT];
|
||||
@@ -917,6 +924,8 @@ struct vk_device_struct {
|
||||
vk_pipeline pipeline_fill_f32;
|
||||
vk_pipeline pipeline_fill_f16;
|
||||
|
||||
vk_pipeline pipeline_sleep;
|
||||
|
||||
// [src/dst 0=fp32,1=fp16,2=bf16]
|
||||
vk_pipeline pipeline_glu[GGML_GLU_OP_COUNT][3];
|
||||
|
||||
@@ -1023,6 +1032,8 @@ struct vk_device_struct {
|
||||
std::map<std::string, uint64_t> fusion_counts;
|
||||
bool disable_descriptor_reuse;
|
||||
std::atomic<uint64_t> buffer_destroy_count {};
|
||||
bool disable_stream_k;
|
||||
bool force_stream_k;
|
||||
bool disable_host_visible_vidmem;
|
||||
bool allow_sysmem_fallback;
|
||||
bool disable_graph_optimize;
|
||||
|
||||
@@ -1726,6 +1726,11 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec
|
||||
return supported;
|
||||
}
|
||||
|
||||
// MMQ uses the mixed signedness dot product (MMQ_SUDOT) when the signed one is not accelerated
|
||||
static bool ggml_vk_mmq_use_sudot(const vk_device& device) {
|
||||
return device->integer_dot_mixed && !device->integer_dot_signed;
|
||||
}
|
||||
|
||||
static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) {
|
||||
|
||||
// FLOAT_TYPE in the shader is float16_t with fp16 support, otherwise float.
|
||||
@@ -1782,7 +1787,12 @@ static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std:
|
||||
const uint32_t buf_b_size = BN * BK_STEP * block_b_size;
|
||||
const uint32_t mmid_row_ids = mul_mat_id ? (BN * 2u * (uint32_t)sizeof(uint16_t)) : 0u;
|
||||
|
||||
const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + lut_size;
|
||||
// Signed quants declare buf_b_qsum: 2 int32_t per B block with MMQ_SUDOT, a single int32_t without
|
||||
const bool b_sums = src0_type == GGML_TYPE_Q8_0 || src0_type == GGML_TYPE_MXFP4 || src0_type == GGML_TYPE_IQ4_XS ||
|
||||
src0_type == GGML_TYPE_IQ3_S || src0_type == GGML_TYPE_Q3_K || src0_type == GGML_TYPE_Q6_K;
|
||||
const uint32_t qsum_size = !b_sums ? 0u : ggml_vk_mmq_use_sudot(device) ? BN * BK_STEP * 2u * 4u : 4u;
|
||||
|
||||
const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + lut_size + qsum_size;
|
||||
const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize;
|
||||
|
||||
VK_LOG_DEBUG("ggml_vk_matmul_int_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), "
|
||||
@@ -1906,7 +1916,7 @@ static bool ggml_vk_fa_type_needs_shmem(ggml_type type) {
|
||||
|
||||
static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type, ggml_type v_type) {
|
||||
#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
|
||||
return device->integer_dot_product && device->subgroup_clustered &&
|
||||
return device->integer_dot_product && device->integer_dot_signed && device->subgroup_clustered &&
|
||||
!ggml_vk_fa_type_needs_shmem(v_type) &&
|
||||
(k_type == GGML_TYPE_Q4_0 || k_type == GGML_TYPE_Q4_1 ||
|
||||
k_type == GGML_TYPE_Q5_0 || k_type == GGML_TYPE_Q5_1 ||
|
||||
@@ -2467,6 +2477,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
FOR_EACH_LUT_FP4_TYPE(X)
|
||||
|
||||
const int mul_mat_id_param_count = 5;
|
||||
constexpr uint32_t stream_k_spec_id = 15;
|
||||
|
||||
using spec_fn_t = std::function<std::vector<uint32_t>(const std::vector<uint32_t>&, bool)>;
|
||||
auto const &create_mm_pipelines = [&](
|
||||
@@ -2478,6 +2489,11 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0,
|
||||
bool create_aligned = true, bool pin_subgroup_to_warp = false
|
||||
) {
|
||||
GGML_ASSERT(!key.stream_k || (!key.mul_mat_id && param_count == 3 && push_constant_size == sizeof(vk_mat_mat_push_constants) && create_aligned));
|
||||
// Non-ID matmul shaders declare the Stream-K scratch binding in all specializations.
|
||||
if (!key.mul_mat_id) {
|
||||
++param_count;
|
||||
}
|
||||
auto& vec = device->pipeline_matmul[key];
|
||||
const bool first_call = vec.empty();
|
||||
for (size_t i = 0; i < tile_configs.size(); i++) {
|
||||
@@ -2495,25 +2511,40 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
vk_matmul_pipeline_pair pair{};
|
||||
pair.align = tc.align;
|
||||
pair.narrow_n_max = tc.narrow_n_max;
|
||||
std::string suffix = "_" + std::to_string(i);
|
||||
pair.unaligned = std::make_shared<vk_pipeline_struct>();
|
||||
if (!key.stream_k) {
|
||||
pair.unaligned = std::make_shared<vk_pipeline_struct>();
|
||||
}
|
||||
if (create_aligned) {
|
||||
pair.aligned = std::make_shared<vk_pipeline_struct>();
|
||||
}
|
||||
if (key.stream_k) {
|
||||
// Tile selection reads unaligned even when selecting an aligned pipeline.
|
||||
pair.unaligned = pair.aligned;
|
||||
}
|
||||
vec.push_back(pair);
|
||||
}
|
||||
|
||||
ggml_vk_create_pipeline(device, vec[i].unaligned,
|
||||
vec[i].unaligned->name.empty() ? (shader_name + "_" + std::to_string(i)).c_str() : vec[i].unaligned->name.c_str(),
|
||||
spv_len, spv_data, "main", param_count, push_constant_size,
|
||||
tc.wg_denoms, spec_fn(tc.warptile, false), 1,
|
||||
disable_robustness, rfs, rsgs);
|
||||
if (!key.stream_k) {
|
||||
ggml_vk_create_pipeline(device, vec[i].unaligned,
|
||||
vec[i].unaligned->name.empty() ? (shader_name + "_" + std::to_string(i)).c_str() : vec[i].unaligned->name.c_str(),
|
||||
spv_len, spv_data, "main", param_count, push_constant_size,
|
||||
tc.wg_denoms, spec_fn(tc.warptile, false), 1,
|
||||
disable_robustness, rfs, rsgs);
|
||||
}
|
||||
|
||||
if (vec[i].aligned) {
|
||||
auto spec = spec_fn(tc.warptile, true);
|
||||
if (key.stream_k) {
|
||||
if (spec.size() <= stream_k_spec_id) {
|
||||
spec.resize(stream_k_spec_id + 1);
|
||||
}
|
||||
spec[stream_k_spec_id] = 1;
|
||||
}
|
||||
const std::string name = shader_name + (key.stream_k ? "_stream_k_" : "_aligned_") + std::to_string(i);
|
||||
ggml_vk_create_pipeline(device, vec[i].aligned,
|
||||
vec[i].aligned->name.empty() ? (shader_name + "_aligned_" + std::to_string(i)).c_str() : vec[i].aligned->name.c_str(),
|
||||
vec[i].aligned->name.empty() ? name.c_str() : vec[i].aligned->name.c_str(),
|
||||
spv_len, spv_data, "main", param_count, push_constant_size,
|
||||
tc.wg_denoms, spec_fn(tc.warptile, true), tc.align,
|
||||
tc.wg_denoms, spec, tc.align,
|
||||
disable_robustness, rfs, rsgs);
|
||||
}
|
||||
}
|
||||
@@ -2611,13 +2642,19 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
}
|
||||
|
||||
spec_fn_t cm2_spec = [&](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a); };
|
||||
auto create_cm2_pipelines = [&](vk_matmul_pipeline_key key, const std::vector<vk_tile_config>& tc,
|
||||
const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, const spec_fn_t& spec) {
|
||||
create_mm_pipelines(key, tc, name, len, data, pc_size, pc, spec, true);
|
||||
key.stream_k = true;
|
||||
create_mm_pipelines(key, tc, name, len, data, pc_size, pc, spec, true);
|
||||
};
|
||||
|
||||
// F16 x F16
|
||||
create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm2_len, matmul_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm2_len, matmul_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_cm2_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm2_len, matmul_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
create_cm2_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm2_len, matmul_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)
|
||||
if (device->coopmat_bf16_support) {
|
||||
create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm2_len, matmul_bf16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_cm2_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm2_len, matmul_bf16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
}
|
||||
#endif
|
||||
for (const auto type : non_lut_quant_types) {
|
||||
@@ -2627,18 +2664,18 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
}
|
||||
auto& tc = ((type >= GGML_TYPE_Q2_K && type <= GGML_TYPE_Q6_K) || type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0) ? tc_mmq_k : tc_mmq;
|
||||
spec_fn_t qs = [&, type](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); };
|
||||
create_mm_pipelines({type, GGML_TYPE_F16, false, true}, tc, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm2_len, matmul_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true);
|
||||
create_mm_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16", matmul_quant_f16_cm2_len, matmul_quant_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true);
|
||||
create_cm2_pipelines({type, GGML_TYPE_F16, false, true}, tc, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm2_len, matmul_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs);
|
||||
create_cm2_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16", matmul_quant_f16_cm2_len, matmul_quant_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs);
|
||||
}
|
||||
create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q4_k_f16_f16acc", matmul_q4_k_f16_f16acc_cm2_len, matmul_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q4_k_f16", matmul_q4_k_f16_cm2_len, matmul_q4_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q5_k_f16_f16acc", matmul_q5_k_f16_f16acc_cm2_len, matmul_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q5_k_f16", matmul_q5_k_f16_cm2_len, matmul_q5_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_cm2_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q4_k_f16_f16acc", matmul_q4_k_f16_f16acc_cm2_len, matmul_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
create_cm2_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q4_k_f16", matmul_q4_k_f16_cm2_len, matmul_q4_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
create_cm2_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q5_k_f16_f16acc", matmul_q5_k_f16_f16acc_cm2_len, matmul_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
create_cm2_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q5_k_f16", matmul_q5_k_f16_cm2_len, matmul_q5_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
#define X_CM2(TYPE, tstr) \
|
||||
{ auto tc = filter_tc(tc_mmq, TYPE, false); \
|
||||
if (!tc.empty()) { \
|
||||
create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm2_len, matmul_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \
|
||||
create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm2_len, matmul_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \
|
||||
create_cm2_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm2_len, matmul_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec); \
|
||||
create_cm2_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm2_len, matmul_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec); \
|
||||
} }
|
||||
FOR_EACH_LUT_TYPE_NONFP4(X_CM2)
|
||||
#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT)
|
||||
@@ -2646,8 +2683,8 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
#define X_CM2_OCP(TYPE, tstr) \
|
||||
{ auto tc = filter_tc(tc_mmq, TYPE, false); \
|
||||
if (!tc.empty()) { \
|
||||
create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_ocp_f16acc", matmul_##tstr##_f16_ocp_f16acc_cm2_len, matmul_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \
|
||||
create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16_ocp", matmul_##tstr##_f16_ocp_cm2_len, matmul_##tstr##_f16_ocp_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \
|
||||
create_cm2_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_ocp_f16acc", matmul_##tstr##_f16_ocp_f16acc_cm2_len, matmul_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec); \
|
||||
create_cm2_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16_ocp", matmul_##tstr##_f16_ocp_cm2_len, matmul_##tstr##_f16_ocp_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec); \
|
||||
} }
|
||||
FOR_EACH_LUT_FP4_TYPE(X_CM2_OCP)
|
||||
#undef X_CM2_OCP
|
||||
@@ -2925,8 +2962,13 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
uint32_t rsgs = 0) {
|
||||
auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id, true);
|
||||
if (!tc.empty()) {
|
||||
spec_fn_t identity = [](const std::vector<uint32_t>& wt, bool) { return wt; };
|
||||
create_mm_pipelines(key, tc, name, len, data, pc_size, pc, identity, false, rsgs > 0, rsgs, false);
|
||||
// constant_id 11 is MMQ_SUDOT
|
||||
spec_fn_t mmq_spec = [&](const std::vector<uint32_t>& wt, bool) {
|
||||
std::vector<uint32_t> spec = wt;
|
||||
spec.push_back(ggml_vk_mmq_use_sudot(device) ? 1u : 0u);
|
||||
return spec;
|
||||
};
|
||||
create_mm_pipelines(key, tc, name, len, data, pc_size, pc, mmq_spec, false, rsgs > 0, rsgs, false);
|
||||
}
|
||||
};
|
||||
#endif
|
||||
@@ -3515,6 +3557,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_get_rows_back_f32, "get_rows_back_f32", get_rows_back_f32_len, get_rows_back_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {256, 1, 1}, {}, 1, true);
|
||||
|
||||
ggml_vk_create_pipeline(device, device->pipeline_matmul_split_k_reduce, "split_k_reduce", split_k_reduce_len, split_k_reduce_data, "main", 2, 2 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_matmul_stream_k_fixup, "stream_k_fixup", stream_k_fixup_len, stream_k_fixup_data, "main", 2, sizeof(vk_mat_mat_stream_k_fixup_push_constants), {1, 32, 1}, {}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_split_k_reduce, "fa_split_k_reduce", fa_split_k_reduce_len, fa_split_k_reduce_data, "main", 3, sizeof(vk_op_flash_attn_split_k_reduce_push_constants), {1, device->subgroup_size, 1}, {device->subgroup_size}, 1, true);
|
||||
|
||||
#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT)
|
||||
@@ -3849,6 +3892,10 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_fill_f32, "fill_f32", fill_f32_len, fill_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_fill_f16, "fill_f16", fill_f16_len, fill_f16_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
|
||||
|
||||
if (device->shader_clock) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_sleep, "sleep", sleep_len, sleep_data, "main", 1, sizeof(vk_op_sleep_push_constants), {1, 1, 1}, {}, 1);
|
||||
}
|
||||
|
||||
#define CREATE_GLU(name, op) \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
|
||||
@@ -4478,6 +4525,7 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
bool ocp_microscaling_extension = false;
|
||||
bool shader_float8_extension = false;
|
||||
bool workgroup_memory_explicit_layout_support = false;
|
||||
bool shader_clock_support = false;
|
||||
|
||||
for (const auto& properties : ext_props) {
|
||||
if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) {
|
||||
@@ -4528,6 +4576,9 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
} else if (strcmp(VK_EXT_SHADER_FLOAT8_EXTENSION_NAME, properties.extensionName) == 0) {
|
||||
shader_float8_extension = true;
|
||||
#endif
|
||||
} else if (strcmp("VK_KHR_shader_clock", properties.extensionName) == 0 &&
|
||||
!getenv("GGML_VK_DISABLE_SHADER_CLOCK")) {
|
||||
shader_clock_support = true;
|
||||
} else if (strcmp("VK_VALVE_shader_mixed_float_dot_product", properties.extensionName) == 0 &&
|
||||
!getenv("GGML_VK_DISABLE_DOT2")) {
|
||||
dot2_f16_support = true;
|
||||
@@ -4716,7 +4767,10 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
device->coopmat_support = false;
|
||||
}
|
||||
|
||||
device->integer_dot_product = device->integer_dot_product && shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated;
|
||||
device->integer_dot_signed = shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated;
|
||||
device->integer_dot_mixed = shader_integer_dot_product_props.integerDotProduct4x8BitPackedMixedSignednessAccelerated;
|
||||
device->integer_dot_unsigned = shader_integer_dot_product_props.integerDotProduct4x8BitPackedUnsignedAccelerated;
|
||||
device->integer_dot_product = device->integer_dot_product && (device->integer_dot_signed || device->integer_dot_mixed);
|
||||
|
||||
device->min_imported_host_pointer_alignment = external_memory_host_props.minImportedHostPointerAlignment;
|
||||
|
||||
@@ -4873,6 +4927,14 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
device_extensions.push_back("VK_KHR_shader_integer_dot_product");
|
||||
}
|
||||
|
||||
VkPhysicalDeviceShaderClockFeaturesKHR shader_clock_features {};
|
||||
shader_clock_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CLOCK_FEATURES_KHR;
|
||||
if (shader_clock_support) {
|
||||
last_struct->pNext = (VkBaseOutStructure *)&shader_clock_features;
|
||||
last_struct = (VkBaseOutStructure *)&shader_clock_features;
|
||||
device_extensions.push_back("VK_KHR_shader_clock");
|
||||
}
|
||||
|
||||
VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE dot2_features {};
|
||||
dot2_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE;
|
||||
if (dot2_f16_support) {
|
||||
@@ -4979,6 +5041,7 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
device->shader_int64 = device_features2.features.shaderInt64;
|
||||
device->buffer_device_address = vk12_features.bufferDeviceAddress;
|
||||
device->vulkan_memory_model = vk12_features.vulkanMemoryModel;
|
||||
device->shader_clock = shader_clock_support && shader_clock_features.shaderDeviceClock;
|
||||
|
||||
if (device->subgroup_size_control) {
|
||||
device->subgroup_min_size = subgroup_size_control_props.minSubgroupSize;
|
||||
@@ -5375,6 +5438,8 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr;
|
||||
|
||||
device->disable_fusion = getenv("GGML_VK_DISABLE_FUSION") != nullptr;
|
||||
device->disable_stream_k = getenv("GGML_VK_DISABLE_STREAM_K") != nullptr;
|
||||
device->force_stream_k = getenv("GGML_VK_FORCE_STREAM_K") != nullptr;
|
||||
|
||||
device->disable_descriptor_reuse = getenv("GGML_VK_DISABLE_DESCRIPTOR_REUSE") != nullptr;
|
||||
|
||||
@@ -5593,7 +5658,8 @@ static void ggml_vk_print_gpu_info(size_t idx) {
|
||||
const bool rebar = ggml_vk_has_rebar(physical_device);
|
||||
|
||||
integer_dot_product = integer_dot_product
|
||||
&& shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated
|
||||
&& (shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated ||
|
||||
shader_integer_dot_product_props.integerDotProduct4x8BitPackedMixedSignednessAccelerated)
|
||||
&& shader_integer_dot_product_features.shaderIntegerDotProduct;
|
||||
|
||||
coopmat_support = coopmat_support
|
||||
@@ -6408,14 +6474,113 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m,
|
||||
return split_k;
|
||||
}
|
||||
|
||||
static uint32_t ggml_vk_guess_stream_k_groups(
|
||||
ggml_backend_vk_context * ctx, const vk_pipeline & pipeline,
|
||||
uint32_t m, uint32_t n, uint32_t k, uint32_t stride_d, uint32_t batch, uint32_t split_k, bool disable_split_k) {
|
||||
if (disable_split_k) {
|
||||
return 0;
|
||||
}
|
||||
if (ctx->device->disable_stream_k) {
|
||||
return 0;
|
||||
}
|
||||
// TODO: Tune the CM1 and scalar workgroup counts before enabling Stream-K.
|
||||
if (!ctx->device->coopmat2) {
|
||||
return 0;
|
||||
}
|
||||
if (ctx->device->shader_core_count == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const uint32_t bm = pipeline->wg_denoms[0];
|
||||
const uint32_t bn = pipeline->wg_denoms[1];
|
||||
if (bm % 32 != 0 || m % bm != 0 || n % bn != 0 || k % 256 != 0 || stride_d != m) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const uint32_t k_units = k / 256;
|
||||
if (k_units == 1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const uint64_t tiles = uint64_t(m / bm) * (n / bn) * batch;
|
||||
const uint64_t total_work_units = tiles * k_units;
|
||||
if (total_work_units > UINT32_MAX) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Coopmat2 tiles are tuned for one resident workgroup per SM.
|
||||
const uint32_t target_groups = ctx->device->shader_core_count;
|
||||
const uint64_t regular_groups = tiles * split_k;
|
||||
const uint64_t regular_waves = CEIL_DIV(regular_groups, target_groups);
|
||||
const uint64_t regular_capacity = regular_waves * target_groups;
|
||||
|
||||
const uint32_t stream_k_groups = std::min<uint64_t>(target_groups, total_work_units);
|
||||
if (stream_k_groups > ctx->device->properties.limits.maxComputeWorkGroupCount[0] ||
|
||||
bm / 32 > ctx->device->properties.limits.maxComputeWorkGroupCount[1]) {
|
||||
return 0;
|
||||
}
|
||||
if (!ctx->device->force_stream_k) {
|
||||
if (regular_waves == 1 && stream_k_groups == regular_groups) {
|
||||
return 0;
|
||||
}
|
||||
// Stream-K does not offset its overhead when the regular grid fills most of the last wave.
|
||||
// Use a lower cutoff for short K with no K split and narrow N tiles.
|
||||
if ((split_k == 1 && k <= 4096 && bn <= 128 && regular_groups * 4 >= regular_capacity * 3) ||
|
||||
regular_groups * 10 >= regular_capacity * 9) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return stream_k_groups;
|
||||
}
|
||||
|
||||
void ggml_vk_matmul(
|
||||
ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline,
|
||||
vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer,
|
||||
uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d,
|
||||
uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d,
|
||||
uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3,
|
||||
uint32_t split_k, uint32_t stream_k_groups, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3,
|
||||
uint32_t padded_n, uint64_t b_col_bytes, uint64_t d_col_bytes) {
|
||||
VK_LOG_DEBUG("ggml_vk_matmul(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), split_k: (" << (split_k_buffer.buffer != nullptr ? split_k_buffer.buffer->buffer : VK_NULL_HANDLE) << ", " << split_k_buffer.offset << ", " << split_k_buffer.size << "), m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", split_k: " << split_k << ", batch: " << batch << ", ne02: " << ne02 << ", ne12: " << ne12 << ", broadcast2: " << broadcast2 << ", broadcast3: " << broadcast3 << ", padded_n: " << padded_n << ")");
|
||||
if (stream_k_groups != 0) {
|
||||
GGML_ASSERT(batch_stride_d == m * n);
|
||||
if (ctx->prealloc_split_k_need_sync) {
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
}
|
||||
|
||||
ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
|
||||
ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_stream_k_fixup, 1);
|
||||
|
||||
const uint32_t bm = pipeline->wg_denoms[0];
|
||||
const uint32_t bn = pipeline->wg_denoms[1];
|
||||
const uint32_t k_units_per_tile = k / 256;
|
||||
const uint32_t blocks_m = m / bm;
|
||||
const uint32_t blocks_n = n / bn;
|
||||
const uint32_t total_work_units = blocks_m * blocks_n * batch * k_units_per_tile;
|
||||
const uint32_t work_units_per_group = total_work_units / stream_k_groups;
|
||||
const uint32_t groups_with_extra_unit = total_work_units % stream_k_groups;
|
||||
|
||||
vk_mat_mat_push_constants pc1 = {
|
||||
m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, 0, batch, k, ne02, ne12, broadcast2, broadcast3, padded_n,
|
||||
k_units_per_tile, work_units_per_group, groups_with_extra_unit,
|
||||
0, 0, 0, 0, 0, 0, 0, 0
|
||||
};
|
||||
init_pushconst_fastdiv(pc1, blocks_m, blocks_n);
|
||||
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, split_k_buffer }, pc1, { stream_k_groups * pipeline->wg_denoms[0], 1, 1 });
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
|
||||
vk_mat_mat_stream_k_fixup_push_constants pc2 = {
|
||||
m, n, bm, bn, k_units_per_tile, blocks_m, blocks_n, work_units_per_group, groups_with_extra_unit,
|
||||
0, 0, 0, 0, 0, 0, 0
|
||||
};
|
||||
init_pushconst_fastdiv(pc2);
|
||||
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_matmul_stream_k_fixup, { split_k_buffer, d }, pc2, { stream_k_groups, bm, 1 });
|
||||
ctx->prealloc_split_k_need_sync = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (split_k == 1) {
|
||||
// a small tile can still produce more workgroups than the device allows on the n axis:
|
||||
// split the dispatch into n-chunks, shifting the B and D column windows
|
||||
@@ -6438,8 +6603,12 @@ void ggml_vk_matmul(
|
||||
while (base_work_group_z < batch) {
|
||||
uint32_t groups_z = std::min(batch - base_work_group_z, ctx->device->properties.limits.maxComputeWorkGroupCount[2]);
|
||||
|
||||
const vk_mat_mat_push_constants pc = { m, n_chunk, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k, ne02, ne12, broadcast2, broadcast3, uint32_t(padded_n - n_base) };
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b_chunk, d_chunk }, pc, { m, n_chunk, groups_z });
|
||||
vk_mat_mat_push_constants pc = {
|
||||
m, n_chunk, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k, ne02, ne12, broadcast2, broadcast3, uint32_t(padded_n - n_base),
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
||||
};
|
||||
init_pushconst_fastdiv(pc);
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b_chunk, d_chunk, d_chunk }, pc, { m, n_chunk, groups_z });
|
||||
base_work_group_z += groups_z;
|
||||
}
|
||||
}
|
||||
@@ -6462,9 +6631,13 @@ void ggml_vk_matmul(
|
||||
while (base_work_group_z < batch) {
|
||||
uint32_t groups_z = std::min(batch - base_work_group_z, ctx->device->properties.limits.maxComputeWorkGroupCount[2]);
|
||||
|
||||
const vk_mat_mat_push_constants pc1 = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k_split, ne02, ne12, broadcast2, broadcast3, padded_n };
|
||||
vk_mat_mat_push_constants pc1 = {
|
||||
m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k_split, ne02, ne12, broadcast2, broadcast3, padded_n,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
||||
};
|
||||
init_pushconst_fastdiv(pc1);
|
||||
// Make sure enough workgroups get assigned for split k to work
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, split_k_buffer }, pc1, { (CEIL_DIV(m, pipeline->wg_denoms[0]) * pipeline->wg_denoms[0]) * split_k, n, groups_z });
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, split_k_buffer, split_k_buffer }, pc1, { (CEIL_DIV(m, pipeline->wg_denoms[0]) * pipeline->wg_denoms[0]) * split_k, n, groups_z });
|
||||
base_work_group_z += groups_z;
|
||||
}
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
@@ -6518,10 +6691,11 @@ static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_t
|
||||
return ctx->device->fp16 && prec_allows_f16;
|
||||
}
|
||||
|
||||
static const std::vector<vk_matmul_pipeline_pair>* ggml_vk_get_mul_mat_mat_pipeline_map(
|
||||
ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec, bool mul_mat_id = false) {
|
||||
static const std::vector<vk_matmul_pipeline_pair> * ggml_vk_get_mul_mat_mat_pipeline_map(
|
||||
ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec,
|
||||
bool mul_mat_id = false, bool stream_k = false) {
|
||||
bool f16acc = ggml_vk_get_mul_mat_mat_f16acc(ctx, src0_type, src1_type, prec);
|
||||
vk_matmul_pipeline_key key{src0_type, src1_type, mul_mat_id, f16acc};
|
||||
vk_matmul_pipeline_key key{src0_type, src1_type, mul_mat_id, f16acc, stream_k};
|
||||
auto it = ctx->device->pipeline_matmul.find(key);
|
||||
if (it == ctx->device->pipeline_matmul.end() || it->second.empty()) {
|
||||
// Try without f16acc
|
||||
@@ -6894,25 +7068,50 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
|
||||
const bool y_f32_kernel = y_kernel_type == GGML_TYPE_F32;
|
||||
const bool y_needs_reformat = !quantize_y && (!ggml_vk_dim01_contiguous(src1) || y_kernel_type != src1->type);
|
||||
|
||||
const std::vector<vk_matmul_pipeline_pair>* mmp_map =
|
||||
ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, ggml_prec_get_acc(dst));
|
||||
ggml_type mmp_type_a = src0->type;
|
||||
const ggml_type mmp_type_b = y_kernel_type;
|
||||
const std::vector<vk_matmul_pipeline_pair> * mmp_map =
|
||||
ggml_vk_get_mul_mat_mat_pipeline_map(ctx, mmp_type_a, mmp_type_b, ggml_prec_get_acc(dst));
|
||||
|
||||
const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig;
|
||||
const bool qy_needs_dequant = y_needs_reformat;
|
||||
|
||||
if (qx_needs_dequant) {
|
||||
// dequant src0 to f16
|
||||
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, ggml_prec_get_acc(dst));
|
||||
mmp_type_a = f16_type;
|
||||
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, mmp_type_a, mmp_type_b, ggml_prec_get_acc(dst));
|
||||
}
|
||||
|
||||
GGML_ASSERT(mmp_map != nullptr);
|
||||
|
||||
const std::vector<vk_matmul_pipeline_pair> * regular_mmp_map = mmp_map;
|
||||
bool use_stream_k = ctx->device->coopmat2 && !ctx->device->disable_stream_k && !disable_split_k;
|
||||
if (use_stream_k) {
|
||||
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, mmp_type_a, mmp_type_b, ggml_prec_get_acc(dst), false, true);
|
||||
if (mmp_map == nullptr) {
|
||||
mmp_map = regular_mmp_map;
|
||||
use_stream_k = false;
|
||||
}
|
||||
}
|
||||
|
||||
const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, ne11, false));
|
||||
const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && ne11 > 8;
|
||||
const bool needs_64b_indexing = ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange;
|
||||
if (use_stream_k && !aligned) {
|
||||
mmp_map = regular_mmp_map;
|
||||
use_stream_k = false;
|
||||
}
|
||||
|
||||
vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, ne11, aligned, false);
|
||||
|
||||
if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) {
|
||||
const bool narrow_n = mmp_map->back().narrow_n_max != 0 && ne11 <= mmp_map->back().narrow_n_max;
|
||||
const uint32_t split_k = ggml_vk_guess_split_k(ctx, ne01, ne11, ne10, disable_split_k, pipeline, narrow_n);
|
||||
const uint32_t stream_k_groups = use_stream_k ? ggml_vk_guess_stream_k_groups(ctx, pipeline, ne01, ne11, ne10, stride_d, ne12 * ne13, split_k, disable_split_k) : 0;
|
||||
if (use_stream_k && stream_k_groups == 0) {
|
||||
pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *regular_mmp_map, ne01, ne11, aligned, false);
|
||||
}
|
||||
|
||||
if (needs_64b_indexing) {
|
||||
pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline);
|
||||
}
|
||||
|
||||
@@ -6923,9 +7122,6 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
|
||||
const uint64_t y_ne = padded_n * ne10 * ne12 * ne13;
|
||||
const uint64_t d_ne = ggml_nelements(dst);
|
||||
|
||||
const bool narrow_n = mmp_map->back().narrow_n_max != 0 && ne11 <= mmp_map->back().narrow_n_max;
|
||||
const uint32_t split_k = ggml_vk_guess_split_k(ctx, ne01, ne11, ne10, disable_split_k, pipeline, narrow_n);
|
||||
|
||||
const uint64_t qx_sz = ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type);
|
||||
const uint64_t qy_sz = ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type);
|
||||
const uint64_t x_sz = !qx_needs_dequant ? qx_sz : sizeof(ggml_fp16_t) * x_ne;
|
||||
@@ -6953,12 +7149,13 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
|
||||
to_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1);
|
||||
}
|
||||
|
||||
const uint64_t stream_k_size = stream_k_groups != 0 ? uint64_t(stream_k_groups) * pipeline->wg_denoms[0] * pipeline->wg_denoms[1] * sizeof(float) : 0;
|
||||
const uint64_t split_k_size = stream_k_groups != 0 ? stream_k_size : (split_k > 1 ? d_sz * split_k : 0);
|
||||
{
|
||||
const uint64_t split_k_size = split_k > 1 ? d_sz * split_k : 0;
|
||||
if (
|
||||
(qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) ||
|
||||
(qy_needs_dequant && y_sz > ctx->device->properties.limits.maxStorageBufferRange) ||
|
||||
(split_k > 1 && split_k_size > ctx->device->properties.limits.maxStorageBufferRange)) {
|
||||
(split_k_size > ctx->device->properties.limits.maxStorageBufferRange)) {
|
||||
GGML_ABORT("Requested preallocation size is too large");
|
||||
}
|
||||
if (qx_needs_dequant && ctx->prealloc_size_x < x_sz) {
|
||||
@@ -6969,7 +7166,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
|
||||
ctx->prealloc_size_y = y_sz;
|
||||
ggml_vk_preallocate_buffers(ctx, subctx);
|
||||
}
|
||||
if (split_k > 1 && ctx->prealloc_size_split_k < split_k_size) {
|
||||
if (split_k_size != 0 && ctx->prealloc_size_split_k < split_k_size) {
|
||||
ctx->prealloc_size_split_k = split_k_size;
|
||||
ggml_vk_preallocate_buffers(ctx, subctx);
|
||||
}
|
||||
@@ -6984,7 +7181,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
|
||||
if (quantize_y) {
|
||||
ggml_pipeline_request_descriptor_sets(ctx, to_q8_1, 1);
|
||||
}
|
||||
if (split_k > 1) {
|
||||
if (stream_k_groups == 0 && split_k > 1) {
|
||||
ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, 1);
|
||||
}
|
||||
}
|
||||
@@ -7092,10 +7289,10 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
|
||||
ggml_vk_matmul(
|
||||
ctx, subctx, pipeline,
|
||||
{ d_X, x_buf_offset, x_range }, { d_Y, y_buf_offset, y_range },
|
||||
ggml_vk_subbuffer(ctx, d_D, d_buf_offset), { ctx->prealloc_split_k, 0, d_sz * split_k },
|
||||
ggml_vk_subbuffer(ctx, d_D, d_buf_offset), { ctx->prealloc_split_k, 0, split_k_size },
|
||||
ne01, ne11, ne10,
|
||||
ne10, ne10, stride_d, stride_batch_x, stride_batch_y, stride_batch_d,
|
||||
split_k, ne12*ne13, ne02, ne12, r2, r3, padded_n,
|
||||
split_k, stream_k_groups, ne12*ne13, ne02, ne12, r2, r3, padded_n,
|
||||
b_col_bytes, (uint64_t)dst->nb[1]
|
||||
); // NOLINT
|
||||
|
||||
@@ -7234,7 +7431,7 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context&
|
||||
|
||||
const ggml_type y_kernel_type = widen_bf16 ? GGML_TYPE_F32 : src1->type;
|
||||
const bool f16_f32_kernel = y_kernel_type == GGML_TYPE_F32;
|
||||
bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && (swap_inputs || ggml_vk_src1_prec_allows_q8(dst)) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type);
|
||||
bool quantize_y = ctx->device->integer_dot_signed && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && (swap_inputs || ggml_vk_src1_prec_allows_q8(dst)) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type);
|
||||
|
||||
vk_pipeline to_fp16_vk_0 = nullptr;
|
||||
vk_pipeline to_fp16_vk_1 = nullptr;
|
||||
@@ -8302,7 +8499,7 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte
|
||||
const bool y_non_contig = !ggml_vk_dim01_contiguous(src1);
|
||||
|
||||
const bool f16_f32_kernel = src1->type == GGML_TYPE_F32;
|
||||
bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type);
|
||||
bool quantize_y = ctx->device->integer_dot_signed && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type);
|
||||
|
||||
vk_pipeline to_fp16_vk_0 = nullptr;
|
||||
vk_pipeline to_fp16_vk_1 = nullptr;
|
||||
@@ -10055,6 +10252,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
return ctx->device->pipeline_arange_f32;
|
||||
}
|
||||
return nullptr;
|
||||
case GGML_OP_SLEEP:
|
||||
return ctx->device->pipeline_sleep;
|
||||
case GGML_OP_FILL:
|
||||
if (dst->type == GGML_TYPE_F32) {
|
||||
return ctx->device->pipeline_fill_f32;
|
||||
@@ -11291,6 +11490,31 @@ void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements);
|
||||
}
|
||||
|
||||
static void ggml_vk_sleep(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
|
||||
VK_LOG_DEBUG("ggml_vk_sleep(dst=" << dst << ", us=" << ggml_get_op_params_i32(dst, 0) << ")");
|
||||
|
||||
// Vulkan does not specify the period of the shader realtime clock this assumes it matches the timestamp counter.
|
||||
// A driver where this does not hold could sleep too short or too long by that ratio
|
||||
const float ns_per_tick = ctx->device->properties.limits.timestampPeriod;
|
||||
const uint64_t ns = 1000 * (uint64_t) ggml_get_op_params_i32(dst, 0);
|
||||
const uint64_t ticks = std::min<uint64_t>(ns_per_tick > 0.0f ? (uint64_t)(ns / ns_per_tick) : ns, std::numeric_limits<uint32_t>::max());
|
||||
|
||||
vk_op_sleep_push_constants pc = {
|
||||
(uint32_t) ticks,
|
||||
};
|
||||
|
||||
vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, nullptr, nullptr, nullptr, dst, GGML_OP_SLEEP);
|
||||
GGML_ASSERT(pipeline != nullptr);
|
||||
|
||||
ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
|
||||
vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst);
|
||||
|
||||
// a single invocation spins, the buffer is bound only to keep the spin from being optimized out
|
||||
std::array<uint32_t, 3> elements = { 1, 1, 1 };
|
||||
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements);
|
||||
}
|
||||
|
||||
void ggml_vk_sin(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
|
||||
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SIN, vk_op_unary_push_constants_init(src0, dst));
|
||||
}
|
||||
@@ -13137,6 +13361,10 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in
|
||||
case GGML_OP_FILL:
|
||||
ggml_vk_fill(ctx, compute_ctx, node);
|
||||
|
||||
break;
|
||||
case GGML_OP_SLEEP:
|
||||
ggml_vk_sleep(ctx, compute_ctx, node);
|
||||
|
||||
break;
|
||||
case GGML_OP_SCALE:
|
||||
if (ctx->fused_hc_post_gate) {
|
||||
@@ -16588,6 +16816,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
return op->type == GGML_TYPE_F32;
|
||||
case GGML_OP_FILL:
|
||||
return op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;
|
||||
case GGML_OP_SLEEP:
|
||||
return device->shader_clock;
|
||||
case GGML_OP_SCALE:
|
||||
return ggml_is_contiguous(op->src[0]) && ggml_vk_is_float_elementwise(op);
|
||||
case GGML_OP_PAD:
|
||||
|
||||
@@ -22,6 +22,6 @@ void main() {
|
||||
if (i10 == i11) {
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(data_a[get_aoffset() + i13*p.nb03 + i12*p.nb02 + 0*p.nb01 + i10*p.nb00]);
|
||||
} else {
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0);
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0.0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
#version 450
|
||||
|
||||
#include "utils.glsl"
|
||||
|
||||
layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
layout (binding = 0) readonly buffer S4 {vec4 data_s4[];};
|
||||
layout (binding = 1) buffer D4 {vec4 data_d4[];};
|
||||
|
||||
layout (push_constant) uniform parameter {
|
||||
uint M;
|
||||
uint N;
|
||||
uint BM;
|
||||
uint BN;
|
||||
uint k_units_per_tile;
|
||||
uint blocks_m;
|
||||
uint blocks_n;
|
||||
uint work_units_per_group;
|
||||
uint groups_with_extra_unit;
|
||||
uint k_units_mp;
|
||||
uint blocks_m_mp;
|
||||
uint blocks_n_mp;
|
||||
uint work_units_mp;
|
||||
uint large_group_work_mp;
|
||||
uint div_Ls0;
|
||||
uint div_Ls1;
|
||||
} p;
|
||||
|
||||
const uint RESOLVE_ROWS = 32;
|
||||
|
||||
// A normal matmul workgroup accumulates one BM x BN output tile over the full K range.
|
||||
// Stream-K divides that accumulation into work units. Each work unit covers 256 consecutive K values and produces a BM x BN partial sum.
|
||||
// Work units are ordered [K unit, M tile, N tile, batch] and divided into one contiguous range per main workgroup.
|
||||
// A main workgroup writes its partial sum to D when it reaches the end of an output tile, or to scratch when its range stops within a tile.
|
||||
// Scratch has shape [BM, BN, stream_k_groups] and stores float partial sums.
|
||||
// Element [r, c, g] is workgroup g's contribution to local output element [r, c] in the last output tile of its range.
|
||||
// The output tile for each scratch slot is derived from the workgroup's range and is not stored in scratch.
|
||||
// Only the last output tile in a workgroup's range can stop early, so one scratch slot is enough.
|
||||
// Fixup workgroup (x, y) handles main workgroup x and rows 32*y through 32*y+31.
|
||||
// If main workgroup x starts within a tile and completes it, fixup adds the earlier groups' partial sums from scratch to D.
|
||||
|
||||
// Find the main workgroup whose range contains work.
|
||||
// With q = work_units_per_group and r = groups_with_extra_unit, the first r workgroups get q + 1 units and all others get q.
|
||||
// group(work) = work / (q + 1) when work < r*(q + 1), otherwise r + (work - r*(q + 1))/q.
|
||||
uint group_for_work(uint work) {
|
||||
const uint large_group_work = p.work_units_per_group + 1;
|
||||
const uint large_work = large_group_work * p.groups_with_extra_unit;
|
||||
if (work < large_work) {
|
||||
return fastdiv(work, p.large_group_work_mp, fastdiv_L(p.div_Ls1, 1));
|
||||
}
|
||||
return p.groups_with_extra_unit + fastdiv(work - large_work, p.work_units_mp, fastdiv_L(p.div_Ls1, 0));
|
||||
}
|
||||
|
||||
void main() {
|
||||
const uint group_idx = gl_WorkGroupID.x;
|
||||
const uint work_unit = group_idx * p.work_units_per_group + min(group_idx, p.groups_with_extra_unit);
|
||||
const uint work_unit_end = work_unit + p.work_units_per_group + uint(group_idx < p.groups_with_extra_unit);
|
||||
|
||||
const uint tile_idx = fastdiv(work_unit, p.k_units_mp, fastdiv_L(p.div_Ls0, 0));
|
||||
const uint tile_work = tile_idx * p.k_units_per_tile;
|
||||
// Only a range that starts within a tile and completes it needs fixup.
|
||||
if (work_unit == tile_work || work_unit_end < tile_work + p.k_units_per_tile) {
|
||||
return;
|
||||
}
|
||||
|
||||
const uint first_group = group_for_work(tile_work);
|
||||
const uint outer_tile = fastdiv(tile_idx, p.blocks_m_mp, fastdiv_L(p.div_Ls0, 1));
|
||||
const uint ir = tile_idx - outer_tile * p.blocks_m;
|
||||
const uint batch_idx = fastdiv(outer_tile, p.blocks_n_mp, fastdiv_L(p.div_Ls0, 2));
|
||||
const uint ic = outer_tile - batch_idx * p.blocks_n;
|
||||
const uint row_base = gl_WorkGroupID.y * RESOLVE_ROWS;
|
||||
|
||||
for (uint stripe_elem = gl_LocalInvocationIndex * 4; stripe_elem < RESOLVE_ROWS * p.BN; stripe_elem += gl_WorkGroupSize.x * 4) {
|
||||
const uint r = row_base + (stripe_elem & (RESOLVE_ROWS - 1));
|
||||
const uint c = stripe_elem / RESOLVE_ROWS;
|
||||
const uint elem_idx = c * p.BM + r;
|
||||
const uint dst_idx = batch_idx * p.M * p.N + (ic * p.BN + c) * p.M + ir * p.BM + r;
|
||||
|
||||
// D contains the completing segment. Add the earlier segments from scratch.
|
||||
vec4 result = data_d4[dst_idx / 4];
|
||||
for (uint previous_group = first_group; previous_group < group_idx; ++previous_group) {
|
||||
result += data_s4[(previous_group * p.BM * p.BN + elem_idx) / 4];
|
||||
}
|
||||
data_d4[dst_idx / 4] = result;
|
||||
}
|
||||
}
|
||||
@@ -38,6 +38,7 @@ layout (constant_id = 12) const uint MmTypeA = 0;
|
||||
|
||||
#include "types.glsl"
|
||||
#include "dot_product_funcs.glsl"
|
||||
#include "utils.glsl"
|
||||
|
||||
#ifndef MULMAT_QUANT
|
||||
#ifndef LOAD_VEC_A
|
||||
@@ -121,6 +122,9 @@ layout (binding = 0) readonly buffer BUF_Q5_K_P32 { block_q5_K_packed32 data[]
|
||||
layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
|
||||
layout (binding = 1) readonly buffer B_SCALAR {B_TYPE_SCALAR data_b_scalar[];};
|
||||
layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
|
||||
#ifndef MUL_MAT_ID
|
||||
layout (binding = 3) writeonly buffer S {D_TYPE data_s[];};
|
||||
#endif
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
layout (binding = 3) readonly buffer IDS {int data_ids[];};
|
||||
@@ -155,6 +159,18 @@ layout (push_constant) uniform parameter
|
||||
uint ne12;
|
||||
uint broadcast2;
|
||||
uint broadcast3;
|
||||
uint padded_N;
|
||||
uint k_units_per_tile;
|
||||
uint work_units_per_group;
|
||||
uint groups_with_extra_unit;
|
||||
uint k_units_mp;
|
||||
uint blocks_m_mp;
|
||||
uint blocks_n_mp;
|
||||
uint ne12_mp;
|
||||
uint broadcast2_mp;
|
||||
uint broadcast3_mp;
|
||||
uint div_Ls0;
|
||||
uint div_Ls1;
|
||||
#endif
|
||||
} p;
|
||||
|
||||
@@ -189,6 +205,9 @@ layout(constant_id = 13) const bool APPLY_SLM_A_RESHAPE = false;
|
||||
const uint SHMEM_STRIDE_PAD = 1;
|
||||
const bool APPLY_SLM_A_RESHAPE = false;
|
||||
#endif
|
||||
#ifndef MUL_MAT_ID
|
||||
layout(constant_id = 15) const bool stream_k = false;
|
||||
#endif
|
||||
#define SHMEM_STRIDE (BK / 2 + SHMEM_STRIDE_PAD)
|
||||
|
||||
shared FLOAT_TYPEV2 buf_a[BM * SHMEM_STRIDE];
|
||||
@@ -218,27 +237,24 @@ void main() {
|
||||
_ne1 = data_row_map[2 + 2 * tile_idx];
|
||||
const uint ic = 0;
|
||||
#else
|
||||
const uint ic = gl_WorkGroupID.y;
|
||||
const uint group_idx = gl_WorkGroupID.x;
|
||||
uint stream_k_work_unit;
|
||||
uint stream_k_work_unit_end;
|
||||
|
||||
if (stream_k) {
|
||||
stream_k_work_unit = group_idx * p.work_units_per_group + min(group_idx, p.groups_with_extra_unit);
|
||||
stream_k_work_unit_end = stream_k_work_unit + p.work_units_per_group + uint(group_idx < p.groups_with_extra_unit);
|
||||
}
|
||||
#endif
|
||||
#if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT)
|
||||
init_iq_shmem(gl_WorkGroupSize);
|
||||
#endif
|
||||
|
||||
#ifndef MUL_MAT_ID
|
||||
const uint batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
|
||||
|
||||
const uint i13 = batch_idx / p.ne12;
|
||||
const uint i12 = batch_idx % p.ne12;
|
||||
|
||||
const uint i03 = i13 / p.broadcast3;
|
||||
const uint i02 = i12 / p.broadcast2;
|
||||
|
||||
const uint batch_idx_a = i03 * p.ne02 + i02;
|
||||
#endif
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
const uint blocks_m = (p.M + BM - 1) / BM;
|
||||
const uint ir = gl_WorkGroupID.x % blocks_m;
|
||||
const uint ik = gl_WorkGroupID.x / blocks_m;
|
||||
#endif
|
||||
|
||||
const uint WNITER = (WM * WN) / (WARP * TM * TN * WMITER);
|
||||
const uint WSUBM = WM / WMITER;
|
||||
@@ -288,6 +304,41 @@ void main() {
|
||||
const uint loadstride_a = gl_WorkGroupSize.x * LOAD_VEC_A_EFF * LOAD_VEC_BATCH_A / BK;
|
||||
const uint loadstride_b = gl_WorkGroupSize.x * LOAD_VEC_B_EFF * LOAD_VEC_BATCH_B / BK;
|
||||
|
||||
#ifndef MUL_MAT_ID
|
||||
do {
|
||||
uint stream_k_tile_k_unit;
|
||||
uint stream_k_tile_k_units;
|
||||
uint ir;
|
||||
uint ic;
|
||||
uint batch_idx;
|
||||
uint ik;
|
||||
|
||||
if (stream_k) {
|
||||
const uint tile_idx = fastdiv(stream_k_work_unit, p.k_units_mp, fastdiv_L(p.div_Ls0, 0));
|
||||
stream_k_tile_k_unit = stream_k_work_unit - tile_idx * p.k_units_per_tile;
|
||||
stream_k_tile_k_units = min(stream_k_work_unit_end - stream_k_work_unit, p.k_units_per_tile - stream_k_tile_k_unit);
|
||||
|
||||
const uint blocks_m = p.M / BM;
|
||||
const uint blocks_n = p.N / BN;
|
||||
const uint outer_tile = fastdiv(tile_idx, p.blocks_m_mp, fastdiv_L(p.div_Ls0, 1));
|
||||
ir = tile_idx - outer_tile * blocks_m;
|
||||
batch_idx = fastdiv(outer_tile, p.blocks_n_mp, fastdiv_L(p.div_Ls0, 2));
|
||||
ic = outer_tile - batch_idx * blocks_n;
|
||||
} else {
|
||||
const uint blocks_m = (p.M + BM - 1) / BM;
|
||||
ic = gl_WorkGroupID.y;
|
||||
batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
|
||||
ir = group_idx % blocks_m;
|
||||
ik = group_idx / blocks_m;
|
||||
}
|
||||
|
||||
const uint i13 = fastdiv(batch_idx, p.ne12_mp, fastdiv_L(p.div_Ls1, 0));
|
||||
const uint i12 = batch_idx - i13 * p.ne12;
|
||||
const uint i03 = fastdiv(i13, p.broadcast3_mp, fastdiv_L(p.div_Ls1, 2));
|
||||
const uint i02 = fastdiv(i12, p.broadcast2_mp, fastdiv_L(p.div_Ls1, 1));
|
||||
const uint batch_idx_a = i03 * p.ne02 + i02;
|
||||
#endif
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
load_row_ids(row_begin);
|
||||
uint required_work_items = _ne1 * BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B;
|
||||
@@ -298,8 +349,15 @@ void main() {
|
||||
const uint start_k = 0;
|
||||
const uint end_k = p.K;
|
||||
#else
|
||||
const uint start_k = ik * p.k_split;
|
||||
const uint end_k = min(p.K, (ik + 1) * p.k_split);
|
||||
uint start_k;
|
||||
uint end_k;
|
||||
if (stream_k) {
|
||||
start_k = stream_k_tile_k_unit * 256;
|
||||
end_k = (stream_k_tile_k_unit + stream_k_tile_k_units) * 256;
|
||||
} else {
|
||||
start_k = ik * p.k_split;
|
||||
end_k = min(p.K, (ik + 1) * p.k_split);
|
||||
}
|
||||
#endif
|
||||
|
||||
uint pos_a =
|
||||
@@ -438,7 +496,17 @@ void main() {
|
||||
const uint dc = ic * BN + warp_c * WN;
|
||||
|
||||
#ifndef MUL_MAT_ID
|
||||
const uint offsets = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches;
|
||||
uint offsets = batch_idx * p.batch_stride_d;
|
||||
if (!stream_k) {
|
||||
offsets += ik * p.batch_stride_d * p.num_batches;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef MUL_MAT_ID
|
||||
bool stream_k_store_scratch;
|
||||
if (stream_k) {
|
||||
stream_k_store_scratch = stream_k_tile_k_unit + stream_k_tile_k_units != p.k_units_per_tile;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef COOPMAT
|
||||
@@ -462,36 +530,44 @@ void main() {
|
||||
}
|
||||
}
|
||||
#else
|
||||
const bool is_aligned = p.stride_d % 4 == 0; // Assumption: D_TYPE == float
|
||||
|
||||
[[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {
|
||||
[[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) {
|
||||
const bool is_in_bounds = dr + (cm_row + 1) * TM <= p.M && dc + (cm_col + 1) * TN <= p.N;
|
||||
|
||||
if (is_aligned && is_in_bounds) {
|
||||
// Full coopMat is within bounds and stride_d is aligned with 16B
|
||||
coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> cm_dtype = coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(sums[cm_col * cms_per_row + cm_row]);
|
||||
coopMatStore(cm_dtype, data_d, offsets + (dc + cm_col * TN) * p.stride_d + dr + cm_row * TM, p.stride_d, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
} else if (is_in_bounds) {
|
||||
// Full coopMat is within bounds, but stride_d is not aligned
|
||||
coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
[[unroll]] for (uint col = 0; col < TN; col += storestride) {
|
||||
data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
|
||||
coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> cm_dtype = coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(sums[cm_col * cms_per_row + cm_row]);
|
||||
if (stream_k) {
|
||||
if (stream_k_store_scratch) {
|
||||
const uint scratch_offset = group_idx * BM * BN + (warp_c * WN + cm_col * TN) * BM + warp_r * WM + cm_row * TM;
|
||||
coopMatStore(cm_dtype, data_s, scratch_offset, BM, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
} else {
|
||||
coopMatStore(cm_dtype, data_d, offsets + (dc + cm_col * TN) * p.stride_d + dr + cm_row * TM, p.stride_d, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
}
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
} else if (dr + cm_row * TM < p.M && dc + cm_col * TN < p.N) {
|
||||
// Partial coopMat is within bounds
|
||||
coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
} else {
|
||||
const bool is_aligned = p.stride_d % 4 == 0; // Assumption: D_TYPE == float
|
||||
const bool is_in_bounds = dr + (cm_row + 1) * TM <= p.M && dc + (cm_col + 1) * TN <= p.N;
|
||||
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
[[unroll]] for (uint col = 0; col < TN; col += storestride) {
|
||||
if (dr + cm_row * TM + store_r < p.M && dc + cm_col * TN + col + store_c < p.N) {
|
||||
if (is_aligned && is_in_bounds) {
|
||||
// Full coopMat is within bounds and stride_d is aligned with 16B
|
||||
coopMatStore(cm_dtype, data_d, offsets + (dc + cm_col * TN) * p.stride_d + dr + cm_row * TM, p.stride_d, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
} else if (is_in_bounds) {
|
||||
// Full coopMat is within bounds, but stride_d is not aligned
|
||||
coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
[[unroll]] for (uint col = 0; col < TN; col += storestride) {
|
||||
data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
|
||||
}
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
} else if (dr + cm_row * TM < p.M && dc + cm_col * TN < p.N) {
|
||||
// Partial coopMat is within bounds
|
||||
coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
[[unroll]] for (uint col = 0; col < TN; col += storestride) {
|
||||
if (dr + cm_row * TM + store_r < p.M && dc + cm_col * TN + col + store_c < p.N) {
|
||||
data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
|
||||
}
|
||||
}
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
}
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -519,11 +595,22 @@ void main() {
|
||||
data_d[row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
|
||||
}
|
||||
#else
|
||||
if (dr_warp + 2 * cr < p.M && dc_warp + cc < p.N) {
|
||||
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr] = D_TYPE(sums[sums_idx].x);
|
||||
}
|
||||
if (dr_warp + 2 * cr + 1 < p.M && dc_warp + cc < p.N) {
|
||||
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
|
||||
if (stream_k) {
|
||||
const uint scratch_offset = group_idx * BM * BN + (dc_warp + cc - ic * BN) * BM + dr_warp + 2 * cr - ir * BM;
|
||||
if (stream_k_store_scratch) {
|
||||
data_s[scratch_offset] = D_TYPE(sums[sums_idx].x);
|
||||
data_s[scratch_offset + 1] = D_TYPE(sums[sums_idx].y);
|
||||
} else {
|
||||
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr] = D_TYPE(sums[sums_idx].x);
|
||||
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
|
||||
}
|
||||
} else {
|
||||
if (dr_warp + 2 * cr < p.M && dc_warp + cc < p.N) {
|
||||
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr] = D_TYPE(sums[sums_idx].x);
|
||||
}
|
||||
if (dr_warp + 2 * cr + 1 < p.M && dc_warp + cc < p.N) {
|
||||
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
|
||||
}
|
||||
}
|
||||
#endif // MUL_MAT_ID
|
||||
}
|
||||
@@ -531,4 +618,10 @@ void main() {
|
||||
}
|
||||
}
|
||||
#endif // COOPMAT
|
||||
#ifndef MUL_MAT_ID
|
||||
if (stream_k) {
|
||||
stream_k_work_unit += stream_k_tile_k_units;
|
||||
}
|
||||
} while (stream_k && stream_k_work_unit < stream_k_work_unit_end);
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -61,6 +61,10 @@ uint mm_quant_k() {
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef MUL_MAT_ID
|
||||
layout (constant_id = 15) const bool stream_k = false;
|
||||
#endif
|
||||
|
||||
layout (push_constant) uniform parameter
|
||||
{
|
||||
uint M;
|
||||
@@ -91,6 +95,17 @@ layout (push_constant) uniform parameter
|
||||
uint broadcast3;
|
||||
// N dimension for the B matrix can be >= p.N
|
||||
uint padded_N;
|
||||
uint k_units_per_tile;
|
||||
uint work_units_per_group;
|
||||
uint groups_with_extra_unit;
|
||||
uint k_units_mp;
|
||||
uint blocks_m_mp;
|
||||
uint blocks_n_mp;
|
||||
uint ne12_mp;
|
||||
uint broadcast2_mp;
|
||||
uint broadcast3_mp;
|
||||
uint div_Ls0;
|
||||
uint div_Ls1;
|
||||
#endif
|
||||
} p;
|
||||
|
||||
@@ -102,6 +117,9 @@ layout (binding = 0) readonly buffer A {uint8_t data_a[];};
|
||||
#endif
|
||||
layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
|
||||
layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
|
||||
#ifndef MUL_MAT_ID
|
||||
layout (binding = 3) writeonly buffer S {D_TYPE data_s[];};
|
||||
#endif
|
||||
#if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)
|
||||
layout (binding = 1) readonly buffer B4 {B_TYPEV4 data_b_v4[];};
|
||||
#endif
|
||||
@@ -261,6 +279,17 @@ void main() {
|
||||
const uint row_begin = tile_idx * BN;
|
||||
_ne1 = data_row_map[2 + 2 * tile_idx];
|
||||
const uint ic = 0;
|
||||
#else
|
||||
const uint group_idx = gl_WorkGroupID.x;
|
||||
uint stream_k_work_unit;
|
||||
uint stream_k_work_unit_end;
|
||||
|
||||
if (stream_k) {
|
||||
// A work unit covers 256 consecutive K elements of one output tile.
|
||||
// Work units are flattened as [K unit, M tile, N tile, batch].
|
||||
stream_k_work_unit = group_idx * p.work_units_per_group + min(group_idx, p.groups_with_extra_unit);
|
||||
stream_k_work_unit_end = stream_k_work_unit + p.work_units_per_group + uint(group_idx < p.groups_with_extra_unit);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
@@ -278,43 +307,61 @@ void main() {
|
||||
#endif
|
||||
|
||||
#ifndef MUL_MAT_ID
|
||||
const uint batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
|
||||
do {
|
||||
uint stream_k_tile_k_unit;
|
||||
uint stream_k_tile_k_units;
|
||||
uint ir;
|
||||
uint ic;
|
||||
uint batch_idx;
|
||||
uint ik;
|
||||
|
||||
const uint i13 = batch_idx / p.ne12;
|
||||
const uint i12 = batch_idx % p.ne12;
|
||||
if (stream_k) {
|
||||
// The 256-element granularity does not split any supported quant block.
|
||||
const uint tile_idx = fastdiv(stream_k_work_unit, p.k_units_mp, fastdiv_L(p.div_Ls0, 0));
|
||||
stream_k_tile_k_unit = stream_k_work_unit - tile_idx * p.k_units_per_tile;
|
||||
stream_k_tile_k_units = min(stream_k_work_unit_end - stream_k_work_unit, p.k_units_per_tile - stream_k_tile_k_unit);
|
||||
|
||||
const uint i03 = i13 / p.broadcast3;
|
||||
const uint i02 = i12 / p.broadcast2;
|
||||
|
||||
const uint batch_idx_a = i03 * p.ne02 + i02;
|
||||
const uint blocks_m = p.M / BM;
|
||||
const uint blocks_n = p.N / BN;
|
||||
const uint outer_tile = fastdiv(tile_idx, p.blocks_m_mp, fastdiv_L(p.div_Ls0, 1));
|
||||
ir = tile_idx - outer_tile * blocks_m;
|
||||
batch_idx = fastdiv(outer_tile, p.blocks_n_mp, fastdiv_L(p.div_Ls0, 2));
|
||||
ic = outer_tile - batch_idx * blocks_n;
|
||||
} else {
|
||||
const uint blocks_m = (p.M + BM - 1) / BM;
|
||||
// L2-friendly workgroup scheduling
|
||||
const uint blocks_n = (p.N + BN - 1) / BN;
|
||||
#if defined(MULMAT_QUANT)
|
||||
const uint a_panel_bytes = (BM * p.K / mm_quant_k()) * MmABlockBytes;
|
||||
#elif defined(DATA_A_F32)
|
||||
const uint a_panel_bytes = BM * p.K * 4;
|
||||
#elif QUANT_K == 1
|
||||
const uint a_panel_bytes = BM * p.K * 2;
|
||||
#else
|
||||
const uint a_panel_bytes = (BM * p.K) / 2 + (BM * p.K) / 16;
|
||||
#endif
|
||||
const uint group_m = clamp(GROUP_A_BUDGET / max(a_panel_bytes, 1u), 1u, min(blocks_m, 32u));
|
||||
const uint tiles_per_group = group_m * blocks_n;
|
||||
const uint lin = gl_WorkGroupID.y * blocks_m + (gl_WorkGroupID.x % blocks_m);
|
||||
const uint group_id = lin / tiles_per_group;
|
||||
const uint first_m = group_id * group_m;
|
||||
const uint gsize = min(blocks_m - first_m, group_m);
|
||||
const uint in_group = lin - group_id * tiles_per_group;
|
||||
ir = first_m + in_group % gsize;
|
||||
ic = in_group / gsize;
|
||||
batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
|
||||
ik = group_idx / blocks_m;
|
||||
}
|
||||
|
||||
const uint i13 = fastdiv(batch_idx, p.ne12_mp, fastdiv_L(p.div_Ls1, 0));
|
||||
const uint i12 = batch_idx - i13 * p.ne12;
|
||||
const uint i03 = fastdiv(i13, p.broadcast3_mp, fastdiv_L(p.div_Ls1, 2));
|
||||
const uint i02 = fastdiv(i12, p.broadcast2_mp, fastdiv_L(p.div_Ls1, 1));
|
||||
const uint batch_idx_a = i03 * p.ne02 + i02;
|
||||
#else
|
||||
const uint blocks_m = (p.M + BM - 1) / BM;
|
||||
const uint ik = gl_WorkGroupID.x / blocks_m;
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
const uint ir = gl_WorkGroupID.x % blocks_m;
|
||||
#else
|
||||
// L2-friendly workgroup scheduling
|
||||
const uint blocks_n = (p.N + BN - 1) / BN;
|
||||
#if defined(MULMAT_QUANT)
|
||||
const uint a_panel_bytes = (BM * p.K / mm_quant_k()) * MmABlockBytes;
|
||||
#elif defined(DATA_A_F32)
|
||||
const uint a_panel_bytes = BM * p.K * 4;
|
||||
#elif QUANT_K == 1
|
||||
const uint a_panel_bytes = BM * p.K * 2;
|
||||
#else
|
||||
const uint a_panel_bytes = (BM * p.K) / 2 + (BM * p.K) / 16;
|
||||
#endif
|
||||
const uint group_m = clamp(GROUP_A_BUDGET / max(a_panel_bytes, 1u), 1u, min(blocks_m, 32u));
|
||||
const uint tiles_per_group = group_m * blocks_n;
|
||||
const uint lin = gl_WorkGroupID.y * blocks_m + (gl_WorkGroupID.x % blocks_m);
|
||||
const uint group_id = lin / tiles_per_group;
|
||||
const uint first_m = group_id * group_m;
|
||||
const uint gsize = min(blocks_m - first_m, group_m);
|
||||
const uint in_group = lin - group_id * tiles_per_group;
|
||||
const uint ir = first_m + in_group % gsize;
|
||||
const uint ic = in_group / gsize;
|
||||
#endif
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
@@ -325,8 +372,15 @@ void main() {
|
||||
uint start_k = 0;
|
||||
const uint end_k = p.K;
|
||||
#else
|
||||
uint start_k = ik * p.k_split;
|
||||
const uint end_k = min(p.K, (ik + 1) * p.k_split);
|
||||
uint start_k;
|
||||
uint end_k;
|
||||
if (stream_k) {
|
||||
start_k = stream_k_tile_k_unit * 256;
|
||||
end_k = (stream_k_tile_k_unit + stream_k_tile_k_units) * 256;
|
||||
} else {
|
||||
start_k = ik * p.k_split;
|
||||
end_k = min(p.K, (ik + 1) * p.k_split);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef MULMAT_QUANT
|
||||
@@ -341,7 +395,10 @@ void main() {
|
||||
#else
|
||||
uint pos_a = batch_idx_a * (p.batch_stride_a / qk);
|
||||
uint pos_b = batch_idx * p.batch_stride_b;
|
||||
uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches;
|
||||
uint pos_d = batch_idx * p.batch_stride_d;
|
||||
if (!stream_k) {
|
||||
pos_d += ik * p.batch_stride_d * p.num_batches;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef MULMAT_QUANT
|
||||
@@ -404,12 +461,15 @@ void main() {
|
||||
const uint UNROLL_K = (qk == 256) ? 256 : (BK * 8);
|
||||
const uint unroll_count = UNROLL_K / BK;
|
||||
|
||||
// Detect a fast path where all loads are entirely in bounds and no clamping is required
|
||||
if ((ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % START_ALIGN_K) == 0 && (end_k % BK) == 0 &&
|
||||
bool fast_path = stream_k;
|
||||
if (!stream_k) {
|
||||
fast_path = (ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % START_ALIGN_K) == 0 && (end_k % BK) == 0 &&
|
||||
#if !defined(MULMAT_QUANT) && QUANT_K == 1
|
||||
(stride_a % 8) == 0 &&
|
||||
(stride_a % 8) == 0 &&
|
||||
#endif
|
||||
(stride_b % 8) == 0) {
|
||||
(stride_b % 8) == 0;
|
||||
}
|
||||
if (fast_path) {
|
||||
// Hint to the compiler that values are aligned (want 16B alignment)
|
||||
start_k &= ~(START_ALIGN_K-1);
|
||||
stride_b &= ~7;
|
||||
@@ -427,7 +487,7 @@ void main() {
|
||||
// The fetches and stores are pipelined to hide the latency.
|
||||
fetch_scales(ir * BM, pos_a, stride_a, start_k, tid, true);
|
||||
|
||||
if (enable_smaller_matrices && ic * BN + BNover4 >= p.N) {
|
||||
if (!stream_k && enable_smaller_matrices && ic * BN + BNover4 >= p.N) {
|
||||
coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator> sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator>(0.0);
|
||||
for (uint i = 0; i < k_iters; ++i) {
|
||||
|
||||
@@ -470,7 +530,7 @@ void main() {
|
||||
|
||||
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BNover4, ir * BM, BM), tensorViewTranspose);
|
||||
return;
|
||||
} else if (enable_smaller_matrices && ic * BN + BNover2 >= p.N) {
|
||||
} else if (!stream_k && enable_smaller_matrices && ic * BN + BNover2 >= p.N) {
|
||||
coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator> sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator>(0.0);
|
||||
for (uint i = 0; i < k_iters; ++i) {
|
||||
|
||||
@@ -555,8 +615,18 @@ void main() {
|
||||
|
||||
coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator> mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator>(sum);
|
||||
|
||||
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
|
||||
return;
|
||||
if (stream_k) {
|
||||
if (stream_k_tile_k_unit + stream_k_tile_k_units != p.k_units_per_tile) {
|
||||
tensorLayoutNV<2> tensorLayoutS = createTensorLayoutNV(2);
|
||||
tensorLayoutS = setTensorLayoutDimensionNV(tensorLayoutS, BN, BM);
|
||||
tensorLayoutS = setTensorLayoutStrideNV(tensorLayoutS, BM, 1);
|
||||
coopMatStoreTensorNV(mat_d, data_s, group_idx * BM * BN, tensorLayoutS, tensorViewTranspose);
|
||||
} else {
|
||||
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
|
||||
}
|
||||
} else {
|
||||
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
|
||||
}
|
||||
}
|
||||
} else
|
||||
#endif // !defined(MUL_MAT_ID)
|
||||
@@ -720,4 +790,10 @@ void main() {
|
||||
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
|
||||
#endif
|
||||
}
|
||||
#ifndef MUL_MAT_ID
|
||||
if (stream_k) {
|
||||
stream_k_work_unit += stream_k_tile_k_units;
|
||||
}
|
||||
} while (stream_k && stream_k_work_unit < stream_k_work_unit_end);
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -82,6 +82,8 @@ layout (constant_id = 7) const uint TM = 4;
|
||||
layout (constant_id = 8) const uint TN = 2;
|
||||
layout (constant_id = 9) const uint TK = 1; // Only needed for coopmat
|
||||
layout (constant_id = 10) const uint WARP = 32;
|
||||
// Use the mixed signedness dot product, for devices that accelerate only that one
|
||||
layout (constant_id = 11) const bool MMQ_SUDOT = false;
|
||||
|
||||
#define BK 32
|
||||
|
||||
@@ -102,6 +104,13 @@ shared block_b_cache buf_b[BN * BK_STEP];
|
||||
block_a_cache cache_a[WMITER * TM];
|
||||
block_b_cache cache_b;
|
||||
|
||||
#if defined(DATA_A_Q8_0) || defined(DATA_A_MXFP4) || defined(DATA_A_IQ4_XS) || defined(DATA_A_IQ3_S) || defined(DATA_A_Q3_K) || defined(DATA_A_Q6_K)
|
||||
// These quants are signed. MMQ_SUDOT gives them a bias and uses the sum of each half of the q8_1 block to remove it.
|
||||
#define MMQ_B_SUMS
|
||||
shared int32_t buf_b_qsum[MMQ_SUDOT ? BN * BK_STEP * 2 : 1];
|
||||
int32_t cache_b_qsum[2];
|
||||
#endif
|
||||
|
||||
#define LOAD_VEC_A (4 * QUANT_R_MMQ)
|
||||
#define LOAD_VEC_B 16
|
||||
|
||||
|
||||
@@ -4,6 +4,19 @@
|
||||
|
||||
#include "types.glsl"
|
||||
|
||||
int32_t mmq_dot(const int32_t a, const int32_t b) {
|
||||
// With MMQ_SUDOT the A quants must be unsigned
|
||||
if (MMQ_SUDOT) {
|
||||
return dotPacked4x8EXT(uint32_t(a), b);
|
||||
}
|
||||
return dotPacked4x8EXT(a, b);
|
||||
}
|
||||
|
||||
// With MMQ_SUDOT, flip the sign bit of each byte. This adds a bias of 128 to signed quants.
|
||||
int32_t mmq_a_store(const int32_t v) {
|
||||
return MMQ_SUDOT ? v ^ int32_t(0x80808080) : v;
|
||||
}
|
||||
|
||||
// Each iqs value maps to a 32-bit integer
|
||||
|
||||
#if defined(DATA_A_Q2_0)
|
||||
@@ -41,7 +54,7 @@ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
||||
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
int32_t q_sum = 0;
|
||||
[[unroll]] for (uint iqs = 0; iqs < 8; ++iqs) {
|
||||
q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(cache_a[ib_a].dm) * (float(q_sum) * float(cache_b.ds.x) - float(cache_b.ds.y)));
|
||||
@@ -102,8 +115,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
const int32_t qs_b0 = cache_b.qs[iqs];
|
||||
const int32_t qs_b1 = cache_b.qs[iqs + 4];
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a.x, qs_b0);
|
||||
q_sum += dotPacked4x8EXT(qs_a.y, qs_b1);
|
||||
q_sum += mmq_dot(qs_a.x, qs_b0);
|
||||
q_sum += mmq_dot(qs_a.y, qs_b1);
|
||||
}
|
||||
|
||||
#ifdef DATA_A_Q4_0
|
||||
@@ -176,8 +189,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
const int32_t qs_b0 = cache_b.qs[iqs];
|
||||
const int32_t qs_b1 = cache_b.qs[iqs + 4];
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a0, qs_b0);
|
||||
q_sum += dotPacked4x8EXT(qs_a1, qs_b1);
|
||||
q_sum += mmq_dot(qs_a0, qs_b0);
|
||||
q_sum += mmq_dot(qs_a1, qs_b1);
|
||||
}
|
||||
|
||||
#ifdef DATA_A_Q5_0
|
||||
@@ -192,8 +205,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
// 2-byte loads for Q8_0 blocks (34 bytes)
|
||||
void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) {
|
||||
if (is_in_bounds) {
|
||||
buf_a[buf_ib].qs[iqs] = pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2],
|
||||
data_a_packed16[ib].qs[iqs * 2 + 1]));
|
||||
buf_a[buf_ib].qs[iqs] = mmq_a_store(pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2],
|
||||
data_a_packed16[ib].qs[iqs * 2 + 1])));
|
||||
|
||||
if (iqs == 0) {
|
||||
buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d);
|
||||
@@ -221,7 +234,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
||||
const int32_t qs_b = cache_b.qs[iqs];
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, qs_b);
|
||||
q_sum += mmq_dot(qs_a, qs_b);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(q_sum) * float(cache_a[ib_a].dm) * float(cache_b.ds.x));
|
||||
@@ -240,8 +256,8 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
|
||||
const u8vec4 i_a0 = unpack8( qs & 0x0F0F0F0F);
|
||||
const u8vec4 i_a1 = unpack8((qs >> 4) & 0x0F0F0F0F);
|
||||
|
||||
buf_a[buf_ib].qs[iqs ] = pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w]));
|
||||
buf_a[buf_ib].qs[iqs + 4] = pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w]));
|
||||
buf_a[buf_ib].qs[iqs ] = mmq_a_store(pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w])));
|
||||
buf_a[buf_ib].qs[iqs + 4] = mmq_a_store(pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w])));
|
||||
|
||||
if (iqs == 0) {
|
||||
buf_a[buf_ib].d = FLOAT_TYPE(e8m0_to_fp32(data_a[ib].e) * 0.5);
|
||||
@@ -269,7 +285,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
|
||||
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
|
||||
@@ -284,8 +303,8 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
|
||||
const uint32_t vui = data_a_packed32[ib_k].qs[4 * ib32 + iqs];
|
||||
const i32vec2 qs = iq4nl_to_i8x8(vui);
|
||||
|
||||
buf_a[buf_ib].qs[iqs ] = qs.x;
|
||||
buf_a[buf_ib].qs[iqs + 4] = qs.y;
|
||||
buf_a[buf_ib].qs[iqs ] = mmq_a_store(qs.x);
|
||||
buf_a[buf_ib].qs[iqs + 4] = mmq_a_store(qs.y);
|
||||
|
||||
if (iqs == 0) {
|
||||
const uint sl = (data_a_packed32[ib_k].scales_l >> (4 * ib32)) & 0xF;
|
||||
@@ -313,7 +332,10 @@ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
||||
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
int32_t q_sum = 0;
|
||||
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
|
||||
q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
|
||||
@@ -372,8 +394,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
const int32_t scale_m = int32_t(scale >> 4) * 0x01010101; // Duplicate 8-bit value across 32-bits.
|
||||
const int32_t qs_a = int32_t((cache_a[ib_a].qs[iqs / 4] >> ((iqs % 4) * 2)) & 0x03030303);
|
||||
|
||||
sum_d += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]) * (scale & 0xF);
|
||||
sum_m += dotPacked4x8EXT(scale_m, cache_b.qs[iqs]);
|
||||
sum_d += mmq_dot(qs_a, cache_b.qs[iqs]) * (scale & 0xF);
|
||||
sum_m += mmq_dot(scale_m, cache_b.qs[iqs]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(cache_b.ds.x) * (float(cache_a[ib_a].dm.x) * float(sum_d) - float(cache_a[ib_a].dm.y) * float(sum_m)));
|
||||
@@ -435,18 +457,26 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
int32_t q_sum = 0;
|
||||
|
||||
[[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
|
||||
// Subtract 4 from the quants to correct the 3rd bit offset
|
||||
const int32_t qs_a = pack32(unpack8(int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F)) - int8_t(4));
|
||||
// Subtract 4 from the quants to correct the 3rd bit offset. MMQ_SUDOT keeps them unsigned and subtracts 4 times the B sum.
|
||||
const int32_t qs_u = int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F);
|
||||
const int32_t qs_a = MMQ_SUDOT ? qs_u : pack32(unpack8(qs_u) - int8_t(4));
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 4 * cache_b_qsum[0];
|
||||
}
|
||||
result += float(cache_a[ib_a].d_scales[0]) * float(q_sum);
|
||||
q_sum = 0;
|
||||
|
||||
[[unroll]] for (uint iqs = 4; iqs < 8; iqs++) {
|
||||
const int32_t qs_a = pack32(unpack8(int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F)) - int8_t(4));
|
||||
const int32_t qs_u = int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F);
|
||||
const int32_t qs_a = MMQ_SUDOT ? qs_u : pack32(unpack8(qs_u) - int8_t(4));
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 4 * cache_b_qsum[1];
|
||||
}
|
||||
result += float(cache_a[ib_a].d_scales[1]) * float(q_sum);
|
||||
|
||||
@@ -518,7 +548,7 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
||||
#endif
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(cache_b.ds.x) * float(cache_a[ib_a].dm.x) * float(q_sum) - float(cache_a[ib_a].dm.y) * float(cache_b.ds.y));
|
||||
@@ -539,10 +569,11 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
|
||||
const uint qh_shift = ((iqs_k % 32) / 8) * 2;
|
||||
|
||||
const i8vec2 vals00 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 ] >> ql_shift) & uint16_t(0x0F0F))).xy |
|
||||
unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 ] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32);
|
||||
unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 ] >> qh_shift) & uint16_t(0x0303)) << 4)).xy);
|
||||
const i8vec2 vals01 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 + 1] >> ql_shift) & uint16_t(0x0F0F))).xy |
|
||||
unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 + 1] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32);
|
||||
buf_a[buf_ib].qs[iqs] = pack32(i8vec4(vals00.x, vals00.y, vals01.x, vals01.y));
|
||||
unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 + 1] >> qh_shift) & uint16_t(0x0303)) << 4)).xy);
|
||||
// MMQ_SUDOT keeps the quants unsigned and subtracts 32 times the B sum
|
||||
buf_a[buf_ib].qs[iqs] = MMQ_SUDOT ? pack32(i8vec4(vals00, vals01)) : pack32(i8vec4(vals00 - int8_t(32), vals01 - int8_t(32)));
|
||||
|
||||
if (iqs == 0) {
|
||||
const uint is = iqs_k / 4;
|
||||
@@ -574,7 +605,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
[[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
|
||||
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 32 * cache_b_qsum[0];
|
||||
}
|
||||
result += float(cache_a[ib_a].d_scales[0]) * float(q_sum);
|
||||
q_sum = 0;
|
||||
@@ -582,7 +616,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
[[unroll]] for (uint iqs = 4; iqs < 8; iqs++) {
|
||||
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 32 * cache_b_qsum[1];
|
||||
}
|
||||
result += float(cache_a[ib_a].d_scales[1]) * float(q_sum);
|
||||
|
||||
@@ -612,8 +649,8 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
|
||||
const ivec4 m0 = -(ivec4(signs, signs >> 1, signs >> 2, signs >> 3) & 1);
|
||||
const ivec4 m1 = -(ivec4(signs >> 4, signs >> 5, signs >> 6, signs >> 7) & 1);
|
||||
|
||||
buf_a[buf_ib].qs[2 * iqs ] = pack32(i8vec4((vals0 ^ m0) - m0));
|
||||
buf_a[buf_ib].qs[2 * iqs + 1] = pack32(i8vec4((vals1 ^ m1) - m1));
|
||||
buf_a[buf_ib].qs[2 * iqs ] = mmq_a_store(pack32(i8vec4((vals0 ^ m0) - m0)));
|
||||
buf_a[buf_ib].qs[2 * iqs + 1] = mmq_a_store(pack32(i8vec4((vals1 ^ m1) - m1)));
|
||||
|
||||
if (iqs == 0) {
|
||||
const uint scale = (uint(data_a_packed16[ib_k].scales[ib32 / 4]) >> ((ib32 & 3) * 4)) & 0xF;
|
||||
@@ -641,7 +678,10 @@ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
||||
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
int32_t q_sum = 0;
|
||||
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
|
||||
q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
|
||||
@@ -662,6 +702,12 @@ void block_b_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
|
||||
buf_b[buf_ib].qs[iqs * 4 + 1] = values.y;
|
||||
buf_b[buf_ib].qs[iqs * 4 + 2] = values.z;
|
||||
buf_b[buf_ib].qs[iqs * 4 + 3] = values.w;
|
||||
#ifdef MMQ_B_SUMS
|
||||
if (MMQ_SUDOT) {
|
||||
buf_b_qsum[buf_ib * 2 + iqs] = dotPacked4x8EXT(values.x, 0x01010101u) + dotPacked4x8EXT(values.y, 0x01010101u) +
|
||||
dotPacked4x8EXT(values.z, 0x01010101u) + dotPacked4x8EXT(values.w, 0x01010101u);
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
if (iqs == 0) {
|
||||
buf_b[buf_ib].ds = FLOAT_TYPEV2(0.0f);
|
||||
@@ -671,6 +717,11 @@ void block_b_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
|
||||
buf_b[buf_ib].qs[iqs * 4 + 1] = 0;
|
||||
buf_b[buf_ib].qs[iqs * 4 + 2] = 0;
|
||||
buf_b[buf_ib].qs[iqs * 4 + 3] = 0;
|
||||
#ifdef MMQ_B_SUMS
|
||||
if (MMQ_SUDOT) {
|
||||
buf_b_qsum[buf_ib * 2 + iqs] = 0;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@@ -679,4 +730,10 @@ void block_b_to_registers(const uint ib) {
|
||||
[[unroll]] for (uint iqs = 0; iqs < BK / 4; iqs++) {
|
||||
cache_b.qs[iqs] = buf_b[ib].qs[iqs];
|
||||
}
|
||||
#ifdef MMQ_B_SUMS
|
||||
if (MMQ_SUDOT) {
|
||||
cache_b_qsum[0] = buf_b_qsum[ib * 2];
|
||||
cache_b_qsum[1] = buf_b_qsum[ib * 2 + 1];
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
#version 450
|
||||
|
||||
#extension GL_EXT_shader_realtime_clock : require
|
||||
|
||||
layout(local_size_x = 1, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
// never written, it only anchors the spin below so the compiler cannot drop it
|
||||
layout (binding = 0) writeonly buffer D {uint data_d[];};
|
||||
|
||||
layout (push_constant) uniform parameter {
|
||||
uint ticks;
|
||||
} p;
|
||||
|
||||
// bounds the spin in case the clock does not advance, turning a hang into a too-short sleep
|
||||
const uint MAX_ITER = 1u << 27;
|
||||
|
||||
void main() {
|
||||
const uvec2 t0 = clockRealtime2x32EXT();
|
||||
|
||||
uvec2 t = t0;
|
||||
|
||||
for (uint iter = 0; iter < MAX_ITER; ++iter) {
|
||||
t = clockRealtime2x32EXT();
|
||||
|
||||
const uint lo = t.x - t0.x;
|
||||
const uint hi = t.y - t0.y - (t.x < t0.x ? 1u : 0u);
|
||||
|
||||
if (hi != 0u || lo >= p.ticks) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// the store never runs, but the compiler cannot prove it and has to keep the spin
|
||||
if (t.x == 0xFFFFFFFFu && t.y == 0xFFFFFFFFu) {
|
||||
data_d[0] = t.x;
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,6 @@ void main() {
|
||||
if (pass) {
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(data_a[get_aoffset() + src0_idx(idx)]);
|
||||
} else {
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0);
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0.0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -948,6 +948,7 @@ void process_shaders() {
|
||||
string_to_spv("acc_f32", "acc.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
|
||||
|
||||
string_to_spv("split_k_reduce", "mul_mat_split_k_reduce.comp", {});
|
||||
string_to_spv("stream_k_fixup", "mul_mat_stream_k_fixup.comp", {});
|
||||
string_to_spv("fa_split_k_reduce", "flash_attn_split_k_reduce.comp", {});
|
||||
|
||||
string_to_spv("fa_mask_opt", "flash_attn_mask_opt.comp", {});
|
||||
@@ -1027,6 +1028,7 @@ void process_shaders() {
|
||||
string_to_spv("arange_f32", "arange.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
|
||||
string_to_spv("fill_f32", "fill.comp", {{"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
|
||||
string_to_spv("fill_f16", "fill.comp", {{"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
|
||||
string_to_spv("sleep", "sleep.comp", {});
|
||||
|
||||
string_to_spv("silu_back_f32", "silu_back.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
|
||||
|
||||
@@ -1272,6 +1272,7 @@ class ggml_webgpu_shader_lib {
|
||||
std::unordered_map<int, webgpu_pipeline> arange_pipelines; // key is fixed, no variants yet
|
||||
std::unordered_map<int, webgpu_pipeline> count_equal_pipelines; // key is src_overlap
|
||||
std::unordered_map<int, webgpu_pipeline> timestep_embedding_pipelines; // key is fixed, no variants yet
|
||||
std::unordered_map<int, webgpu_pipeline> pool2d_pipelines; // key is ggml_op_pool
|
||||
std::unordered_map<ggml_webgpu_row_norm_pipeline_key, webgpu_pipeline, ggml_webgpu_row_norm_pipeline_key_hash>
|
||||
row_norm_pipelines; // op/inplace
|
||||
|
||||
@@ -2067,6 +2068,34 @@ class ggml_webgpu_shader_lib {
|
||||
return arange_pipelines[1];
|
||||
}
|
||||
|
||||
webgpu_pipeline get_pool2d_pipeline(const ggml_webgpu_shader_lib_context & context) {
|
||||
const int key = ggml_get_op_params_i32(context.dst, 0);
|
||||
|
||||
auto it = pool2d_pipelines.find(key);
|
||||
if (it != pool2d_pipelines.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
std::vector<std::string> defines;
|
||||
std::string variant = "pool2d";
|
||||
if (key == GGML_OP_POOL_MAX) {
|
||||
defines.push_back("POOL_MAX");
|
||||
variant += "_max";
|
||||
} else {
|
||||
variant += "_avg";
|
||||
}
|
||||
|
||||
defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size));
|
||||
|
||||
auto processed = preprocessor.preprocess(wgsl_pool2d, defines);
|
||||
auto decisions = std::make_shared<ggml_webgpu_generic_shader_decisions>();
|
||||
decisions->wg_size = context.max_wg_size;
|
||||
webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, variant);
|
||||
pipeline.context = decisions;
|
||||
pool2d_pipelines[key] = pipeline;
|
||||
return pool2d_pipelines[key];
|
||||
}
|
||||
|
||||
webgpu_pipeline get_pad_pipeline(const ggml_webgpu_shader_lib_context & context) {
|
||||
ggml_webgpu_pad_pipeline_key key = {};
|
||||
key.reflect = context.dst->op == GGML_OP_PAD_REFLECT_1D;
|
||||
|
||||
@@ -961,6 +961,46 @@ static webgpu_encoded_op ggml_webgpu_pad(webgpu_context & ctx, ggml_tensor * src
|
||||
return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y);
|
||||
}
|
||||
|
||||
static webgpu_encoded_op ggml_webgpu_pool2d(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) {
|
||||
ggml_webgpu_shader_lib_context shader_lib_ctx = {};
|
||||
shader_lib_ctx.src0 = src;
|
||||
shader_lib_ctx.dst = dst;
|
||||
shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup;
|
||||
|
||||
webgpu_pipeline pipeline = ctx->shader_lib->get_pool2d_pipeline(shader_lib_ctx);
|
||||
|
||||
auto * decisions = static_cast<ggml_webgpu_generic_shader_decisions *>(pipeline.context.get());
|
||||
|
||||
const uint32_t ne = (uint32_t) ggml_nelements(dst);
|
||||
|
||||
std::vector<uint32_t> params = {
|
||||
ne,
|
||||
(uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src) / ggml_type_size(src->type)),
|
||||
(uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)),
|
||||
(uint32_t) src->ne[0],
|
||||
(uint32_t) src->ne[1],
|
||||
(uint32_t) dst->ne[0],
|
||||
(uint32_t) dst->ne[1],
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 1), // k0
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 2), // k1
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 3), // s0
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 4), // s1
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 5), // p0
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 6), // p1
|
||||
};
|
||||
|
||||
std::vector<wgpu::BindGroupEntry> entries = {
|
||||
ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src),
|
||||
ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst),
|
||||
};
|
||||
|
||||
uint32_t wg_x;
|
||||
uint32_t wg_y;
|
||||
uint32_t total_wg = CEIL_DIV(ne, decisions->wg_size);
|
||||
compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y);
|
||||
return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y);
|
||||
}
|
||||
|
||||
static webgpu_encoded_op ggml_webgpu_solve_tri(webgpu_context & ctx,
|
||||
ggml_tensor * src0,
|
||||
ggml_tensor * src1,
|
||||
@@ -3670,6 +3710,8 @@ static std::optional<webgpu_encoded_op> ggml_webgpu_encode(webgpu_context ctx,
|
||||
}
|
||||
case GGML_OP_ARANGE:
|
||||
return ggml_webgpu_arange(ctx, node);
|
||||
case GGML_OP_POOL_2D:
|
||||
return ggml_webgpu_pool2d(ctx, src0, node);
|
||||
case GGML_OP_PAD_REFLECT_1D:
|
||||
case GGML_OP_PAD:
|
||||
return ggml_webgpu_pad(ctx, src0, node);
|
||||
@@ -5219,6 +5261,9 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
|
||||
case GGML_OP_ARANGE:
|
||||
supports_op = op->type == GGML_TYPE_F32 && ggml_is_contiguous(op);
|
||||
break;
|
||||
case GGML_OP_POOL_2D:
|
||||
supports_op = op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32 && ggml_is_contiguous(src0);
|
||||
break;
|
||||
case GGML_OP_PAD_REFLECT_1D:
|
||||
supports_op = op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32;
|
||||
break;
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
@group(0) @binding(0)
|
||||
var<storage, read_write> src: array<f32>;
|
||||
|
||||
@group(0) @binding(1)
|
||||
var<storage, read_write> dst: array<f32>;
|
||||
|
||||
struct Params {
|
||||
ne: u32, // total number of elements
|
||||
offset_src: u32, // in elements
|
||||
offset_dst: u32, // in elements
|
||||
|
||||
// Logical shapes
|
||||
src_ne0: u32,
|
||||
src_ne1: u32,
|
||||
|
||||
dst_ne0: u32,
|
||||
dst_ne1: u32,
|
||||
|
||||
k0: u32,
|
||||
k1: u32,
|
||||
s0: u32,
|
||||
s1: u32,
|
||||
p0: u32,
|
||||
p1: u32
|
||||
}
|
||||
|
||||
|
||||
@group(0) @binding(2)
|
||||
var<uniform> params: Params;
|
||||
|
||||
@compute @workgroup_size(WG_SIZE)
|
||||
fn main(@builtin(num_workgroups) num_wg: vec3<u32>,
|
||||
@builtin(global_invocation_id) gid: vec3<u32>) {
|
||||
|
||||
let gid_i = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y;
|
||||
if (gid_i >= params.ne) {
|
||||
return;
|
||||
}
|
||||
|
||||
let out_x = gid_i % params.dst_ne0;
|
||||
let out_y = (gid_i / params.dst_ne0) % params.dst_ne1;
|
||||
let out_z = (gid_i / (params.dst_ne0 * params.dst_ne1));
|
||||
|
||||
let x0 = i32(out_x * params.s0) - i32(params.p0);
|
||||
let y0 = i32(out_y * params.s1) - i32(params.p1);
|
||||
let in_x_start = max(0, x0);
|
||||
let in_x_end = min(i32(params.src_ne0), x0 + i32(params.k0));
|
||||
let in_y_start = max(0, y0);
|
||||
let in_y_end = min(i32(params.src_ne1), y0 + i32(params.k1));
|
||||
|
||||
let base = params.offset_src + out_z * params.src_ne0 * params.src_ne1;
|
||||
|
||||
#ifdef POOL_MAX
|
||||
var res: f32 = bitcast<f32>(0xff7fffffu);
|
||||
#else
|
||||
var res: f32 = 0.0;
|
||||
#endif
|
||||
|
||||
for(var y = in_y_start; y < in_y_end; y++){
|
||||
let row = base + u32(y) * params.src_ne0;
|
||||
for(var x = in_x_start; x < in_x_end; x++){
|
||||
#ifdef POOL_MAX
|
||||
res = max(res, src[row + u32(x)]);
|
||||
#else
|
||||
res += src[row + u32(x)];
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
#ifndef POOL_MAX
|
||||
res = res / f32(params.k0 * params.k1);
|
||||
#endif
|
||||
dst[params.offset_dst + gid_i] = res;
|
||||
}
|
||||
+27
-2
@@ -1112,9 +1112,11 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
|
||||
"OPT_STEP_SGD",
|
||||
|
||||
"GLU",
|
||||
|
||||
"SLEEP",
|
||||
};
|
||||
|
||||
static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99");
|
||||
static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100");
|
||||
|
||||
static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
|
||||
"none",
|
||||
@@ -1225,9 +1227,11 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
|
||||
"sgd(x)",
|
||||
|
||||
"glu(x)",
|
||||
|
||||
"sleep(x)",
|
||||
};
|
||||
|
||||
static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99");
|
||||
static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100");
|
||||
|
||||
static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
|
||||
|
||||
@@ -6176,6 +6180,27 @@ struct ggml_tensor * ggml_custom_inplace(
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ggml_sleep
|
||||
|
||||
// the result aliases a, the op only delays and never touches the data
|
||||
struct ggml_tensor * ggml_sleep(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a,
|
||||
int32_t us) {
|
||||
GGML_ASSERT(us >= 0);
|
||||
|
||||
struct ggml_tensor * result = ggml_view_tensor(ctx, a);
|
||||
ggml_format_name(result, "%s (sleep)", a->name);
|
||||
|
||||
ggml_set_op_params_i32(result, 0, us);
|
||||
|
||||
result->op = GGML_OP_SLEEP;
|
||||
result->src[0] = a;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ggml_cross_entropy_loss
|
||||
|
||||
struct ggml_tensor * ggml_cross_entropy_loss(
|
||||
|
||||
@@ -7419,6 +7419,8 @@ class VisionProjectorType:
|
||||
ZONOS3_GEN = "zonos3_gen" # audio generation: 10 codebook heads + ZAC decoder
|
||||
ZONOS3_SPKENC = "zonos3_spkenc" # audio: MECT-B2 speaker encoder
|
||||
YUE2_GEN = "yue2_gen" # audio generation: flow-matching NAR + Oobleck VAE decoder
|
||||
VOXCPM2_GEN = "voxcpm2_gen" # audio generation: residual LM + LocDiT flow matching + AudioVAE decoder
|
||||
VOXCPM2_SPKENC = "voxcpm2_spkenc" # audio: AudioVAE encoder, reference clip -> latents
|
||||
MINIMAX_MUSIC3_GEN = "minimax_music3_gen" # audio generation: RVQ depth decoder + flow-matching DiT + DAC decoder
|
||||
FISH_S2_GEN = "fish_s2_gen" # audio generation: fast AR code predictor + Fish DAC decoder
|
||||
FISH_S2_SPKENC = "fish_s2_spkenc" # audio: Fish DAC encoder + RVQ, reference clip -> prompt rows
|
||||
|
||||
@@ -5,7 +5,7 @@ import os
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
HTTPLIB_VERSION = "refs/tags/v0.66.0"
|
||||
HTTPLIB_VERSION = "refs/tags/v0.67.0"
|
||||
JSON_VERSION = "v3.12.0"
|
||||
|
||||
# stb has no release tag, so we pin a commit (stb_image v2.32)
|
||||
|
||||
@@ -41,7 +41,7 @@ function(mime_from_ext name out_var)
|
||||
set(m "text/html; charset=utf-8")
|
||||
elseif(ext_str STREQUAL "css")
|
||||
set(m "text/css")
|
||||
elseif(ext_str STREQUAL "js")
|
||||
elseif(ext_str STREQUAL "js" OR ext_str STREQUAL "mjs")
|
||||
set(m "application/javascript")
|
||||
elseif(ext_str STREQUAL "json")
|
||||
set(m "application/json")
|
||||
@@ -98,9 +98,12 @@ function(ui_validate_assets files in_dir)
|
||||
set(found_build_json TRUE)
|
||||
elseif(base STREQUAL "version.json")
|
||||
set(found_version_json TRUE)
|
||||
elseif(base MATCHES "^bundle.*\\.js$")
|
||||
# the app's entry: bundle[hash].js/.css from a single-bundle build (builds before the
|
||||
# split one, prebuilt archives included), or entry/start[hash].js and the CSS in
|
||||
# _app/immutable/assets from a split build
|
||||
elseif(base MATCHES "^bundle.*\\.js$" OR f MATCHES "(^|/)_app/immutable/entry/start\\.[^/]*\\.js$")
|
||||
set(found_bundle_js TRUE)
|
||||
elseif(base MATCHES "^bundle.*\\.css$")
|
||||
elseif(base MATCHES "^bundle.*\\.css$" OR f MATCHES "(^|/)_app/immutable/assets/[^/]*\\.css$")
|
||||
set(found_bundle_css TRUE)
|
||||
endif()
|
||||
endforeach()
|
||||
@@ -125,10 +128,10 @@ function(ui_validate_assets files in_dir)
|
||||
list(APPEND missing "version.json")
|
||||
endif()
|
||||
if(NOT found_bundle_js)
|
||||
list(APPEND missing "bundle[hash].js")
|
||||
list(APPEND missing "_app/immutable/entry/start[hash].js (or bundle[hash].js)")
|
||||
endif()
|
||||
if(NOT found_bundle_css)
|
||||
list(APPEND missing "bundle[hash].css")
|
||||
list(APPEND missing "_app/immutable/assets/[name][hash].css (or bundle[hash].css)")
|
||||
endif()
|
||||
|
||||
if(missing)
|
||||
|
||||
@@ -259,6 +259,10 @@ bool llama_batch_allocr::init(
|
||||
decision_order.resize(n_tok, 0);
|
||||
decision_order[i] = batch_inp.tokens[i].decision_order;
|
||||
}
|
||||
if (batch_inp.tokens[i].decision_group != 0) {
|
||||
decision_group.resize(n_tok, 0);
|
||||
decision_group[i] = batch_inp.tokens[i].decision_group;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
@@ -353,6 +357,7 @@ bool llama_batch_allocr::init(
|
||||
/*.output =*/ batch.logits,
|
||||
/*.type =*/ is_embd_vec.empty() ? nullptr : is_embd_vec.data(),
|
||||
/*.decision_order =*/ decision_order.empty() ? nullptr : decision_order.data(),
|
||||
/*.decision_group =*/ decision_group.empty() ? nullptr : decision_group.data(),
|
||||
/*.data =*/ {},
|
||||
};
|
||||
|
||||
@@ -579,6 +584,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
|
||||
/*.output =*/ udata->output.data(),
|
||||
/*.type =*/ nullptr,
|
||||
/*.decision_order =*/ nullptr,
|
||||
/*.decision_group =*/ nullptr,
|
||||
/*.data =*/ std::move(udata),
|
||||
};
|
||||
|
||||
@@ -887,6 +893,7 @@ void llama_batch_allocr::clear() {
|
||||
seq_id_unq .clear();
|
||||
output .clear();
|
||||
decision_order.clear();
|
||||
decision_group.clear();
|
||||
|
||||
for (auto & cur : seq_pos) {
|
||||
cur.clear();
|
||||
@@ -943,6 +950,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
udata->output .resize(n_tokens);
|
||||
udata->type .resize(mixed ? n_tokens : 0);
|
||||
udata->decision_order.resize(decision_order.empty() ? 0 : n_tokens);
|
||||
udata->decision_group.resize(decision_group.empty() ? 0 : n_tokens);
|
||||
|
||||
udata->batch_idxs = idxs;
|
||||
udata->seq_id_data.reserve(n_tokens);
|
||||
@@ -984,6 +992,9 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
if (!decision_order.empty()) {
|
||||
udata->decision_order[i] = decision_order[idxs[i]];
|
||||
}
|
||||
if (!decision_group.empty()) {
|
||||
udata->decision_group[i] = decision_group[idxs[i]];
|
||||
}
|
||||
|
||||
for (int s = 0; s < udata->n_seq_id[i]; ++s) {
|
||||
const llama_seq_id seq_id = batch.seq_id[idxs[i]][s];
|
||||
@@ -1030,6 +1041,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
|
||||
/*.output =*/ udata->output.data(),
|
||||
/*.type =*/ mixed ? udata->type.data() : nullptr,
|
||||
/*.decision_order =*/ udata->decision_order.empty() ? nullptr : udata->decision_order.data(),
|
||||
/*.decision_group =*/ udata->decision_group.empty() ? nullptr : udata->decision_group.data(),
|
||||
/*.data =*/ std::move(udata),
|
||||
};
|
||||
|
||||
@@ -1430,6 +1442,14 @@ bool llama_batch_ext::set_decision_order(int32_t idx, int32_t order) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool llama_batch_ext::set_decision_group(int32_t idx, int32_t group) {
|
||||
if (idx < 0 || idx >= (int32_t) tokens.size() || group < 0) {
|
||||
return false;
|
||||
}
|
||||
tokens[idx].decision_group = group;
|
||||
return true;
|
||||
}
|
||||
|
||||
// llama_batch_ext C API
|
||||
|
||||
llama_batch_ext * llama_batch_ext_init(llama_context * ctx) {
|
||||
@@ -1513,6 +1533,10 @@ bool llama_batch_ext_set_decision_order(llama_batch_ext * batch, int32_t idx, ll
|
||||
return batch->set_decision_order(idx, order);
|
||||
}
|
||||
|
||||
bool llama_batch_ext_set_decision_group(llama_batch_ext * batch, int32_t idx, int32_t group) {
|
||||
return batch->set_decision_group(idx, group);
|
||||
}
|
||||
|
||||
// llama_batch_compat
|
||||
|
||||
void llama_batch_compat::init(llama_batch_ext & dst, const llama_batch & batch_inp, size_t n_embd_row) {
|
||||
|
||||
@@ -61,6 +61,7 @@ struct llama_ubatch {
|
||||
int8_t * output; // [n_tokens] | i | -
|
||||
int8_t * type; // [n_tokens] | i | - (mixed ubatch only, 0 - token, 1 - embd)
|
||||
int32_t * decision_order; // [n_tokens], NULL if no entry has one, see llama_batch_ext_set_decision_order()
|
||||
int32_t * decision_group; // [n_tokens], NULL if no entry has one, see llama_batch_ext_set_decision_group()
|
||||
|
||||
struct data_t {
|
||||
std::vector<llama_token> token;
|
||||
@@ -77,6 +78,7 @@ struct llama_ubatch {
|
||||
std::vector<int8_t> type;
|
||||
std::vector<int32_t> batch_idxs; // original batch index for each token
|
||||
std::vector<int32_t> decision_order;
|
||||
std::vector<int32_t> decision_group;
|
||||
|
||||
std::vector<llama_seq_id> seq_id_data;
|
||||
};
|
||||
@@ -121,6 +123,7 @@ struct llama_batch_ext {
|
||||
int32_t state_row = -1; // row of ctx_state_src's h_nextn device buffer used as state, -1 if none
|
||||
bool output = false; // TODO: have dedicated output flags
|
||||
int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
|
||||
int32_t decision_group = 0; // see llama_batch_ext_set_decision_group()
|
||||
std::unordered_set<llama_seq_id> seq_ids;
|
||||
std::array<llama_pos, GGML_MROPE_SECTIONS> pos = {0, 0, 0, 0};
|
||||
};
|
||||
@@ -155,6 +158,7 @@ struct llama_batch_ext {
|
||||
bool set_token_pos(int32_t idx, const llama_pos * pos_in);
|
||||
bool set_output(int32_t idx, bool output_last);
|
||||
bool set_decision_order(int32_t idx, int32_t order);
|
||||
bool set_decision_group(int32_t idx, int32_t group);
|
||||
};
|
||||
|
||||
// a helper for sanitizing, fulfilling and splitting a batch
|
||||
@@ -241,6 +245,7 @@ private:
|
||||
std::vector<int32_t> seq_idx;
|
||||
std::vector<int8_t> output;
|
||||
std::vector<int32_t> decision_order; // empty if no entry has one
|
||||
std::vector<int32_t> decision_group; // empty if no entry has one
|
||||
|
||||
using pos_set_t = std::set<llama_pos>;
|
||||
using seq_cpl_t = std::vector<bool>;
|
||||
|
||||
@@ -1934,7 +1934,11 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
|
||||
const llama_ubatch ubatch = balloc->split_simple(n_tokens);
|
||||
|
||||
// micro-batching is not possible for non-causal encoding, so we process the batch in a single shot
|
||||
GGML_ASSERT(cparams.n_ubatch >= n_tokens && "encoder requires n_ubatch >= n_tokens");
|
||||
if (n_tokens > cparams.n_ubatch) {
|
||||
LLAMA_LOG_ERROR("%s: the encoder reads the batch in one pass, n_tokens = %u > n_ubatch = %u, increase n_ubatch (-ub)\n",
|
||||
__func__, n_tokens, cparams.n_ubatch);
|
||||
return -1;
|
||||
}
|
||||
|
||||
// TODO: this clear of the buffer can easily be forgotten - need something better
|
||||
// sync first so any in-flight async copies into embd_seq complete before it is freed
|
||||
@@ -3091,7 +3095,6 @@ llm_graph_params llama_context::graph_params(
|
||||
llm_graph_type gtype) const {
|
||||
return {
|
||||
/*.model =*/ &model,
|
||||
/*.hparams =*/ model.hparams,
|
||||
/*.cparams =*/ cparams,
|
||||
/*.ubatch =*/ ubatch,
|
||||
/*.gtype =*/ gtype,
|
||||
@@ -3102,8 +3105,6 @@ llm_graph_params llama_context::graph_params(
|
||||
/*.mctx =*/ mctx,
|
||||
/*.cross =*/ &cross,
|
||||
/*.moe_cache =*/ moe_cache.get(),
|
||||
/*.prec_policy =*/ &model.prec_policy,
|
||||
/*.hdmd =*/ model.hdmd.rot.empty() ? nullptr : &model.hdmd,
|
||||
/*.layer_inp_dev =*/ layer_inp_dev,
|
||||
/*.layer_inp_src =*/ supports_embd_row() ? cparams.ctx_other->get_layer_inp_dev() : nullptr,
|
||||
/*.h_nextn_dev =*/ cparams.embeddings_nextn_dev ? h_nextn_dev : nullptr,
|
||||
|
||||
@@ -116,6 +116,8 @@ LLAMA_API ggml_backend_dev_t llama_model_get_device(const struct llama_model * m
|
||||
|
||||
LLAMA_API llama_memory_breakdown llama_get_memory_breakdown(const struct llama_context * ctx);
|
||||
|
||||
LLAMA_API std::vector<ggml_tensor *> llama_memory_get_tensors(const llama_memory_t mem);
|
||||
|
||||
// the backend buffers that a state write of this memory would touch - empty caches report none
|
||||
LLAMA_API std::vector<ggml_backend_buffer_t> llama_memory_get_buffers(const llama_memory_t mem);
|
||||
|
||||
@@ -137,6 +139,10 @@ enum llama_decision_order {
|
||||
// The embeddings output has one value per entry: row i is the score of option i
|
||||
LLAMA_API bool llama_batch_ext_set_decision_order(struct llama_batch_ext * batch, int32_t idx, enum llama_decision_order order);
|
||||
|
||||
// Questions packed after a shared state in one sequence, contexts without memory only
|
||||
// 0 (default) is the state, k > 0 the k-th question; full attention layers hide other questions from a question
|
||||
LLAMA_API bool llama_batch_ext_set_decision_group(struct llama_batch_ext * batch, int32_t idx, int32_t group);
|
||||
|
||||
// Label of option i of a decision model ("<arch>.decision.labels"), nullptr past the last one
|
||||
LLAMA_API const char * llama_model_decision_label(const struct llama_model * model, uint32_t i);
|
||||
|
||||
|
||||
+226
-40
@@ -608,25 +608,32 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
|
||||
const int64_t n_kv = self_attn_mask.raw->ne[0];
|
||||
const int64_t n_tokens = ubatch->n_tokens;
|
||||
|
||||
const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type, bool causal) {
|
||||
const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type, bool causal, const int32_t * group) {
|
||||
using T = std::remove_reference_t<decltype(*data)>;
|
||||
std::fill(data, data + ne, llama_cast<T>(-INFINITY));
|
||||
|
||||
for (int i1 = 0; i1 < n_tokens; ++i1) {
|
||||
const llama_seq_id s1 = ubatch->seq_id[i1][0];
|
||||
const llama_pos p1 = ubatch->pos[i1];
|
||||
const int32_t g1 = group ? group[i1] : 0;
|
||||
|
||||
const uint64_t idst = i1*n_kv;
|
||||
|
||||
for (int i0 = 0; i0 < n_tokens; ++i0) {
|
||||
const llama_seq_id s0 = ubatch->seq_id[i0][0];
|
||||
const llama_pos p0 = ubatch->pos[i0];
|
||||
const int32_t g0 = group ? group[i0] : 0;
|
||||
|
||||
// mask different sequences
|
||||
if (s0 != s1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// mask other questions, see llama_batch_ext_set_decision_group()
|
||||
if (g0 != 0 && g1 != 0 && g0 != g1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// mask future tokens
|
||||
if (causal && p0 > p1) {
|
||||
continue;
|
||||
@@ -653,18 +660,18 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
|
||||
GGML_ASSERT(self_attn_mask.raw);
|
||||
GGML_ASSERT(ggml_backend_buffer_is_host(self_attn_mask.raw->buffer));
|
||||
if (self_attn_mask.raw->type == GGML_TYPE_F16) {
|
||||
fill_mask((ggml_fp16_t *) self_attn_mask.raw->data, ggml_nelements(self_attn_mask.raw), 0, LLAMA_SWA_TYPE_NONE, causal);
|
||||
fill_mask((ggml_fp16_t *) self_attn_mask.raw->data, ggml_nelements(self_attn_mask.raw), 0, LLAMA_SWA_TYPE_NONE, causal, ubatch->decision_group);
|
||||
} else {
|
||||
fill_mask((float *) self_attn_mask.raw->data, ggml_nelements(self_attn_mask.raw), 0, LLAMA_SWA_TYPE_NONE, causal);
|
||||
fill_mask((float *) self_attn_mask.raw->data, ggml_nelements(self_attn_mask.raw), 0, LLAMA_SWA_TYPE_NONE, causal, ubatch->decision_group);
|
||||
}
|
||||
|
||||
if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
|
||||
GGML_ASSERT(self_attn_mask_swa.raw);
|
||||
GGML_ASSERT(ggml_backend_buffer_is_host(self_attn_mask_swa.raw->buffer));
|
||||
if (self_attn_mask_swa.raw->type == GGML_TYPE_F16) {
|
||||
fill_mask((ggml_fp16_t *) self_attn_mask_swa.raw->data, ggml_nelements(self_attn_mask_swa.raw), hparams.n_swa, hparams.swa_type, causal_swa);
|
||||
fill_mask((ggml_fp16_t *) self_attn_mask_swa.raw->data, ggml_nelements(self_attn_mask_swa.raw), hparams.n_swa, hparams.swa_type, causal_swa, nullptr);
|
||||
} else {
|
||||
fill_mask((float *) self_attn_mask_swa.raw->data, ggml_nelements(self_attn_mask_swa.raw), hparams.n_swa, hparams.swa_type, causal_swa);
|
||||
fill_mask((float *) self_attn_mask_swa.raw->data, ggml_nelements(self_attn_mask_swa.raw), hparams.n_swa, hparams.swa_type, causal_swa, nullptr);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1393,6 +1400,91 @@ bool llm_graph_input_mem_hybrid_iswa::can_reuse(const llm_graph_params & params)
|
||||
return res;
|
||||
}
|
||||
|
||||
void llm_graph_input_mem_hybrid_iswa_rel::set_input(const llama_ubatch * ubatch) {
|
||||
const auto * attn_ctx = mctx->get_attn();
|
||||
|
||||
// base tensors may not be allocated if there are no non-SWA attention layers
|
||||
if (inp_attn->self_k_idxs && inp_attn->self_k_idxs->buffer) {
|
||||
attn_ctx->get_base()->set_input_k_idxs(inp_attn->self_k_idxs, ubatch);
|
||||
attn_ctx->get_base()->set_input_v_idxs(inp_attn->self_v_idxs, ubatch);
|
||||
}
|
||||
|
||||
if (inp_attn->self_attn_mask.raw && inp_attn->self_attn_mask.raw->buffer) {
|
||||
attn_ctx->get_base()->set_input_attn_mask(inp_attn->self_attn_mask, ubatch, cparams.causal_attn);
|
||||
}
|
||||
|
||||
if (self_rel_pos->buffer) {
|
||||
attn_ctx->get_base()->set_input_relative_pos(self_rel_pos, ubatch, extent);
|
||||
}
|
||||
|
||||
// swa tensors may not be allocated if there are no SWA attention layers
|
||||
if (inp_attn->self_k_idxs_swa && inp_attn->self_k_idxs_swa->buffer) {
|
||||
attn_ctx->get_swa()->set_input_k_idxs(inp_attn->self_k_idxs_swa, ubatch);
|
||||
attn_ctx->get_swa()->set_input_v_idxs(inp_attn->self_v_idxs_swa, ubatch);
|
||||
}
|
||||
|
||||
if (inp_attn->self_attn_mask_swa.raw && inp_attn->self_attn_mask_swa.raw->buffer) {
|
||||
attn_ctx->get_swa()->set_input_attn_mask(inp_attn->self_attn_mask_swa, ubatch, cparams.causal_attn);
|
||||
}
|
||||
|
||||
if (self_rel_pos_swa->buffer) {
|
||||
attn_ctx->get_swa()->set_input_relative_pos(self_rel_pos_swa, ubatch, extent_swa);
|
||||
}
|
||||
|
||||
const int64_t n_rs = mctx->get_recr()->get_n_rs();
|
||||
|
||||
if (inp_rs->s_copy) {
|
||||
GGML_ASSERT(ggml_backend_buffer_is_host(inp_rs->s_copy->buffer));
|
||||
int32_t * data = (int32_t *) inp_rs->s_copy->data;
|
||||
|
||||
// assuming copy destinations ALWAYS happen ONLY on the cells between head and head+n
|
||||
for (uint32_t i = 0; i < n_rs; ++i) {
|
||||
data[i] = mctx->get_recr()->s_copy(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool llm_graph_input_mem_hybrid_iswa_rel::can_reuse(const llm_graph_params & params) {
|
||||
const auto * mctx = static_cast<const llama_memory_hybrid_iswa_context *>(params.mctx);
|
||||
|
||||
this->mctx = mctx;
|
||||
|
||||
bool res = true;
|
||||
|
||||
const auto * attn_ctx = mctx->get_attn();
|
||||
|
||||
// the masks are always dense, see build_inp_mem_hybrid_iswa_rel()
|
||||
|
||||
// base tensors may not be allocated if there are no non-SWA attention layers
|
||||
if (inp_attn->self_k_idxs && inp_attn->self_k_idxs->buffer) {
|
||||
res &= inp_attn->self_k_idxs->ne[0] == params.ubatch.n_tokens;
|
||||
}
|
||||
|
||||
res &= can_reuse_kq_mask(inp_attn->self_attn_mask.raw, attn_ctx->get_base(), params.ubatch, params.cparams);
|
||||
|
||||
res &= self_rel_pos->ne[0] == attn_ctx->get_base()->get_n_kv();
|
||||
res &= self_rel_pos->ne[1] == params.ubatch.n_tokens;
|
||||
|
||||
// swa tensors may not be allocated if there are no SWA attention layers
|
||||
if (inp_attn->self_k_idxs_swa && inp_attn->self_k_idxs_swa->buffer) {
|
||||
res &= inp_attn->self_k_idxs_swa->ne[0] == params.ubatch.n_tokens;
|
||||
}
|
||||
|
||||
res &= can_reuse_kq_mask(inp_attn->self_attn_mask_swa.raw, attn_ctx->get_swa(), params.ubatch, params.cparams);
|
||||
|
||||
res &= self_rel_pos_swa->ne[0] == attn_ctx->get_swa()->get_n_kv();
|
||||
res &= self_rel_pos_swa->ne[1] == params.ubatch.n_tokens;
|
||||
|
||||
res &= inp_rs->s_copy->ne[0] == mctx->get_recr()->get_n_rs();
|
||||
|
||||
res &= inp_rs->s_copy_main->ne[0] == params.ubatch.n_seqs;
|
||||
|
||||
res &= inp_rs->head == mctx->get_recr()->get_head();
|
||||
res &= inp_rs->rs_z == mctx->get_recr()->get_rs_z();
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void llm_graph_input_sampling::set_input(const llama_ubatch * ubatch) {
|
||||
// set the inputs only for the active samplers in the current ubatch
|
||||
std::unordered_set<llama_seq_id> active_samplers;
|
||||
@@ -1587,9 +1679,10 @@ void llm_graph_result::set_params(const llm_graph_params & params) {
|
||||
|
||||
llm_graph_context::llm_graph_context(const llm_graph_params & params) :
|
||||
model (*params.model),
|
||||
hparams (params.hparams),
|
||||
hparams (model.hparams),
|
||||
cparams (params.cparams),
|
||||
ubatch (params.ubatch),
|
||||
prec_policy (model.prec_policy),
|
||||
n_embd (hparams.n_embd),
|
||||
n_layer (hparams.n_layer()),
|
||||
n_layer_nextn (hparams.n_layer_nextn),
|
||||
@@ -1623,8 +1716,7 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) :
|
||||
mctx (params.mctx),
|
||||
cross (params.cross),
|
||||
moe_cache (params.moe_cache),
|
||||
prec_policy (params.prec_policy),
|
||||
hdmd (params.hdmd),
|
||||
hdmd (model.hdmd.rot.empty() ? nullptr : &model.hdmd),
|
||||
layer_inp_dev (params.layer_inp_dev),
|
||||
layer_inp_src (params.layer_inp_src),
|
||||
h_nextn_dev (params.h_nextn_dev),
|
||||
@@ -1721,9 +1813,7 @@ ggml_tensor * llm_graph_context::build_lora_mm(
|
||||
ggml_tensor * cur_mm = build_hadamard_input(w, cur);
|
||||
ggml_tensor * res = ggml_mul_mat(ctx0, w, cur_mm);
|
||||
|
||||
if (prec_policy) {
|
||||
prec_policy->apply(res);
|
||||
}
|
||||
prec_policy.apply(res);
|
||||
|
||||
if (w->type == GGML_TYPE_NVFP4) {
|
||||
ggml_prec_set_acc(res, GGML_PREC_BF16);
|
||||
@@ -1805,9 +1895,7 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
|
||||
ggml_mul_mat_id(ctx0, w, cur_mm, ids) :
|
||||
ggml_mul_mat_id(ctx0, cached ? moe_cache->get_experts(w) : w, cur_mm, slots);
|
||||
|
||||
if (prec_policy) {
|
||||
prec_policy->apply(res);
|
||||
}
|
||||
prec_policy.apply(res);
|
||||
|
||||
if (w->type == GGML_TYPE_NVFP4) {
|
||||
ggml_prec_set_acc(res, GGML_PREC_BF16);
|
||||
@@ -3110,7 +3198,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
|
||||
ggml_tensor * cur;
|
||||
|
||||
const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr && (!sinks || ggml_is_vector(sinks));
|
||||
const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr;
|
||||
GGML_ASSERT(use_flash_attn || kv_rows == nullptr);
|
||||
if (use_flash_attn) {
|
||||
GGML_ASSERT(kq_b == nullptr && "Flash attention does not support KQ bias yet");
|
||||
@@ -3193,16 +3281,8 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
cb(kq, "kq_plus_kq_b", il);
|
||||
}
|
||||
|
||||
if (sinks && !ggml_is_vector(sinks)) {
|
||||
// Per-query sinks are extra logits with zero values.
|
||||
const int64_t n_kv = kq->ne[0];
|
||||
kq = ggml_add(ctx0, ggml_scale(ctx0, kq, kq_scale), ggml_cast(ctx0, kq_mask, GGML_TYPE_F32));
|
||||
kq = ggml_soft_max(ctx0, ggml_concat(ctx0, kq, sinks, 0));
|
||||
kq = ggml_cont(ctx0, ggml_view_4d(ctx0, kq, n_kv, kq->ne[1], kq->ne[2], kq->ne[3], kq->nb[1], kq->nb[2], kq->nb[3], 0));
|
||||
} else {
|
||||
kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias);
|
||||
ggml_soft_max_add_sinks(kq, sinks);
|
||||
}
|
||||
kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias);
|
||||
ggml_soft_max_add_sinks(kq, sinks);
|
||||
cb(kq, "kq_soft_max", il);
|
||||
|
||||
if (!v_trans) {
|
||||
@@ -3684,7 +3764,9 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_attn(
|
||||
// the iswa build_attn with the KQ mask and kv_rows given by the caller, which has to keep them in sync
|
||||
static ggml_tensor * build_attn_iswa_impl(
|
||||
const llm_graph_context & gctx,
|
||||
llm_graph_input_attn_kv_iswa * inp,
|
||||
ggml_tensor * wo,
|
||||
ggml_tensor * wo_b,
|
||||
@@ -3693,11 +3775,18 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * k_cur,
|
||||
ggml_tensor * v_cur,
|
||||
ggml_tensor * kq_b,
|
||||
ggml_tensor * kq_mask,
|
||||
ggml_tensor * kv_rows,
|
||||
ggml_tensor * sinks,
|
||||
ggml_tensor * v_mla,
|
||||
float kq_scale,
|
||||
int il,
|
||||
ggml_tensor * kq_mask) const {
|
||||
int il) {
|
||||
ggml_context * ctx0 = gctx.ctx0;
|
||||
ggml_cgraph * gf = gctx.gf;
|
||||
|
||||
const auto & hparams = gctx.hparams;
|
||||
const auto & cparams = gctx.cparams;
|
||||
|
||||
const bool is_swa = hparams.is_swa(il);
|
||||
|
||||
auto * k_rot = is_swa ? inp->self_k_rot_swa[il] : inp->self_k_rot[il];
|
||||
@@ -3735,7 +3824,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
// whole seq fits into batch in training mode
|
||||
const bool use_kv_cur = cparams.training && k_cur && v_cur;
|
||||
if (use_kv_cur) {
|
||||
GGML_ASSERT(mctx_cur->get_n_kv() == n_tokens);
|
||||
GGML_ASSERT(mctx_cur->get_n_kv() == gctx.n_tokens);
|
||||
}
|
||||
|
||||
const bool store_kv = !use_kv_cur || hparams.n_layer_kv_from_start >= 0;
|
||||
@@ -3753,25 +3842,20 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));
|
||||
}
|
||||
|
||||
if (!kq_mask) {
|
||||
kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw;
|
||||
}
|
||||
const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows;
|
||||
|
||||
ggml_tensor * q = q_cur;
|
||||
ggml_tensor * k = use_kv_cur ? k_cur : mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = use_kv_cur ? v_cur : mctx_cur->get_v(ctx0, il);
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, 0, kq_scale, il);
|
||||
ggml_tensor * cur = gctx.build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, 0, kq_scale, il);
|
||||
cur = build_attn_scale_kv_output(ctx0, mctx_cur, cur, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
gctx.cb(cur, "kqv_out", il);
|
||||
|
||||
if (v_rot) {
|
||||
cur = llama_mul_mat_hadamard(ctx0, cur, v_rot);
|
||||
}
|
||||
|
||||
if (wo) {
|
||||
cur = build_lora_mm(wo, cur, wo_s);
|
||||
cur = gctx.build_lora_mm(wo, cur, wo_s);
|
||||
}
|
||||
|
||||
if (wo_b) {
|
||||
@@ -3785,6 +3869,27 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_attn(
|
||||
llm_graph_input_attn_kv_iswa * inp,
|
||||
ggml_tensor * wo,
|
||||
ggml_tensor * wo_b,
|
||||
ggml_tensor * wo_s,
|
||||
ggml_tensor * q_cur,
|
||||
ggml_tensor * k_cur,
|
||||
ggml_tensor * v_cur,
|
||||
ggml_tensor * kq_b,
|
||||
ggml_tensor * sinks,
|
||||
ggml_tensor * v_mla,
|
||||
float kq_scale,
|
||||
int il) const {
|
||||
const bool is_swa = hparams.is_swa(il);
|
||||
|
||||
const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw;
|
||||
const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows;
|
||||
|
||||
return build_attn_iswa_impl(*this, inp, wo, wo_b, wo_s, q_cur, k_cur, v_cur, kq_b, kq_mask, kv_rows, sinks, v_mla, kq_scale, il);
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_attn(
|
||||
llm_graph_input_attn_k_iswa * inp,
|
||||
ggml_tensor * wo,
|
||||
@@ -4096,7 +4201,7 @@ ggml_tensor * llm_graph_context::build_rs(
|
||||
// Clear a single state which will then be copied to the other cleared states.
|
||||
// Note that this is a no-op when the view is zero-sized.
|
||||
ggml_tensor * state_zero = ggml_view_1d(ctx0, states, state_size*(rs_zero >= 0), rs_zero*states->nb[1]*(rs_zero >= 0));
|
||||
ggml_build_forward_expand(gf, ggml_scale_inplace(ctx0, state_zero, 0));
|
||||
ggml_build_forward_expand(gf, ggml_fill_inplace(ctx0, state_zero, 0.0f));
|
||||
|
||||
// copy states
|
||||
// NOTE: assuming the copy destinations are ALL contained between rs_head and rs_head + n_rs
|
||||
@@ -4303,6 +4408,43 @@ llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa()
|
||||
|
||||
auto inp_attn = std::make_unique<llm_graph_input_attn_kv_iswa>(hparams, cparams, attn_ctx);
|
||||
|
||||
{
|
||||
inp_attn->self_k_idxs = attn_ctx->get_base()->build_input_k_idxs(ctx0, ubatch);
|
||||
inp_attn->self_v_idxs = attn_ctx->get_base()->build_input_v_idxs(ctx0, ubatch);
|
||||
|
||||
inp_attn->self_attn_mask = build_attn_inp_attn_mask(ctx0, attn_ctx->get_base(), ubatch, cparams);
|
||||
}
|
||||
|
||||
{
|
||||
inp_attn->self_k_idxs_swa = attn_ctx->get_swa()->build_input_k_idxs(ctx0, ubatch);
|
||||
inp_attn->self_v_idxs_swa = attn_ctx->get_swa()->build_input_v_idxs(ctx0, ubatch);
|
||||
|
||||
inp_attn->self_attn_mask_swa = build_attn_inp_attn_mask(ctx0, attn_ctx->get_swa(), ubatch, cparams);
|
||||
}
|
||||
|
||||
inp_attn->self_k_rot = attn_ctx->get_base()->get_k_rot();
|
||||
inp_attn->self_v_rot = attn_ctx->get_base()->get_v_rot();
|
||||
|
||||
inp_attn->self_k_rot_swa = attn_ctx->get_swa()->get_k_rot();
|
||||
inp_attn->self_v_rot_swa = attn_ctx->get_swa()->get_v_rot();
|
||||
|
||||
auto inp = std::make_unique<llm_graph_input_mem_hybrid_iswa>(cparams, std::move(inp_attn), std::move(inp_rs), mctx_cur);
|
||||
|
||||
return (llm_graph_input_mem_hybrid_iswa *) res->add_input(std::move(inp));
|
||||
}
|
||||
|
||||
llm_graph_input_mem_hybrid_iswa_rel * llm_graph_context::build_inp_mem_hybrid_iswa_rel(
|
||||
uint32_t extent,
|
||||
uint32_t extent_swa) const {
|
||||
const auto * mctx_cur = static_cast<const llama_memory_hybrid_iswa_context *>(mctx);
|
||||
|
||||
auto inp_rs = build_rs_inp_impl(ctx0, ubatch, mctx_cur->get_recr());
|
||||
|
||||
const auto * attn_ctx = mctx_cur->get_attn();
|
||||
|
||||
auto inp_attn = std::make_unique<llm_graph_input_attn_kv_iswa>(hparams, cparams, attn_ctx);
|
||||
|
||||
// dense masks: the bias is gathered by the distance to each cell of the K/V view
|
||||
{
|
||||
inp_attn->self_k_idxs = attn_ctx->get_base()->build_input_k_idxs(ctx0, ubatch);
|
||||
inp_attn->self_v_idxs = attn_ctx->get_base()->build_input_v_idxs(ctx0, ubatch);
|
||||
@@ -4323,9 +4465,53 @@ llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa()
|
||||
inp_attn->self_k_rot_swa = attn_ctx->get_swa()->get_k_rot();
|
||||
inp_attn->self_v_rot_swa = attn_ctx->get_swa()->get_v_rot();
|
||||
|
||||
auto inp = std::make_unique<llm_graph_input_mem_hybrid_iswa>(cparams, std::move(inp_attn), std::move(inp_rs), mctx_cur);
|
||||
auto inp = std::make_unique<llm_graph_input_mem_hybrid_iswa_rel>(cparams, std::move(inp_attn), std::move(inp_rs), mctx_cur, extent, extent_swa);
|
||||
|
||||
return (llm_graph_input_mem_hybrid_iswa *) res->add_input(std::move(inp));
|
||||
inp->self_rel_pos = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, attn_ctx->get_base()->get_n_kv(), ubatch.n_tokens);
|
||||
ggml_set_input(inp->self_rel_pos);
|
||||
ggml_set_name(inp->self_rel_pos, "attn_inp_rel_pos");
|
||||
|
||||
inp->self_rel_pos_swa = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, attn_ctx->get_swa()->get_n_kv(), ubatch.n_tokens);
|
||||
ggml_set_input(inp->self_rel_pos_swa);
|
||||
ggml_set_name(inp->self_rel_pos_swa, "attn_inp_rel_pos_swa");
|
||||
|
||||
return (llm_graph_input_mem_hybrid_iswa_rel *) res->add_input(std::move(inp));
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_attn(
|
||||
llm_graph_input_mem_hybrid_iswa_rel * inp,
|
||||
ggml_tensor * wo,
|
||||
ggml_tensor * wo_b,
|
||||
ggml_tensor * wo_s,
|
||||
ggml_tensor * q_cur,
|
||||
ggml_tensor * k_cur,
|
||||
ggml_tensor * v_cur,
|
||||
ggml_tensor * kq_b,
|
||||
ggml_tensor * sinks,
|
||||
float kq_scale,
|
||||
int il) const {
|
||||
const bool is_swa = hparams.is_swa(il);
|
||||
|
||||
auto * inp_attn = inp->get_attn();
|
||||
|
||||
const auto & attn_mask = is_swa ? inp_attn->self_attn_mask_swa : inp_attn->self_attn_mask;
|
||||
|
||||
// TODO: support kv_rows - fill the distances over the kv_rows cell lists and pass the rows to build_attn_mha,
|
||||
// needs a kv_rows flash attention with a per-head mask (CUDA has none)
|
||||
GGML_ASSERT(attn_mask.rows == nullptr);
|
||||
|
||||
// a mask per head, in the dense layout of the input mask, which flash attention takes in F16
|
||||
ggml_tensor * kq_mask = attn_mask.raw;
|
||||
if (kq_mask->type != GGML_TYPE_F32) {
|
||||
kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F32);
|
||||
}
|
||||
kq_mask = ggml_add(ctx0, kq_b, kq_mask);
|
||||
if (cparams.flash_attn) {
|
||||
kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16);
|
||||
}
|
||||
cb(kq_mask, "kq_mask_rel_pos", il);
|
||||
|
||||
return build_attn_iswa_impl(*this, inp_attn, wo, wo_b, wo_s, q_cur, k_cur, v_cur, nullptr, kq_mask, nullptr, sinks, nullptr, kq_scale, il);
|
||||
}
|
||||
|
||||
void llm_graph_context::build_dense_out(
|
||||
|
||||
+64
-9
@@ -774,6 +774,48 @@ public:
|
||||
const llama_memory_hybrid_iswa_context * mctx;
|
||||
};
|
||||
|
||||
// hybrid iswa memory with a per-head attention bias that depends on the distance from the query to the cell (Inkling)
|
||||
// the KQ masks are always dense (rows == nullptr) and are filled from the same cells as the distances, so that the bias
|
||||
// gathered by distance lines up with the mask
|
||||
class llm_graph_input_mem_hybrid_iswa_rel : public llm_graph_input_i {
|
||||
public:
|
||||
llm_graph_input_mem_hybrid_iswa_rel(
|
||||
const llama_cparams & cparams,
|
||||
std::unique_ptr<llm_graph_input_attn_kv_iswa> inp_attn,
|
||||
std::unique_ptr<llm_graph_input_rs> inp_rs,
|
||||
const llama_memory_hybrid_iswa_context * mctx,
|
||||
uint32_t extent,
|
||||
uint32_t extent_swa) :
|
||||
inp_attn(std::move(inp_attn)),
|
||||
inp_rs(std::move(inp_rs)),
|
||||
extent(extent),
|
||||
extent_swa(extent_swa),
|
||||
cparams(cparams),
|
||||
mctx(mctx) { }
|
||||
virtual ~llm_graph_input_mem_hybrid_iswa_rel() = default;
|
||||
|
||||
void set_input(const llama_ubatch * ubatch) override;
|
||||
|
||||
bool can_reuse(const llm_graph_params & params) override;
|
||||
|
||||
std::unique_ptr<llm_graph_input_attn_kv_iswa> inp_attn;
|
||||
std::unique_ptr<llm_graph_input_rs> inp_rs;
|
||||
|
||||
llm_graph_input_attn_kv_iswa * get_attn() const { return inp_attn.get(); }
|
||||
llm_graph_input_rs * get_recr() const { return inp_rs.get(); }
|
||||
|
||||
// distance from the token to the cell, extent for the empty, future and more distant cells
|
||||
ggml_tensor * self_rel_pos = nullptr; // I32 [n_kv, n_batch]
|
||||
ggml_tensor * self_rel_pos_swa = nullptr; // I32 [n_kv_swa, n_batch]
|
||||
|
||||
const uint32_t extent;
|
||||
const uint32_t extent_swa;
|
||||
|
||||
const llama_cparams cparams;
|
||||
|
||||
const llama_memory_hybrid_iswa_context * mctx;
|
||||
};
|
||||
|
||||
class llm_graph_input_sampling : public llm_graph_input_i {
|
||||
public:
|
||||
llm_graph_input_sampling(std::map<llama_seq_id, llama_sampler *> samplers) :
|
||||
@@ -804,7 +846,6 @@ class llm_graph_result;
|
||||
struct llm_graph_params {
|
||||
const llama_model * model = nullptr;
|
||||
|
||||
llama_hparams hparams;
|
||||
llama_cparams cparams;
|
||||
|
||||
llama_ubatch ubatch; // note: intentionally make a copy
|
||||
@@ -820,10 +861,6 @@ struct llm_graph_params {
|
||||
const llama_cross * cross;
|
||||
const llama_moe_cache * moe_cache;
|
||||
|
||||
const llama_prec_policy * prec_policy = nullptr;
|
||||
|
||||
const llama_hadamard * hdmd = nullptr;
|
||||
|
||||
ggml_tensor * layer_inp_dev = nullptr; // own layer input device buffer (write)
|
||||
ggml_tensor * layer_inp_src = nullptr; // ctx_other's layer input device buffer (read)
|
||||
ggml_tensor * h_nextn_dev = nullptr; // own h_nextn device buffer (write)
|
||||
@@ -1056,6 +1093,8 @@ struct llm_graph_context {
|
||||
const llama_cparams & cparams;
|
||||
const llama_ubatch & ubatch;
|
||||
|
||||
const llama_prec_policy & prec_policy;
|
||||
|
||||
const int64_t n_embd;
|
||||
const int64_t n_layer;
|
||||
const int64_t n_layer_nextn;
|
||||
@@ -1096,8 +1135,6 @@ struct llm_graph_context {
|
||||
const llama_cross * cross;
|
||||
const llama_moe_cache * moe_cache;
|
||||
|
||||
const llama_prec_policy * prec_policy;
|
||||
|
||||
const llama_hadamard * hdmd;
|
||||
|
||||
ggml_tensor * layer_inp_dev;
|
||||
@@ -1406,8 +1443,7 @@ struct llm_graph_context {
|
||||
ggml_tensor * sinks, // [n_head_q]
|
||||
ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v]
|
||||
float kq_scale,
|
||||
int il,
|
||||
ggml_tensor * kq_mask = nullptr) const;
|
||||
int il) const;
|
||||
|
||||
llm_graph_input_attn_k_iswa * build_attn_inp_k_iswa() const;
|
||||
|
||||
@@ -1500,6 +1536,25 @@ struct llm_graph_context {
|
||||
|
||||
llm_graph_input_mem_hybrid_iswa * build_inp_mem_hybrid_iswa() const;
|
||||
|
||||
// extent, extent_swa: the distance that the bias tables of the full and the SWA layers stop at
|
||||
llm_graph_input_mem_hybrid_iswa_rel * build_inp_mem_hybrid_iswa_rel(
|
||||
uint32_t extent,
|
||||
uint32_t extent_swa) const;
|
||||
|
||||
// kq_b is added to the KQ mask, after the KQ scale
|
||||
ggml_tensor * build_attn(
|
||||
llm_graph_input_mem_hybrid_iswa_rel * inp,
|
||||
ggml_tensor * wo,
|
||||
ggml_tensor * wo_b,
|
||||
ggml_tensor * wo_s,
|
||||
ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
|
||||
ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]
|
||||
ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]
|
||||
ggml_tensor * kq_b, // [n_kv, n_tokens/n_stream, n_head_q, n_stream]
|
||||
ggml_tensor * sinks, // [n_head_q]
|
||||
float kq_scale,
|
||||
int il) const;
|
||||
|
||||
//
|
||||
// pooling
|
||||
//
|
||||
|
||||
@@ -169,6 +169,7 @@ static llama_ubatch dsv4_build_raw_write_ubatch(const llama_ubatch & ubatch) {
|
||||
/*.output =*/ data->output.data(),
|
||||
/*.type =*/ nullptr,
|
||||
/*.decision_order =*/ nullptr,
|
||||
/*.decision_group =*/ nullptr,
|
||||
/*.data =*/ data,
|
||||
};
|
||||
|
||||
|
||||
+15
-6
@@ -112,9 +112,9 @@ std::map<ggml_backend_buffer_type_t, size_t> llama_memory_bufs::memory_breakdown
|
||||
return ret;
|
||||
}
|
||||
|
||||
// collects the buffers of the tensors that a state write visits, without keeping the data
|
||||
struct llama_memory_buffers_writer : public llama_io_write_i {
|
||||
std::set<ggml_backend_buffer_t> buffers;
|
||||
// collects the tensors that a state write visits, without keeping the data
|
||||
struct llama_memory_tensors_writer : public llama_io_write_i {
|
||||
std::set<ggml_tensor *> tensors;
|
||||
size_t size = 0;
|
||||
|
||||
void write(const void *, size_t n) override {
|
||||
@@ -122,7 +122,7 @@ struct llama_memory_buffers_writer : public llama_io_write_i {
|
||||
}
|
||||
|
||||
void write_tensor(ggml_tensor * tensor, size_t, size_t n) override {
|
||||
buffers.insert(tensor->buffer);
|
||||
tensors.insert(tensor);
|
||||
size += n;
|
||||
}
|
||||
|
||||
@@ -132,13 +132,22 @@ struct llama_memory_buffers_writer : public llama_io_write_i {
|
||||
};
|
||||
|
||||
std::vector<ggml_backend_buffer_t> llama_memory_get_buffers(const llama_memory_t mem) {
|
||||
std::set<ggml_backend_buffer_t> buffers;
|
||||
for (ggml_tensor * tensor : llama_memory_get_tensors(mem)) {
|
||||
buffers.insert(tensor->buffer);
|
||||
}
|
||||
|
||||
return std::vector<ggml_backend_buffer_t>(buffers.begin(), buffers.end());
|
||||
}
|
||||
|
||||
std::vector<ggml_tensor *> llama_memory_get_tensors(const llama_memory_t mem) {
|
||||
if (mem == nullptr) {
|
||||
return {};
|
||||
}
|
||||
|
||||
llama_memory_buffers_writer writer;
|
||||
llama_memory_tensors_writer writer;
|
||||
|
||||
mem->state_write(writer);
|
||||
|
||||
return std::vector<ggml_backend_buffer_t>(writer.buffers.begin(), writer.buffers.end());
|
||||
return std::vector<ggml_tensor *>(writer.tensors.begin(), writer.tensors.end());
|
||||
}
|
||||
|
||||
+2
-10
@@ -5,7 +5,6 @@
|
||||
#include "ggml.h"
|
||||
|
||||
#include <cstring>
|
||||
#include <climits>
|
||||
#include <cstdlib>
|
||||
#include <stdexcept>
|
||||
#include <cerrno>
|
||||
@@ -32,9 +31,6 @@
|
||||
#endif
|
||||
#include <malloc.h>
|
||||
#include <windows.h>
|
||||
#ifndef PATH_MAX
|
||||
#define PATH_MAX MAX_PATH
|
||||
#endif
|
||||
#include <io.h>
|
||||
#endif
|
||||
|
||||
@@ -513,7 +509,7 @@ struct llama_file::impl {
|
||||
if (errno == EINTR) {
|
||||
continue;
|
||||
}
|
||||
if (errno == EFAULT || errno == EINVAL) {
|
||||
if (errno == EFAULT || errno == EINVAL || errno == EIO) {
|
||||
LLAMA_LOG_WARN("%s: file '%s' falling back to buffered I/O due to %s\n",
|
||||
__func__, fname.c_str(), strerror(errno));
|
||||
const size_t read_offset = tell();
|
||||
@@ -522,7 +518,7 @@ struct llama_file::impl {
|
||||
alignment = 1;
|
||||
init_fp("rb");
|
||||
seek(read_offset, SEEK_SET);
|
||||
read_raw_unsafe(ptr, len);
|
||||
read_raw_unsafe(static_cast<char *>(ptr) + bytes_read, to_read);
|
||||
return;
|
||||
}
|
||||
throw std::runtime_error(format("read error: %s", strerror(errno)));
|
||||
@@ -1140,7 +1136,3 @@ void llama_prefetch(llama_memory_ranges mr) {
|
||||
GGML_UNUSED(mr);
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t llama_path_max() {
|
||||
return PATH_MAX;
|
||||
}
|
||||
|
||||
@@ -88,5 +88,3 @@ using llama_memory_ranges = std::vector<llama_memory_range>;
|
||||
|
||||
// Prefetch the host pages covering these memory ranges.
|
||||
void llama_prefetch(llama_memory_ranges mr);
|
||||
|
||||
size_t llama_path_max();
|
||||
|
||||
@@ -84,7 +84,7 @@ const char * llama_ftype_name(llama_ftype ftype) {
|
||||
static std::vector<std::string> llama_get_list_splits(const std::string & path, const int idx, const int n_split) {
|
||||
std::vector<std::string> paths;
|
||||
std::string split_prefix;
|
||||
std::vector<char> buf(llama_path_max(), 0);
|
||||
std::vector<char> buf(path.size() + 1, 0);
|
||||
|
||||
{
|
||||
int ret = llama_split_prefix(buf.data(), buf.size(), path.c_str(), idx, n_split);
|
||||
@@ -610,7 +610,7 @@ llama_model_loader::llama_model_loader(
|
||||
throw std::runtime_error(format("illegal split file idx: %d (file: %s), model must be loaded with the first split", idx, fname.c_str()));
|
||||
}
|
||||
|
||||
std::vector<char> prefix(llama_path_max(), 0);
|
||||
std::vector<char> prefix(fname.size() + 1, 0);
|
||||
if (paths.empty() || llama_split_prefix(prefix.data(), prefix.size(), fname.c_str(), idx, n_split)) {
|
||||
auto backbone = llama_get_list_splits(fname, idx, n_split);
|
||||
auto file_name = [](const std::string & path) {
|
||||
|
||||
+1
-1
@@ -1266,7 +1266,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::
|
||||
GGML_ASSERT(ctx_outs[cur_split] && "Find uninitialized gguf_context");
|
||||
std::string fname = fname_out;
|
||||
if (params->keep_split) {
|
||||
std::vector<char> split_path(llama_path_max(), 0);
|
||||
std::vector<char> split_path(fname_out.size() + sizeof("-00000-of-00000.gguf"), 0);
|
||||
llama_split_path(split_path.data(), split_path.size(), fname_out.c_str(), cur_split, n_split);
|
||||
fname = std::string(split_path.data());
|
||||
}
|
||||
|
||||
@@ -200,7 +200,7 @@ static ggml_tensor * dsv4_view_2d(
|
||||
|
||||
static ggml_tensor * dsv4_append_zero_row(ggml_context * ctx, ggml_tensor * t, bool neg_inf) {
|
||||
ggml_tensor * row = ggml_view_1d(ctx, t, t->ne[0], 0);
|
||||
row = neg_inf ? ggml_scale_bias(ctx, row, 0.0f, -INFINITY) : ggml_scale(ctx, row, 0.0f);
|
||||
row = ggml_fill(ctx, row, neg_inf ? -INFINITY : 0.0f);
|
||||
row = ggml_reshape_2d(ctx, row, t->ne[0], 1);
|
||||
|
||||
return ggml_concat(ctx, t, row, 1);
|
||||
|
||||
+290
-196
@@ -2,111 +2,157 @@
|
||||
#include "../llama-memory-hybrid-iswa.h"
|
||||
|
||||
void llama_model_inkling2::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
|
||||
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
||||
ml.get_key(LLM_KV_ATTENTION_QK_CLIP, hparams.f_qk_clip);
|
||||
ml.get_key(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size);
|
||||
ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);
|
||||
|
||||
// relative-position bias, and the log-scaled positions of the full-attention layers
|
||||
ml.get_key(LLM_KV_ATTENTION_RELATIVE_LENGTH, hparams.n_relative);
|
||||
ml.get_key(LLM_KV_ATTENTION_RELATIVE_EXTENT, hparams.n_relative_extent);
|
||||
ml.get_key(LLM_KV_ATTENTION_QK_CLIP, hparams.f_qk_clip);
|
||||
ml.get_key(LLM_KV_ATTENTION_RELATIVE_CLIP, hparams.f_relative_clip);
|
||||
ml.get_key(LLM_KV_ATTENTION_RELATIVE_CLIP, hparams.f_relative_clip);
|
||||
ml.get_key(LLM_KV_ATTENTION_LOG_SCALE_FLOOR, hparams.n_log_scale_floor);
|
||||
ml.get_key(LLM_KV_ATTENTION_LOG_SCALE_ALPHA, hparams.f_log_scale_alpha);
|
||||
ml.get_key(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size);
|
||||
ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);
|
||||
ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab_padded);
|
||||
|
||||
// the embedding and output tensors are padded past the vocabulary
|
||||
ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab_padded);
|
||||
ml.get_key(LLM_KV_VOCAB_SIZE_UNPADDED, hparams.n_vocab_unpadded);
|
||||
ml.get_key(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta);
|
||||
|
||||
// MoE
|
||||
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
|
||||
|
||||
ml.get_key(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta);
|
||||
ml.get_key(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, hparams.situ_linear_beta);
|
||||
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);
|
||||
ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping);
|
||||
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
|
||||
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
|
||||
|
||||
hparams.f_final_logit_softcapping = 0.0f;
|
||||
ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false);
|
||||
|
||||
// sliding window
|
||||
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
|
||||
ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
|
||||
|
||||
uint32_t n_pattern = 0;
|
||||
ml.get_arr_n(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern);
|
||||
|
||||
if (n_pattern != hparams.n_layer() || !hparams.n_relative || !hparams.n_relative_extent || !hparams.n_swa ||
|
||||
!hparams.attn_res_block_size || !hparams.n_log_scale_floor || hparams.n_shortconv_l_cache < 2 ||
|
||||
hparams.n_shortconv_l_cache > 64 || hparams.n_relative_extent > INT32_MAX || hparams.n_swa > INT32_MAX ||
|
||||
!hparams.n_vocab_unpadded || hparams.n_vocab_unpadded > hparams.n_vocab_padded) {
|
||||
throw std::runtime_error("inkling2: invalid dimensions, vocabulary, or layer pattern");
|
||||
}
|
||||
for (float value : {hparams.f_norm_rms_eps, hparams.f_qk_clip, hparams.f_relative_clip, hparams.situ_beta,
|
||||
hparams.situ_linear_beta, hparams.expert_weights_scale, hparams.f_final_logit_softcapping}) {
|
||||
|
||||
for (float value : {hparams.f_norm_rms_eps, hparams.f_qk_clip, hparams.f_relative_clip, hparams.situ_beta, hparams.situ_linear_beta}) {
|
||||
if (!std::isfinite(value) || value <= 0.0f) {
|
||||
throw std::runtime_error("inkling2: normalization, clipping and scaling parameters must be finite and positive");
|
||||
}
|
||||
}
|
||||
|
||||
if (!std::isfinite(hparams.f_log_scale_alpha) || hparams.f_log_scale_alpha < 0.0f) {
|
||||
throw std::runtime_error("inkling2: invalid logarithmic scale");
|
||||
}
|
||||
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
|
||||
hparams.expert_router_f32 = true;
|
||||
|
||||
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
|
||||
hparams.expert_router_f32 = true;
|
||||
hparams.expert_weights_norm_eps = 1e-20f;
|
||||
|
||||
// no RoPE, the relative-position bias carries the positions
|
||||
hparams.rope_pattern.fill(false);
|
||||
uint64_t max_channels = 0;
|
||||
|
||||
// the convolution state keeps K, V, the attention output and the FFN output of the widest layer
|
||||
uint64_t n_channels_max = 0;
|
||||
|
||||
for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
|
||||
if (!hparams.n_head_kv(il) || hparams.n_head(il) % hparams.n_head_kv(il) ||
|
||||
hparams.n_embd_head_k(il) != hparams.n_embd_head_v(il) || !hparams.n_expert_used(il) ||
|
||||
hparams.n_expert_used(il) > hparams.n_expert || !hparams.n_ff_exp(il)) {
|
||||
throw std::runtime_error("inkling2: invalid attention or expert dimensions");
|
||||
}
|
||||
max_channels = std::max<uint64_t>(max_channels, 2ull * hparams.n_embd_k_gqa(il) + 2ull * hparams.n_embd);
|
||||
|
||||
n_channels_max = std::max<uint64_t>(n_channels_max, 2ull * hparams.n_embd_k_gqa(il) + 2ull * hparams.n_embd);
|
||||
}
|
||||
if (max_channels * (hparams.n_shortconv_l_cache - 1) > UINT32_MAX) {
|
||||
|
||||
if (n_channels_max * (hparams.n_shortconv_l_cache - 1) > UINT32_MAX) {
|
||||
throw std::runtime_error("inkling2: convolution state is too large");
|
||||
}
|
||||
hparams.n_sconv_channels = max_channels;
|
||||
|
||||
hparams.n_sconv_channels = n_channels_max;
|
||||
|
||||
type = LLM_TYPE_UNKNOWN;
|
||||
}
|
||||
|
||||
void llama_model_inkling2::load_arch_tensors(llama_model_loader &) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
if (hparams.n_vocab_unpadded != vocab.n_tokens()) {
|
||||
throw std::runtime_error("inkling2: tokenizer size must equal the unpadded vocabulary size");
|
||||
}
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, hparams.n_vocab_padded}, 0);
|
||||
|
||||
const int64_t n_vocab_padded = hparams.n_vocab_padded;
|
||||
const int64_t n_rel = hparams.n_relative;
|
||||
const int64_t n_conv = hparams.n_shortconv_l_cache;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab_padded}, 0);
|
||||
tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0);
|
||||
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, hparams.n_vocab_padded}, 0);
|
||||
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
|
||||
|
||||
// output
|
||||
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab_padded}, 0);
|
||||
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
|
||||
output_res_query = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_QUERY, "weight"), {n_embd}, 0);
|
||||
|
||||
for (int i = 0; i < n_layer; ++i) {
|
||||
auto & l = layers[i];
|
||||
const int64_t nh = hparams.n_head(i);
|
||||
const int64_t nk = hparams.n_head_kv(i);
|
||||
const int64_t hd = hparams.n_embd_head_k(i);
|
||||
const int64_t nr = hparams.n_relative;
|
||||
const int64_t kernel = hparams.n_shortconv_l_cache;
|
||||
const int64_t extent = hparams.is_swa(i) ? hparams.n_swa : hparams.n_relative_extent;
|
||||
l.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, nh*hd}, 0);
|
||||
l.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, nk*hd}, 0);
|
||||
l.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, nk*hd}, 0);
|
||||
l.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {nh*hd, n_embd}, 0);
|
||||
l.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, nh*hd}, 0);
|
||||
l.attn_rel = create_tensor(tn(LLM_TENSOR_ATTN_REL, "weight", i), {n_embd, nh*nr}, 0);
|
||||
l.attn_rel_bias = create_tensor(tn(LLM_TENSOR_ATTN_REL_BIAS, "weight", i), {nr, extent}, 0);
|
||||
l.attn_q_clip = create_tensor(tn(LLM_TENSOR_ATTN_Q_CLIP, "weight", i), {hd, nh}, 0);
|
||||
l.attn_rel_clip = create_tensor(tn(LLM_TENSOR_ATTN_REL_CLIP, "weight", i), {nr, nh}, 0);
|
||||
l.attn_sink_proj = create_tensor(tn(LLM_TENSOR_ATTN_SINK_PROJ, "weight", i), {n_embd, nh}, 0);
|
||||
l.attn_k_conv = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV, "weight", i), {kernel, nk*hd}, 0);
|
||||
l.attn_v_conv = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV, "weight", i), {kernel, nk*hd}, 0);
|
||||
l.attn_post_conv = create_tensor(tn(LLM_TENSOR_ATTN_POST_CONV, "weight", i), {kernel, n_embd}, 0);
|
||||
l.ffn_post_conv = create_tensor(tn(LLM_TENSOR_FFN_POST_CONV, "weight", i), {kernel, n_embd}, 0);
|
||||
auto & layer = layers[i];
|
||||
|
||||
const int64_t n_head = hparams.n_head(i);
|
||||
const int64_t n_head_kv = hparams.n_head_kv(i);
|
||||
const int64_t n_embd_head = hparams.n_embd_head_k(i);
|
||||
const int64_t n_ff_exp = hparams.n_ff_exp(i);
|
||||
const int64_t n_extent = hparams.is_swa(i) ? hparams.n_swa : hparams.n_relative_extent;
|
||||
|
||||
// attention
|
||||
layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head*n_head}, 0);
|
||||
layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_head*n_head_kv}, 0);
|
||||
layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_head*n_head_kv}, 0);
|
||||
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head*n_head, n_embd}, 0);
|
||||
layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_embd_head*n_head}, 0);
|
||||
|
||||
layer.attn_q_clip = create_tensor(tn(LLM_TENSOR_ATTN_Q_CLIP, "weight", i), {n_embd_head, n_head}, 0);
|
||||
layer.attn_sink_proj = create_tensor(tn(LLM_TENSOR_ATTN_SINK_PROJ, "weight", i), {n_embd, n_head}, 0);
|
||||
|
||||
// relative-position bias
|
||||
layer.attn_rel = create_tensor(tn(LLM_TENSOR_ATTN_REL, "weight", i), {n_embd, n_rel*n_head}, 0);
|
||||
layer.attn_rel_bias = create_tensor(tn(LLM_TENSOR_ATTN_REL_BIAS, "weight", i), {n_rel, n_extent}, 0);
|
||||
layer.attn_rel_clip = create_tensor(tn(LLM_TENSOR_ATTN_REL_CLIP, "weight", i), {n_rel, n_head}, 0);
|
||||
|
||||
// short convolutions
|
||||
layer.attn_k_conv = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV, "weight", i), {n_conv, n_embd_head*n_head_kv}, 0);
|
||||
layer.attn_v_conv = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV, "weight", i), {n_conv, n_embd_head*n_head_kv}, 0);
|
||||
layer.attn_post_conv = create_tensor(tn(LLM_TENSOR_ATTN_POST_CONV, "weight", i), {n_conv, n_embd}, 0);
|
||||
layer.ffn_post_conv = create_tensor(tn(LLM_TENSOR_FFN_POST_CONV, "weight", i), {n_conv, n_embd}, 0);
|
||||
|
||||
if (hparams.is_swa(i)) {
|
||||
l.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
|
||||
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
|
||||
} else {
|
||||
l.attn_q_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_INP_NORM, "weight", i), {n_embd}, 0);
|
||||
l.attn_k_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_INP_NORM, "weight", i), {n_embd}, 0);
|
||||
l.attn_v_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_V_INP_NORM, "weight", i), {n_embd}, 0);
|
||||
l.attn_k_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV_PROJ, "weight", i), {n_embd, kernel*nk}, 0);
|
||||
l.attn_v_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV_PROJ, "weight", i), {n_embd, kernel*nk}, 0);
|
||||
// full-attention layers: separate Q, K and V input norms, and input-dependent taps for the K and V convolutions
|
||||
layer.attn_q_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_INP_NORM, "weight", i), {n_embd}, 0);
|
||||
layer.attn_k_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_INP_NORM, "weight", i), {n_embd}, 0);
|
||||
layer.attn_v_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_V_INP_NORM, "weight", i), {n_embd}, 0);
|
||||
layer.attn_k_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV_PROJ, "weight", i), {n_embd, n_conv*n_head_kv}, 0);
|
||||
layer.attn_v_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV_PROJ, "weight", i), {n_embd, n_conv*n_head_kv}, 0);
|
||||
}
|
||||
|
||||
// block attention residuals
|
||||
if (i > 0) {
|
||||
l.attn_res_query = create_tensor(tn(LLM_TENSOR_ATTN_RES_QUERY, "weight", i), {n_embd, hparams.is_swa(i) ? 1 : 3}, 0);
|
||||
layer.attn_res_query = create_tensor(tn(LLM_TENSOR_ATTN_RES_QUERY, "weight", i), {n_embd, hparams.is_swa(i) ? 1 : 3}, 0);
|
||||
}
|
||||
l.ffn_res_query = create_tensor(tn(LLM_TENSOR_FFN_RES_QUERY, "weight", i), {n_embd}, 0);
|
||||
l.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
|
||||
l.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
|
||||
l.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
|
||||
l.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, 2*hparams.n_ff_exp(i), n_expert}, 0);
|
||||
l.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp(i), n_embd, n_expert}, 0);
|
||||
layer.ffn_res_query = create_tensor(tn(LLM_TENSOR_FFN_RES_QUERY, "weight", i), {n_embd}, 0);
|
||||
|
||||
// MoE
|
||||
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
|
||||
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
|
||||
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
|
||||
layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, 2*n_ff_exp, n_expert}, 0);
|
||||
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,202 +160,250 @@ std::unique_ptr<llm_graph_context> llama_model_inkling2::build_arch_graph(const
|
||||
return std::make_unique<graph>(params);
|
||||
}
|
||||
|
||||
class llm_graph_input_inkling2_pos : public llm_graph_input_i {
|
||||
public:
|
||||
const llama_kv_cache_context * cache[2];
|
||||
ggml_tensor * distance[2];
|
||||
uint32_t extent[2];
|
||||
|
||||
bool can_reuse(const llm_graph_params & params) override {
|
||||
const auto * memory = static_cast<const llama_memory_hybrid_iswa_context *>(params.mctx);
|
||||
cache[0] = memory->get_attn()->get_base();
|
||||
cache[1] = memory->get_attn()->get_swa();
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
if (distance[i]->ne[0] != cache[i]->get_n_kv() || distance[i]->ne[1] != params.ubatch.n_tokens) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void set_input(const llama_ubatch * ubatch) override {
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
if (distance[i]->buffer) {
|
||||
cache[i]->set_input_relative_pos(distance[i], ubatch, extent[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// scale each row down to an L2 norm of at most threshold, after the optional per-channel scale
|
||||
ggml_tensor * llama_model_inkling2::graph::clip(ggml_tensor * cur, ggml_tensor * scale, float threshold) {
|
||||
if (scale) {
|
||||
cur = ggml_mul(ctx0, cur, scale);
|
||||
}
|
||||
auto * norm = ggml_sqrt(ctx0, ggml_sum_rows(ctx0, ggml_sqr(ctx0, cur)));
|
||||
|
||||
ggml_tensor * norm = ggml_sqrt(ctx0, ggml_sum_rows(ctx0, ggml_sqr(ctx0, cur)));
|
||||
norm = ggml_scale_bias(ctx0, norm, 1.0f, 1e-6f);
|
||||
auto * factor = ggml_div(ctx0, ggml_fill(ctx0, norm, threshold), norm);
|
||||
|
||||
ggml_tensor * factor = ggml_div(ctx0, ggml_fill(ctx0, norm, threshold), norm);
|
||||
|
||||
return ggml_mul(ctx0, cur, ggml_clamp(ctx0, factor, 0.0f, 1.0f));
|
||||
}
|
||||
|
||||
// attention over the depth: mix the earlier block outputs and cur with learned queries
|
||||
ggml_tensor * llama_model_inkling2::graph::mix(const std::vector<ggml_tensor *> & blocks, ggml_tensor * cur, ggml_tensor * query) {
|
||||
auto * states = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
|
||||
for (auto * block : blocks) {
|
||||
ggml_tensor * states = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
|
||||
for (ggml_tensor * block : blocks) {
|
||||
states = ggml_concat(ctx0, ggml_reshape_3d(ctx0, block, n_embd, 1, n_tokens), states, 1);
|
||||
}
|
||||
states = ggml_rms_norm(ctx0, states, 1e-6f);
|
||||
auto * queries = ggml_repeat(ctx0, query, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd, query->ne[1], n_tokens));
|
||||
auto * scores = ggml_mul_mat(ctx0, states, queries);
|
||||
|
||||
ggml_tensor * queries = ggml_repeat(ctx0, query, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd, query->ne[1], n_tokens));
|
||||
|
||||
ggml_tensor * scores = ggml_mul_mat(ctx0, states, queries);
|
||||
scores = ggml_soft_max_ext(ctx0, scores, nullptr, 1.0f/sqrtf(n_embd), 0.0f);
|
||||
|
||||
return ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, states)), scores);
|
||||
}
|
||||
|
||||
ggml_tensor * llama_model_inkling2::graph::convolve(ggml_tensor * cur, ggml_tensor * weight, ggml_tensor * dynamic,
|
||||
ggml_tensor * state, ggml_tensor * history, llm_graph_input_rs * inp, int64_t offset) {
|
||||
const int64_t kernel = hparams.n_shortconv_l_cache;
|
||||
const int64_t channels = cur->ne[0];
|
||||
const int64_t nt = ubatch.n_seq_tokens;
|
||||
const int64_t ns = ubatch.n_seqs;
|
||||
auto * past = ggml_view_3d(ctx0, history, kernel - 1, channels, ns, history->nb[1], history->nb[2], offset*history->nb[1]);
|
||||
auto * x = ggml_transpose(ctx0, ggml_reshape_3d(ctx0, cur, channels, nt, ns));
|
||||
// causal short convolution with a residual; dynamic adds input-dependent taps per head
|
||||
// the last n_conv - 1 inputs of each sequence live in the recurrent state at channel offset `offset`
|
||||
ggml_tensor * llama_model_inkling2::graph::convolve(
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * weight,
|
||||
ggml_tensor * dynamic,
|
||||
ggml_tensor * state,
|
||||
ggml_tensor * history,
|
||||
llm_graph_input_rs * inp,
|
||||
int64_t offset) {
|
||||
const int64_t n_conv = hparams.n_shortconv_l_cache;
|
||||
const int64_t n_channels = cur->ne[0];
|
||||
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
|
||||
const int64_t n_seqs = ubatch.n_seqs;
|
||||
|
||||
ggml_tensor * past = ggml_view_3d(ctx0, history, n_conv - 1, n_channels, n_seqs, history->nb[1], history->nb[2], offset*history->nb[1]);
|
||||
|
||||
ggml_tensor * x = ggml_transpose(ctx0, ggml_reshape_3d(ctx0, cur, n_channels, n_seq_tokens, n_seqs));
|
||||
x = ggml_concat(ctx0, past, x, 0);
|
||||
build_conv_state_store(inp, state, x, kernel - 1, offset*(kernel - 1)*ggml_element_size(state));
|
||||
auto * out = ggml_ssm_conv(ctx0, x, weight);
|
||||
out = ggml_reshape_2d(ctx0, out, channels, n_tokens);
|
||||
|
||||
build_conv_state_store(inp, state, x, n_conv - 1, offset*(n_conv - 1)*ggml_element_size(state));
|
||||
|
||||
ggml_tensor * out = ggml_ssm_conv(ctx0, x, weight);
|
||||
out = ggml_reshape_2d(ctx0, out, n_channels, n_tokens);
|
||||
|
||||
if (dynamic) {
|
||||
const int64_t heads = dynamic->ne[0]/kernel;
|
||||
dynamic = ggml_reshape_3d(ctx0, dynamic, kernel, heads, n_tokens);
|
||||
for (int64_t lag = 0; lag < kernel; ++lag) {
|
||||
auto * tap = ggml_view_3d(ctx0, dynamic, 1, heads, n_tokens, dynamic->nb[1], dynamic->nb[2], lag*dynamic->nb[0]);
|
||||
auto * window = ggml_view_3d(ctx0, x, nt, channels, ns, x->nb[1], x->nb[2], (kernel - 1 - lag)*x->nb[0]);
|
||||
const int64_t n_heads = dynamic->ne[0]/n_conv;
|
||||
|
||||
dynamic = ggml_reshape_3d(ctx0, dynamic, n_conv, n_heads, n_tokens);
|
||||
|
||||
for (int64_t lag = 0; lag < n_conv; ++lag) {
|
||||
ggml_tensor * tap = ggml_view_3d(ctx0, dynamic, 1, n_heads, n_tokens, dynamic->nb[1], dynamic->nb[2], lag*dynamic->nb[0]);
|
||||
ggml_tensor * window = ggml_view_3d(ctx0, x, n_seq_tokens, n_channels, n_seqs, x->nb[1], x->nb[2], (n_conv - 1 - lag)*x->nb[0]);
|
||||
|
||||
window = ggml_cont(ctx0, ggml_transpose(ctx0, window));
|
||||
window = ggml_reshape_3d(ctx0, window, channels/heads, heads, n_tokens);
|
||||
auto * term = ggml_reshape_2d(ctx0, ggml_mul(ctx0, window, tap), channels, n_tokens);
|
||||
window = ggml_reshape_3d(ctx0, window, n_channels/n_heads, n_heads, n_tokens);
|
||||
|
||||
ggml_tensor * term = ggml_reshape_2d(ctx0, ggml_mul(ctx0, window, tap), n_channels, n_tokens);
|
||||
|
||||
out = ggml_add(ctx0, out, term);
|
||||
}
|
||||
}
|
||||
return ggml_scale(ctx0, ggml_add(ctx0, cur, out), 1.0f/sqrtf((dynamic ? 2 : 1)*kernel + 1));
|
||||
|
||||
return ggml_scale(ctx0, ggml_add(ctx0, cur, out), 1.0f/sqrtf((dynamic ? 2 : 1)*n_conv + 1));
|
||||
}
|
||||
|
||||
llama_model_inkling2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) {
|
||||
auto * inp = build_inp_mem_hybrid_iswa();
|
||||
auto * inp = build_inp_mem_hybrid_iswa_rel(hparams.n_relative_extent, hparams.n_swa);
|
||||
auto * attn = inp->get_attn();
|
||||
auto * recr = inp->get_recr();
|
||||
auto * positions = build_inp_pos();
|
||||
auto * out_ids = build_inp_out_ids();
|
||||
auto rel = std::make_unique<llm_graph_input_inkling2_pos>();
|
||||
rel->cache[0] = attn->mctx->get_base();
|
||||
rel->cache[1] = attn->mctx->get_swa();
|
||||
rel->extent[0] = hparams.n_relative_extent;
|
||||
rel->extent[1] = hparams.n_swa;
|
||||
ggml_tensor * distances[2];
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
distances[i] = rel->distance[i] = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, rel->cache[i]->get_n_kv(), n_tokens);
|
||||
ggml_set_input(distances[i]);
|
||||
}
|
||||
res->add_input(std::move(rel));
|
||||
auto * tau = ggml_scale_bias(ctx0, ggml_cast(ctx0, positions, GGML_TYPE_F32), 1.0f, 1.0f);
|
||||
|
||||
ggml_tensor * inp_pos = build_inp_pos();
|
||||
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
||||
|
||||
// log-scaled positions: 1 + alpha*log(max(1, (pos + 1)/floor)), multiplies q and r of the full-attention layers
|
||||
ggml_tensor * tau = ggml_scale_bias(ctx0, ggml_cast(ctx0, inp_pos, GGML_TYPE_F32), 1.0f, 1.0f);
|
||||
tau = ggml_log(ctx0, ggml_clamp(ctx0, ggml_scale(ctx0, tau, 1.0f/hparams.n_log_scale_floor), 1.0f, INFINITY));
|
||||
tau = ggml_reshape_3d(ctx0, ggml_scale_bias(ctx0, tau, hparams.f_log_scale_alpha, 1.0f), 1, 1, n_tokens);
|
||||
auto * cur = build_inp_embd(model.tok_embd);
|
||||
|
||||
ggml_tensor * cur = build_inp_embd(model.tok_embd);
|
||||
|
||||
// important: do not normalize raw embeddings input (i.e. encoded image embeddings)
|
||||
if (ubatch.token) {
|
||||
cur = build_norm(cur, model.tok_norm, nullptr, LLM_NORM_RMS, -1);
|
||||
}
|
||||
|
||||
std::vector<ggml_tensor *> blocks;
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
const auto & l = model.layers[il];
|
||||
const bool swa = hparams.is_swa(il);
|
||||
const int64_t nh = hparams.n_head(il);
|
||||
const int64_t nk = hparams.n_head_kv(il);
|
||||
const int64_t hd = hparams.n_embd_head_k(il);
|
||||
const int64_t kv = nk*hd;
|
||||
auto * state = recr->mctx->get_r_l(il);
|
||||
auto * history = build_rs(recr, state, hparams.n_embd_r(), ubatch.n_seqs);
|
||||
const auto & layer = model.layers[il];
|
||||
|
||||
const bool is_swa = hparams.is_swa(il);
|
||||
const int64_t n_head = hparams.n_head(il);
|
||||
const int64_t n_head_kv = hparams.n_head_kv(il);
|
||||
const int64_t n_embd_head = hparams.n_embd_head_k(il);
|
||||
const int64_t n_embd_k_gqa = n_embd_head*n_head_kv;
|
||||
|
||||
ggml_tensor * state = recr->mctx->get_r_l(il);
|
||||
ggml_tensor * history = build_rs(recr, state, hparams.n_embd_r(), ubatch.n_seqs);
|
||||
history = ggml_reshape_3d(ctx0, history, hparams.n_shortconv_l_cache - 1, hparams.n_sconv_channels, ubatch.n_seqs);
|
||||
auto * residual = cur;
|
||||
|
||||
ggml_tensor * inpSA = cur;
|
||||
|
||||
// attention inputs: SWA layers mix one, full-attention layers one each for Q, K and V
|
||||
ggml_tensor * inputs[3] = {cur, cur, cur};
|
||||
if (l.attn_res_query) {
|
||||
auto * mixed = mix(blocks, cur, l.attn_res_query);
|
||||
if (layer.attn_res_query) {
|
||||
ggml_tensor * mixed = mix(blocks, cur, layer.attn_res_query);
|
||||
for (int j = 0; j < 3; ++j) {
|
||||
inputs[j] = ggml_cont(ctx0, ggml_view_2d(ctx0, mixed, n_embd, n_tokens, mixed->nb[2], (swa ? 0 : j)*mixed->nb[1]));
|
||||
inputs[j] = ggml_cont(ctx0, ggml_view_2d(ctx0, mixed, n_embd, n_tokens, mixed->nb[2], (is_swa ? 0 : j)*mixed->nb[1]));
|
||||
}
|
||||
}
|
||||
if (swa) {
|
||||
inputs[0] = build_norm(inputs[0], l.attn_norm, nullptr, LLM_NORM_RMS, il);
|
||||
|
||||
if (is_swa) {
|
||||
inputs[0] = build_norm(inputs[0], layer.attn_norm, nullptr, LLM_NORM_RMS, il);
|
||||
inputs[1] = inputs[2] = inputs[0];
|
||||
} else {
|
||||
inputs[0] = build_norm(inputs[0], l.attn_q_inp_norm, nullptr, LLM_NORM_RMS, il);
|
||||
inputs[1] = build_norm(inputs[1], l.attn_k_inp_norm, nullptr, LLM_NORM_RMS, il);
|
||||
inputs[2] = build_norm(inputs[2], l.attn_v_inp_norm, nullptr, LLM_NORM_RMS, il);
|
||||
inputs[0] = build_norm(inputs[0], layer.attn_q_inp_norm, nullptr, LLM_NORM_RMS, il);
|
||||
inputs[1] = build_norm(inputs[1], layer.attn_k_inp_norm, nullptr, LLM_NORM_RMS, il);
|
||||
inputs[2] = build_norm(inputs[2], layer.attn_v_inp_norm, nullptr, LLM_NORM_RMS, il);
|
||||
}
|
||||
auto * q = build_lora_mm(l.wq, inputs[0]);
|
||||
auto * k = convolve(build_lora_mm(l.wk, inputs[1]), l.attn_k_conv,
|
||||
swa ? nullptr : build_lora_mm(l.attn_k_conv_proj, inputs[1]), state, history, recr, 0);
|
||||
auto * v = convolve(build_lora_mm(l.wv, inputs[2]), l.attn_v_conv,
|
||||
swa ? nullptr : build_lora_mm(l.attn_v_conv_proj, inputs[2]), state, history, recr, kv);
|
||||
auto * r = build_lora_mm(l.attn_rel, inputs[0]);
|
||||
q = clip(ggml_reshape_3d(ctx0, q, hd, nh, n_tokens), l.attn_q_clip, hparams.f_qk_clip);
|
||||
k = clip(ggml_reshape_3d(ctx0, k, hd, nk, n_tokens), nullptr, hparams.f_qk_clip);
|
||||
v = ggml_rms_norm(ctx0, ggml_reshape_3d(ctx0, v, hd, nk, n_tokens), hparams.f_norm_rms_eps);
|
||||
r = clip(ggml_reshape_3d(ctx0, r, hparams.n_relative, nh, n_tokens), l.attn_rel_clip, hparams.f_relative_clip);
|
||||
if (!swa) {
|
||||
q = ggml_mul(ctx0, q, tau);
|
||||
r = ggml_mul(ctx0, r, tau);
|
||||
|
||||
// self-attention
|
||||
{
|
||||
ggml_tensor * Qcur = build_lora_mm(layer.wq, inputs[0]);
|
||||
ggml_tensor * Kcur = convolve(build_lora_mm(layer.wk, inputs[1]), layer.attn_k_conv,
|
||||
is_swa ? nullptr : build_lora_mm(layer.attn_k_conv_proj, inputs[1]), state, history, recr, 0);
|
||||
ggml_tensor * Vcur = convolve(build_lora_mm(layer.wv, inputs[2]), layer.attn_v_conv,
|
||||
is_swa ? nullptr : build_lora_mm(layer.attn_v_conv_proj, inputs[2]), state, history, recr, n_embd_k_gqa);
|
||||
ggml_tensor * Rcur = build_lora_mm(layer.attn_rel, inputs[0]);
|
||||
|
||||
Qcur = clip(ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), layer.attn_q_clip, hparams.f_qk_clip);
|
||||
Kcur = clip(ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), nullptr, hparams.f_qk_clip);
|
||||
Vcur = ggml_rms_norm(ctx0, ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens), hparams.f_norm_rms_eps);
|
||||
Rcur = clip(ggml_reshape_3d(ctx0, Rcur, hparams.n_relative, n_head, n_tokens), layer.attn_rel_clip, hparams.f_relative_clip);
|
||||
|
||||
if (!is_swa) {
|
||||
Qcur = ggml_mul(ctx0, Qcur, tau);
|
||||
Rcur = ggml_mul(ctx0, Rcur, tau);
|
||||
}
|
||||
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
cb(Rcur, "Rcur", il);
|
||||
|
||||
ggml_tensor * rel_pos = is_swa ? inp->self_rel_pos_swa : inp->self_rel_pos;
|
||||
|
||||
const int64_t n_stream = (is_swa ? attn->self_attn_mask_swa.raw : attn->self_attn_mask.raw)->ne[3];
|
||||
const int64_t n_kv = rel_pos->ne[0];
|
||||
|
||||
ggml_tensor * sinks = build_lora_mm(layer.attn_sink_proj, inputs[0]);
|
||||
|
||||
// project r on the bias table first, then gather by distance: only the gathered bias is n_kv x n_tokens x n_head
|
||||
ggml_tensor * table = ggml_mul_mat(ctx0, ggml_pad(ctx0, layer.attn_rel_bias, 0, 1, 0, 0), Rcur);
|
||||
|
||||
// the sink logit t is per query: exp(s - t)/(1 + sum(exp(s - t))) = exp(s)/(exp(t) + sum(exp(s))), with a zero sink
|
||||
table = ggml_sub(ctx0, table, ggml_reshape_3d(ctx0, sinks, 1, n_head, n_tokens));
|
||||
table = ggml_cont(ctx0, ggml_permute(ctx0, table, 1, 0, 2, 3));
|
||||
|
||||
ggml_tensor * kq_b = ggml_get_rows(ctx0, table, rel_pos);
|
||||
kq_b = ggml_reshape_4d(ctx0, kq_b, n_head, n_kv, n_tokens/n_stream, n_stream);
|
||||
kq_b = ggml_cont(ctx0, ggml_permute(ctx0, kq_b, 2, 0, 1, 3));
|
||||
cb(kq_b, "kq_b", il);
|
||||
|
||||
sinks = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_head), 0.0f);
|
||||
|
||||
cur = build_attn(inp,
|
||||
nullptr, nullptr, nullptr,
|
||||
Qcur, Kcur, Vcur, kq_b, sinks, 1.0f/sqrtf(n_embd_head), il);
|
||||
|
||||
cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, build_lora_mm(layer.wqkv_gate, inputs[0])));
|
||||
cur = convolve(build_lora_mm(layer.wo, cur), layer.attn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa);
|
||||
cb(cur, "attn_out", il);
|
||||
}
|
||||
auto * mask = swa ? attn->self_attn_mask_swa.raw : attn->self_attn_mask.raw;
|
||||
const int64_t streams = mask->ne[3];
|
||||
const int64_t n_kv = distances[swa]->ne[0];
|
||||
auto * sinks = build_lora_mm(l.attn_sink_proj, inputs[0]);
|
||||
// project r on the bias table first, then gather by distance: only the gathered bias is n_kv x n_tokens x nh
|
||||
auto * table = ggml_mul_mat(ctx0, ggml_pad(ctx0, l.attn_rel_bias, 0, 1, 0, 0), r);
|
||||
if (cparams.flash_attn) {
|
||||
// exp(s - t)/(1 + sum(exp(s - t))) = exp(s)/(exp(t) + sum(exp(s))).
|
||||
table = ggml_sub(ctx0, table, ggml_reshape_3d(ctx0, sinks, 1, nh, n_tokens));
|
||||
} else {
|
||||
// build_attn scales the bias along with QK; FA adds the mask after scaling QK.
|
||||
table = ggml_scale(ctx0, table, sqrtf(hd));
|
||||
}
|
||||
table = ggml_cont(ctx0, ggml_permute(ctx0, table, 1, 0, 2, 3));
|
||||
auto * bias = ggml_get_rows(ctx0, table, distances[swa]);
|
||||
bias = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_4d(ctx0, bias, nh, n_kv, n_tokens/streams, streams), 2, 0, 1, 3));
|
||||
ggml_tensor * fa_mask = nullptr;
|
||||
if (cparams.flash_attn) {
|
||||
fa_mask = ggml_cast(ctx0, ggml_add(ctx0, bias, ggml_cast(ctx0, mask, GGML_TYPE_F32)), GGML_TYPE_F16);
|
||||
sinks = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, nh), 0.0f);
|
||||
bias = nullptr;
|
||||
} else {
|
||||
sinks = ggml_reshape_4d(ctx0, ggml_cont(ctx0, ggml_permute(ctx0,
|
||||
ggml_reshape_3d(ctx0, sinks, nh, n_tokens/streams, streams), 1, 0, 2, 3)), 1, n_tokens/streams, nh, streams);
|
||||
}
|
||||
cur = build_attn(attn, nullptr, nullptr, nullptr, q, k, v, bias, sinks, nullptr, 1.0f/sqrtf(hd), il, fa_mask);
|
||||
cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, build_lora_mm(l.wqkv_gate, inputs[0])));
|
||||
cur = convolve(build_lora_mm(l.wo, cur), l.attn_post_conv, nullptr, state, history, recr, 2*kv);
|
||||
|
||||
// the first layer of each block starts a new residual stream, the earlier one joins the mixed blocks
|
||||
if (il % hparams.attn_res_block_size == 0) {
|
||||
blocks.push_back(residual);
|
||||
blocks.push_back(inpSA);
|
||||
} else {
|
||||
cur = ggml_add(ctx0, residual, cur);
|
||||
cur = ggml_add(ctx0, inpSA, cur);
|
||||
}
|
||||
residual = cur;
|
||||
cur = ggml_reshape_2d(ctx0, mix(blocks, cur, l.ffn_res_query), n_embd, n_tokens);
|
||||
cur = build_norm(cur, l.ffn_norm, nullptr, LLM_NORM_RMS, il);
|
||||
cur = build_moe_ffn(cur, l.ffn_gate_inp, nullptr, nullptr, l.ffn_down_exps, l.ffn_exp_probs_b,
|
||||
hparams.n_expert, hparams.n_expert_used(il), LLM_FFN_SITU, true, hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID, il, nullptr, l.ffn_gate_up_exps);
|
||||
cur = convolve(cur, l.ffn_post_conv, nullptr, state, history, recr, 2*kv + n_embd);
|
||||
cur = ggml_add(ctx0, residual, cur);
|
||||
|
||||
ggml_tensor * ffn_inp = cur;
|
||||
cb(ffn_inp, "ffn_inp", il);
|
||||
|
||||
// MoE
|
||||
cur = ggml_reshape_2d(ctx0, mix(blocks, cur, layer.ffn_res_query), n_embd, n_tokens);
|
||||
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
|
||||
|
||||
cur = build_moe_ffn(cur,
|
||||
layer.ffn_gate_inp,
|
||||
nullptr,
|
||||
nullptr,
|
||||
layer.ffn_down_exps,
|
||||
layer.ffn_exp_probs_b,
|
||||
hparams.n_expert,
|
||||
hparams.n_expert_used(il),
|
||||
LLM_FFN_SITU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
|
||||
il,
|
||||
nullptr,
|
||||
layer.ffn_gate_up_exps);
|
||||
|
||||
cur = convolve(cur, layer.ffn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa + n_embd);
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
cur = ggml_add(ctx0, ffn_inp, cur);
|
||||
cb(cur, "l_out", il);
|
||||
}
|
||||
|
||||
cur = ggml_reshape_2d(ctx0, mix(blocks, cur, model.output_res_query), n_embd, n_tokens);
|
||||
cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
|
||||
if (out_ids) {
|
||||
cur = ggml_get_rows(ctx0, cur, out_ids);
|
||||
|
||||
if (inp_out_ids) {
|
||||
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
||||
}
|
||||
|
||||
cb(cur, "result_norm", -1);
|
||||
res->t_embd = cur;
|
||||
|
||||
// lm_head
|
||||
cur = build_lora_mm(model.output, cur);
|
||||
ggml_prec_set_acc(cur, GGML_PREC_F32);
|
||||
const float cap = hparams.f_final_logit_softcapping;
|
||||
cur = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, cur, 1.0f/cap)), cap);
|
||||
|
||||
if (hparams.f_final_logit_softcapping != 0.0f) {
|
||||
const float cap = hparams.f_final_logit_softcapping;
|
||||
|
||||
cur = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, cur, 1.0f/cap)), cap);
|
||||
}
|
||||
|
||||
// drop the padded vocabulary
|
||||
cur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, hparams.n_vocab_unpadded, cur->ne[1], cur->nb[1], 0));
|
||||
res->t_logits = cur;
|
||||
|
||||
cb(cur, "result_output", -1);
|
||||
res->t_logits = cur;
|
||||
}
|
||||
|
||||
@@ -18,6 +18,12 @@ void llama_model_limite::load_arch_hparams(llama_model_loader & ml) {
|
||||
std::fill(hparams.mudd_tap_idx.begin(), hparams.mudd_tap_idx.end(), -1);
|
||||
ml.get_key(LLM_KV_MUDD_TAP_COUNT, hparams.n_mudd_taps, false);
|
||||
if (hparams.n_mudd_taps > 0) {
|
||||
// reject metadata where n_layer * n_mudd_taps would index past mudd_tap_idx
|
||||
if (hparams.n_mudd_taps > LLAMA_MAX_LAYERS || (uint64_t) hparams.n_layer() * hparams.n_mudd_taps > LLAMA_MAX_LAYERS) {
|
||||
throw std::runtime_error(format(
|
||||
"invalid limite mudd config: n_layer (%u) * n_mudd_taps (%u) exceeds LLAMA_MAX_LAYERS (%d)",
|
||||
hparams.n_layer(), hparams.n_mudd_taps, LLAMA_MAX_LAYERS));
|
||||
}
|
||||
ml.get_key(LLM_KV_MUDD_FEED_FORWARD_LENGTH, hparams.n_mudd_ff);
|
||||
ml.get_arr(LLM_KV_MUDD_TAP_INDICES, hparams.mudd_tap_idx);
|
||||
}
|
||||
|
||||
+8
-1
@@ -2480,7 +2480,14 @@ struct llama_model_inkling2 : public llama_model_base {
|
||||
graph(const llm_graph_params & params);
|
||||
ggml_tensor * mix(const std::vector<ggml_tensor *> & blocks, ggml_tensor * cur, ggml_tensor * query);
|
||||
ggml_tensor * clip(ggml_tensor * cur, ggml_tensor * scale, float threshold);
|
||||
ggml_tensor * convolve(ggml_tensor * cur, ggml_tensor * weight, ggml_tensor * dynamic, ggml_tensor * state, ggml_tensor * history, llm_graph_input_rs * inp, int64_t offset);
|
||||
ggml_tensor * convolve(
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * weight,
|
||||
ggml_tensor * dynamic,
|
||||
ggml_tensor * state,
|
||||
ggml_tensor * history,
|
||||
llm_graph_input_rs * inp,
|
||||
int64_t offset);
|
||||
};
|
||||
|
||||
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
|
||||
|
||||
@@ -85,6 +85,8 @@ llama_model_qwen3vl::graph::graph(const llm_graph_params & params) : llm_graph_c
|
||||
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
res->t_layer_inp[il] = inpL;
|
||||
|
||||
ggml_tensor * inpSA = inpL;
|
||||
|
||||
// norm
|
||||
|
||||
@@ -367,6 +367,8 @@ endif()
|
||||
llama_build(test-backend-ops.cpp LINK ggml Threads::Threads)
|
||||
|
||||
llama_build_and_test(test-backend-tensor.cpp LINK ggml)
|
||||
llama_build_and_test(test-sleep.cpp LINK ggml)
|
||||
llama_build_and_test(test-backend-sched.cpp LINK ggml)
|
||||
|
||||
if (BUILD_SHARED_LIBS)
|
||||
llama_build_and_test(test-llama-mmap.cpp ${PROJECT_SOURCE_DIR}/src/llama-impl.cpp ${PROJECT_SOURCE_DIR}/src/llama-mmap.cpp LABEL behavior)
|
||||
|
||||
@@ -72,8 +72,9 @@ std::vector<case_run> run_cases(model_run & mr, const std::vector<const ctx_test
|
||||
} else {
|
||||
// the recurrent cases are also run against a cache pre-filled with a known
|
||||
// pattern, so that a restore reading cells it never wrote cannot pass
|
||||
// with 0xff repeated every float of the state becomes NaN to test that the next sequence is not affected
|
||||
|
||||
for (const uint8_t fill : { 0, 0x3e }) {
|
||||
for (const uint8_t fill : { 0, 0x3e, 0xff }) {
|
||||
TLOG_INF("(fill 0x%02x)\n", fill);
|
||||
|
||||
mr.fill = fill;
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
#include "test-log.h"
|
||||
|
||||
#include "../../src/llama-ext.h" // llama_memory_get_buffers, llama_arch_from_string
|
||||
#include "../../src/llama-ext.h" // llama_memory_get_buffers, llama_memory_get_tensors, llama_arch_from_string
|
||||
|
||||
#include "common.h"
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
#include <regex>
|
||||
#include <stdexcept>
|
||||
|
||||
const double NMSE_THRESHOLD = 1e-5;
|
||||
@@ -297,6 +298,13 @@ bool corrupt_state(std::vector<uint8_t> & data) {
|
||||
// context
|
||||
//
|
||||
|
||||
// the recurrent state tensors: the conv and ssm rows of llama_memory_recurrent and the DeepSeek V4 compressor state
|
||||
static bool is_state_tensor(const ggml_tensor * tensor) {
|
||||
static const std::regex re_state("^cache_(r|s|ple_r)_l[0-9]+$|_state_");
|
||||
|
||||
return std::regex_search(tensor->name, re_state);
|
||||
}
|
||||
|
||||
// fill the buffers of the cache before a restore - a restore that reads cells it never wrote
|
||||
// then produces garbage instead of silently passing
|
||||
llama_context_ptr init_ctx(llama_model * model, llama_context_params cparams, uint8_t fill) {
|
||||
@@ -313,11 +321,29 @@ llama_context_ptr init_ctx(llama_model * model, llama_context_params cparams, ui
|
||||
llama_synchronize(ctx.get());
|
||||
|
||||
const auto buffers = llama_memory_get_buffers(llama_get_memory(ctx.get()));
|
||||
const auto tensors = llama_memory_get_tensors(llama_get_memory(ctx.get()));
|
||||
llama_memory_clear(llama_get_memory(ctx.get()), true);
|
||||
if (buffers.empty()) {
|
||||
TLOG_ERR("%s: no cache buffers found\n", __func__);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// the KV cache only masks the free cells, so NaN there would corrupt the output
|
||||
if (fill == 0xff) {
|
||||
size_t n_state = 0;
|
||||
for (ggml_tensor * tensor : tensors) {
|
||||
if (is_state_tensor(tensor)) {
|
||||
ggml_backend_tensor_memset(tensor, fill, 0, ggml_nbytes(tensor));
|
||||
n_state++;
|
||||
}
|
||||
}
|
||||
if (n_state == 0) {
|
||||
TLOG_ERR("%s: no state tensors found\n", __func__);
|
||||
return nullptr;
|
||||
}
|
||||
return ctx;
|
||||
}
|
||||
|
||||
for (auto * buffer : buffers) {
|
||||
ggml_backend_buffer_clear(buffer, fill);
|
||||
}
|
||||
|
||||
@@ -9657,6 +9657,33 @@ struct test_fill : public test_case {
|
||||
}
|
||||
};
|
||||
|
||||
// GGML_OP_SLEEP (for scheduler testing only)
|
||||
struct test_sleep : public test_case {
|
||||
const ggml_type type;
|
||||
const std::array<int64_t, 4> ne;
|
||||
const int32_t us;
|
||||
|
||||
std::string vars() override { return VARS_TO_STR3(type, ne, us); }
|
||||
|
||||
// the op only delays, the output aliases the input and must match it exactly
|
||||
double max_nmse_err() override { return 0.0; }
|
||||
|
||||
test_sleep(int32_t us = 100, ggml_type type = GGML_TYPE_F32,
|
||||
std::array<int64_t, 4> ne = { 10, 10, 4, 3 })
|
||||
: type(type), ne(ne), us(us) {}
|
||||
|
||||
ggml_tensor * build_graph(ggml_context * ctx) override {
|
||||
ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]);
|
||||
ggml_set_name(a, "a");
|
||||
|
||||
ggml_tensor * out = ggml_sleep(ctx, a, us);
|
||||
|
||||
ggml_set_name(out, "out");
|
||||
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
// GGML_OP_SOLVE_TRI
|
||||
struct test_solve_tri : public test_case {
|
||||
const ggml_type type;
|
||||
@@ -11917,6 +11944,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
// > 2^31-element dst (8.9 GB), about 45 GB peak memory. Too large to be enabled by default.
|
||||
// kernel_mul_mm dst offset of the last batch is past INT32_MAX: rows 0-31 use full tiles, rows 32-39 use edge tiles.
|
||||
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1048576, 40, 64, {1, 1}, {53, 1}));
|
||||
// CUDA mul_mat_q: > 2^31-element dst (8.9 GB), the dst offset of column tiles starting past column 8648 does not fit in 32 bits
|
||||
// (same shape class as a 248k-vocab output head with a 16k ubatch of outputs)
|
||||
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 248320, 8960, 32, {1, 1}, {1, 1}));
|
||||
|
||||
// kernel_mul_mm_id: the index of tokens 32768-33791 does not fit in 16 bits (about 0.3 GB peak memory).
|
||||
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 4, 2, false, 64, 33792, 64));
|
||||
@@ -12262,6 +12292,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
|
||||
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 576, 512, 576, {1,1}, {1,1}));
|
||||
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 1, 2048, 8192, {1, 1}, {1, 1}));
|
||||
|
||||
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F16, GGML_TYPE_F16, 128, 128, 4352, {1, 1}, {1, 1}));
|
||||
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 64, 192, 17664, {1, 1}, {1, 1}));
|
||||
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, 128, 128, 4096, {1, 1}, {2, 1}));
|
||||
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 128, 256, 4096, {1, 1}, {1, 1}));
|
||||
|
||||
for (ggml_type type_a : all_types) {
|
||||
test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1, 64, 256, {1, 1}, {1, 1}));
|
||||
}
|
||||
@@ -12987,6 +13023,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_fill(-152.0f, GGML_TYPE_F16, { 800, 600, 4, 4 }));
|
||||
test_cases.emplace_back(new test_fill(3.5f, GGML_TYPE_F16, { 2048, 512, 2, 2 }));
|
||||
|
||||
test_cases.emplace_back(new test_sleep(0));
|
||||
test_cases.emplace_back(new test_sleep(10, GGML_TYPE_F16, { 128, 4, 2, 2 }));
|
||||
test_cases.emplace_back(new test_sleep(10, GGML_TYPE_F32, { 128, 4, 2, 2 }));
|
||||
|
||||
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
|
||||
test_cases.emplace_back(new test_diag(type));
|
||||
test_cases.emplace_back(new test_diag(type, { 79, 1, 19, 13 }));
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -170,6 +170,40 @@ static bool test_tensor_n(ggml_backend_buffer_type_t buft) {
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
std::vector<ggml_tensor *> batch;
|
||||
ggml_backend_buffer_ptr scatter_buf = alloc_tensors(ctx.get(), buft, { 64 }, batch);
|
||||
if (!scatter_buf) {
|
||||
return false;
|
||||
}
|
||||
batch.resize(128, batch[0]);
|
||||
uint8_t input[32], expected[256], out[256];
|
||||
for (size_t i = 0; i < sizeof(input); i++) {
|
||||
input[i] = (uint8_t) (i + 1);
|
||||
}
|
||||
std::vector<size_t> offsets(batch.size()), sizes(batch.size(), 1), data_offsets(batch.size());
|
||||
for (bool overlap : { false, true }) {
|
||||
memset(expected, 0, sizeof(expected));
|
||||
ggml_backend_tensor_set(batch[0], expected, 0, sizeof(expected));
|
||||
for (size_t i = 0; i < batch.size(); i++) {
|
||||
offsets[i] = 2*((i*5) % batch.size()) + 1;
|
||||
data_offsets[i] = i % sizeof(input);
|
||||
}
|
||||
// Repeated sources are valid; overlapping destinations must retain copy order.
|
||||
if (overlap) {
|
||||
offsets.back() = offsets.front();
|
||||
}
|
||||
for (size_t i = 0; i < batch.size(); i++) {
|
||||
expected[offsets[i]] = input[data_offsets[i]];
|
||||
}
|
||||
ggml_backend_tensor_set_n(batch.data(), input, offsets.data(), sizes.data(), data_offsets.data(), batch.size());
|
||||
ggml_backend_tensor_get(batch[0], out, 0, sizeof(out));
|
||||
if (memcmp(out, expected, sizeof(out)) != 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
// Tests for GGML_OP_SLEEP, on every backend that supports it. Three properties are checked:
|
||||
// - the op only delays: input = output
|
||||
// - the op does not return early: a chain of n sleeps must take at least n*us
|
||||
// Only a lower bound is asserted for the duration. A busy-wait cannot finish early, so that bound is
|
||||
// deterministic, whereas an upper bound would be flaky on a loaded machine.
|
||||
// Known limitation: Vulkan does not specify the tick rate of the shader clock, see ggml_vk_sleep. A driver
|
||||
// whose clock runs slower than assumed could undershoot the request and fail this bound.
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml-cpp.h"
|
||||
|
||||
#include <cinttypes>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <random>
|
||||
#include <vector>
|
||||
|
||||
enum test_result {
|
||||
TEST_OK,
|
||||
TEST_FAIL,
|
||||
TEST_SKIP,
|
||||
};
|
||||
|
||||
static test_result test_sleep(ggml_backend_t backend, ggml_type type, int64_t ne, int32_t us, int n_nodes) {
|
||||
const size_t graph_size = n_nodes + 16;
|
||||
|
||||
ggml_init_params params_static = {
|
||||
/*.mem_size =*/ ggml_tensor_overhead(),
|
||||
/*.mem_buffer =*/ nullptr,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ggml_context_ptr ctx_static(ggml_init(params_static));
|
||||
|
||||
ggml_tensor * x = ggml_new_tensor_1d(ctx_static.get(), type, ne);
|
||||
ggml_set_name(x, "x");
|
||||
ggml_set_input(x);
|
||||
|
||||
ggml_init_params params_compute = {
|
||||
/*.mem_size =*/ n_nodes*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false),
|
||||
/*.mem_buffer =*/ nullptr,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ggml_context_ptr ctx_compute(ggml_init(params_compute));
|
||||
ggml_cgraph * gf = ggml_new_graph_custom(ctx_compute.get(), graph_size, false);
|
||||
|
||||
ggml_tensor * out = x;
|
||||
for (int i = 0; i < n_nodes; i++) {
|
||||
out = ggml_sleep(ctx_compute.get(), out, us);
|
||||
}
|
||||
ggml_set_name(out, "out");
|
||||
ggml_build_forward_expand(gf, out);
|
||||
|
||||
// the backends log to stdout while computing, so the result line is only printed once it is complete
|
||||
char desc[128];
|
||||
snprintf(desc, sizeof(desc), " %-6s %-4s ne=%-9" PRId64 " us=%-7d n_nodes=%d ",
|
||||
ggml_backend_name(backend), ggml_type_name(type), ne, us, n_nodes);
|
||||
|
||||
test_result result = TEST_OK;
|
||||
|
||||
if (!ggml_backend_supports_op(backend, out)) {
|
||||
printf("%snot supported\n", desc);
|
||||
result = TEST_SKIP;
|
||||
} else {
|
||||
ggml_backend_buffer_ptr buf_static(ggml_backend_alloc_ctx_tensors(ctx_static.get(), backend));
|
||||
|
||||
// random bytes rather than random floats, so that bit patterns a value-wise copy could
|
||||
// mangle, such as NaNs and denormals, are covered as well
|
||||
std::vector<uint8_t> data_in(ggml_nbytes(x));
|
||||
std::mt19937 rng(1234);
|
||||
for (size_t i = 0; i < data_in.size(); i++) {
|
||||
data_in[i] = rng() & 0xFF;
|
||||
}
|
||||
ggml_backend_tensor_set(x, data_in.data(), 0, data_in.size());
|
||||
|
||||
ggml_gallocr_ptr galloc(ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)));
|
||||
ggml_gallocr_alloc_graph(galloc.get(), gf);
|
||||
|
||||
// warm up, so that one-time costs such as lazy CUDA module loading are not timed
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
const ggml_status status = ggml_backend_graph_compute(backend, gf);
|
||||
const int64_t t_us = ggml_time_us() - t_start_us;
|
||||
|
||||
std::vector<uint8_t> data_out(ggml_nbytes(out));
|
||||
ggml_backend_tensor_get(out, data_out.data(), 0, data_out.size());
|
||||
|
||||
const int64_t t_min_us = (int64_t) n_nodes * us;
|
||||
|
||||
const bool ok_status = status == GGML_STATUS_SUCCESS;
|
||||
const bool ok_alias = out->data == x->data;
|
||||
const bool ok_data = memcmp(data_in.data(), data_out.data(), data_in.size()) == 0;
|
||||
const bool ok_time = t_us >= t_min_us;
|
||||
|
||||
printf("%selapsed=%7" PRId64 " us (>= %6" PRId64 ") alias: %-4s data: %-4s time: %s\n",
|
||||
desc, t_us, t_min_us, ok_alias ? "OK" : "FAIL", ok_data ? "OK" : "FAIL", ok_time ? "OK" : "FAIL");
|
||||
|
||||
if (!ok_status) {
|
||||
printf(" compute failed: %s\n", ggml_status_to_string(status));
|
||||
}
|
||||
|
||||
result = ok_status && ok_alias && ok_data && ok_time ? TEST_OK : TEST_FAIL;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
int main() {
|
||||
ggml_time_init();
|
||||
ggml_backend_load_all();
|
||||
|
||||
int n_ok = 0;
|
||||
int n_fail = 0;
|
||||
int n_skip = 0;
|
||||
|
||||
for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
|
||||
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
|
||||
|
||||
ggml_backend_ptr backend(ggml_backend_dev_init(dev, nullptr));
|
||||
if (backend == nullptr) {
|
||||
printf(" failed to initialize %s\n", ggml_backend_dev_name(dev));
|
||||
n_fail++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// a zero duration must still be a valid no-op, longer ones are checked against the clock,
|
||||
// and the last case verifies that a chain of sleeps accumulates instead of collapsing into one
|
||||
const struct {
|
||||
ggml_type type;
|
||||
int64_t ne;
|
||||
int32_t us;
|
||||
int n_nodes;
|
||||
} cases[] = {
|
||||
{ GGML_TYPE_F32, 1, 0, 1 },
|
||||
{ GGML_TYPE_F32, 4096, 10000, 1 },
|
||||
{ GGML_TYPE_F16, 1024, 10000, 1 },
|
||||
{ GGML_TYPE_F32, 1 << 20, 5000, 4 },
|
||||
};
|
||||
|
||||
for (const auto & c : cases) {
|
||||
switch (test_sleep(backend.get(), c.type, c.ne, c.us, c.n_nodes)) {
|
||||
case TEST_OK: n_ok++; break;
|
||||
case TEST_FAIL: n_fail++; break;
|
||||
case TEST_SKIP: n_skip++; break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
printf("%d passed, %d failed, %d skipped\n", n_ok, n_fail, n_skip);
|
||||
|
||||
return n_fail > 0;
|
||||
}
|
||||
@@ -2379,6 +2379,18 @@ int llama_bench(int argc, char ** argv) {
|
||||
}
|
||||
|
||||
llama_context * ctx = llama_init_from_model(lmodel, cparams);
|
||||
|
||||
// a model without memory reads a prompt in one ubatch
|
||||
const int n_tokens_max = std::max(inst.n_prompt, inst.n_depth);
|
||||
const bool resize = ctx != NULL && !llama_get_memory(ctx) && (int) llama_n_ubatch(ctx) < n_tokens_max;
|
||||
if (resize) {
|
||||
fprintf(stderr, "%s: the model has no memory, setting n_batch = n_ubatch = %d\n", __func__, n_tokens_max);
|
||||
llama_free(ctx);
|
||||
cparams.n_batch = n_tokens_max;
|
||||
cparams.n_ubatch = n_tokens_max;
|
||||
ctx = llama_init_from_model(lmodel, cparams);
|
||||
}
|
||||
|
||||
if (ctx == NULL) {
|
||||
fprintf(stderr, "%s: error: failed to create context with model '%s'\n", __func__, inst.model.c_str());
|
||||
llama_model_free(lmodel);
|
||||
@@ -2386,6 +2398,10 @@ int llama_bench(int argc, char ** argv) {
|
||||
}
|
||||
|
||||
test t(inst, lmodel, ctx);
|
||||
if (resize) {
|
||||
t.n_batch = n_tokens_max;
|
||||
t.n_ubatch = n_tokens_max;
|
||||
}
|
||||
|
||||
llama_memory_clear(llama_get_memory(ctx), false);
|
||||
|
||||
|
||||
@@ -88,6 +88,8 @@ add_library(mtmd
|
||||
models/fish-s2-spkenc.cpp
|
||||
models/zonos3-gen.cpp
|
||||
models/yue2-gen.cpp
|
||||
models/voxcpm2-gen.cpp
|
||||
models/voxcpm2-spkenc.cpp
|
||||
models/snake-vae.cpp
|
||||
models/music3-gen.cpp
|
||||
models/zonos3-spkenc.cpp
|
||||
|
||||
@@ -86,15 +86,16 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i
|
||||
|
||||
### Running a transformer stage on libllama (mtmd-owned `llama_context`)
|
||||
|
||||
Some stages are a full LLM-sized transformer that attends to the keys/values of the backbone, for example the flow-matching NAR of YuE2 (same layer shapes as the backbone, its own weights, bidirectional over its rows, and it reads the backbone KV). Do not re-implement it as an mtmd graph. Instead, mtmd can own a second `llama_model` + `llama_context` (see `mtmd_yue2_nar_load()` and `mtmd_gen_flow_yue2()` in `mtmd.cpp`):
|
||||
Some stages are a full LLM-sized transformer that attends to the keys/values of the backbone, for example the flow-matching NAR of YuE2 (same layer shapes as the backbone, its own weights, bidirectional over its rows, and it reads the backbone KV). Do not re-implement it as an mtmd graph. Instead, mtmd can own a second `llama_model` + `llama_context` (see `mtmd_yue2_nar_load()` and `mtmd_gen_flow_yue2()`, or `mtmd_voxcpm2_res_load()` and `mtmd_gen_code_voxcpm2()` in `mtmd.cpp`):
|
||||
|
||||
- Conversion: store the layers in the mmproj under `a.gen.<stage>.` with the libllama tensor names of an existing arch (e.g. `a.gen.nar.blk.0.attn_q.weight`)
|
||||
- Loading: build a synthetic `gguf_context` with the arch hparams (`general.architecture`, `tokenizer.ggml.model = none`, `vocab_size = 1`, ...) and one tensor info per mmproj tensor (only the type is used, the shapes come from the hparams), then call `llama_model_init_from_user()`. The `set_tensor_data` callback reads the data from the mmproj file, so no copy is kept in the mtmd model
|
||||
- libllama also creates every optional tensor of the arch (biases, `*.scale`, `*.input_scale`, `dense_2`, ...) for a model built from metadata. The callback must fill them with values that leave the graph unchanged: 0 for biases, 1 for scales, identity for `dense_2`. Zeros everywhere silently make every matmul output 0
|
||||
- Loading: build a synthetic `gguf_context` with the arch hparams (`general.architecture`, `tokenizer.ggml.model = none`, `vocab_size = 1`, ...) and one tensor info per mmproj tensor (only the type is used, the shapes come from the hparams), then call `llama_model_init_from_user()` (`mtmd_owned_llama_load()` does this, the caller only writes the hparams). The `set_tensor_data` callback reads the data from the mmproj file, so no copy is kept in the mtmd model
|
||||
- libllama also creates every optional tensor of the arch (biases, `*.scale`, `*.input_scale`, `dense_2`, ...) for a model built from metadata. The callback must fill them with values that leave the graph unchanged: 0 for biases, 1 for scales and `rope_freqs`, identity for `dense_2`. Zeros everywhere silently make every matmul output 0
|
||||
- Backbone keys/values: the helper passes the backbone sequence state (`llama_state_seq_get_data()`) through `mtmd_gen_inp`, mtmd loads it into its context with `llama_state_seq_set_data()`. This only checks the layer count and the K/V types and row sizes, so it works across models with the same KV layout. The two contexts must agree on the KV type and on flash attention (`v_trans`)
|
||||
- Inputs and outputs are embedding rows (`llama_batch::embd`, `llama_get_embeddings()` with `LLAMA_POOLING_TYPE_NONE`), the projections around them stay small mtmd graphs
|
||||
- Non-causal rows: `LLAMA_ATTENTION_TYPE_NON_CAUSAL`, all rows in one ubatch (`n_ubatch` >= rows), they see every cached position of the sequence. Remove them with `llama_memory_seq_rm()` after each pass
|
||||
- Size the context per call (`n_ctx`, `n_ubatch`) instead of keeping one for the whole session
|
||||
- Causal stage with its own KV cache across calls (the residual LM of VoxCPM2): the `mtmd_gen_state` owns the `llama_context`, `mtmd_gen_state_reset()` clears its memory. Options of the arch may switch parts off, e.g. RoPE with `rope.scaling.finetuned = false` on granite
|
||||
|
||||
### Text preprocessing (the `tts` chat template)
|
||||
|
||||
|
||||
@@ -507,7 +507,7 @@ struct mtmd_graph_qwen3tts_gen : mtmd_graph {
|
||||
|
||||
// stateful conv ops: read/update their state via state_in/state_out[state_name]
|
||||
ggml_tensor * causal_conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation, const std::string & state_name) const;
|
||||
ggml_tensor * causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const;
|
||||
ggml_tensor * causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name, int dilation = 1) const;
|
||||
ggml_tensor * causal_conv_transpose1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, const std::string & state_name) const;
|
||||
ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const;
|
||||
|
||||
@@ -780,6 +780,66 @@ struct mtmd_graph_music3_gen : mtmd_graph_snake_vae {
|
||||
ggml_cgraph * build_vocoder();
|
||||
};
|
||||
|
||||
//
|
||||
// VoxCPM2 generation, the residual LM runs on a llama_context of the gen state, see mtmd_gen_code_voxcpm2():
|
||||
// RES_INP = backbone row -> FSQ'd hidden state, residual LM input row, stop score
|
||||
// GEN_CODE = LocDiT flow matching with CFG-Zero* -> next patch, then LocEnc -> next backbone input row
|
||||
// LATENT_IN = LocEnc, latent patches -> backbone input rows (voice reference)
|
||||
// GEN_WAV = causal AudioVAE decoder, latent patches -> PCM
|
||||
//
|
||||
struct mtmd_graph_voxcpm2_gen : mtmd_graph {
|
||||
mtmd_graph_voxcpm2_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process,
|
||||
int n_rows, bool audio_row, int n_steps, float cfg_scale)
|
||||
: mtmd_graph(ctx, img), gen_process(gen_process), n_rows(n_rows), audio_row(audio_row), n_steps(n_steps), cfg_scale(cfg_scale) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
static constexpr int n_latent = 64; // feat_dim
|
||||
static constexpr int n_patch = 4; // latents per backbone row
|
||||
static constexpr int n_dit_tok = 3 + 2 * n_patch; // lm, res, time, cond patch, noisy patch
|
||||
static constexpr int head_dim = 128; // kv_channels
|
||||
static constexpr float rope_theta = 10000.0f;
|
||||
static constexpr int rope_n_ctx_orig = 32768;
|
||||
static constexpr float fsq_scale = 9.0f;
|
||||
static constexpr int n_steps_default = 10; // inference_timesteps
|
||||
static constexpr int n_steps_max = 32;
|
||||
static constexpr float cfg_default = 2.0f; // cfg_value
|
||||
static constexpr int n_ctx_res = 8192; // residual LM positions of one sequence, max_length of the reference
|
||||
static constexpr int dec_ratio = 1920; // decoder samples per latent
|
||||
static constexpr int res_dilations[3] = { 1, 3, 9 };
|
||||
|
||||
// t of the Euler steps with sway sampling, n_steps + 1 values from 1 to 0
|
||||
static std::vector<float> t_span(int n_steps);
|
||||
// SinusoidalPosEmb(t * 1000) of the steps, [n_embd, n_steps]
|
||||
static std::vector<float> time_embd(int n_steps, int n_embd);
|
||||
|
||||
mtmd_subctx_gen_process gen_process;
|
||||
int n_rows;
|
||||
bool audio_row;
|
||||
int n_steps;
|
||||
float cfg_scale;
|
||||
|
||||
// bidirectional MiniCPM layers, x: [n_embd, n_tok, n_seq]
|
||||
ggml_tensor * tfm(ggml_tensor * x, const std::vector<mtmd_layer> & layers, ggml_tensor * norm_w, ggml_tensor * pos) const;
|
||||
// [n_latent, n_patch * n] -> [n_text_embd, n]
|
||||
ggml_tensor * loc_enc(ggml_tensor * latents, int n) const;
|
||||
ggml_tensor * inp_pos(const char * name, int n) const;
|
||||
void build_res_inp();
|
||||
void build_flow();
|
||||
void build_latent_in();
|
||||
void build_dec();
|
||||
};
|
||||
|
||||
std::vector<c2w_state_slot> list_voxcpm2_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
|
||||
|
||||
// AudioVAE encoder: raw waveform -> latent patches, LocEnc then runs on the gen model (LATENT_IN)
|
||||
struct mtmd_graph_voxcpm2_spkenc : mtmd_graph {
|
||||
mtmd_graph_voxcpm2_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
// x: [T, IC], causal: pad zeros on the left
|
||||
ggml_tensor * conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation, int pad, bool depthwise) const;
|
||||
};
|
||||
|
||||
// tiny-voca: small graphs around the stacks, the stacks run on libllama (see mtmd_gen_wav_tiny_voca())
|
||||
// LATENT_IN: token states [n_embd, n_rows] -> duration outputs [2, n_rows]
|
||||
// LATENT_OUT: acoustic states [n_embd, n_rows] -> source-filter params [n_voc, n_rows]
|
||||
|
||||
@@ -315,18 +315,18 @@ ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::causal_conv1d(ggml_tensor * x,
|
||||
return y;
|
||||
}
|
||||
|
||||
// causal depthwise conv1d, stride 1, dilation 1, kernel from w's shape.
|
||||
// causal depthwise conv1d, stride 1, kernel from w's shape.
|
||||
// x: [T, C]. w: [K, 1, C]. returns [T, C]. see causal_conv1d for the state contract.
|
||||
ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name) const {
|
||||
ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::causal_conv1d_dw(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, const std::string & state_name, int dilation) const {
|
||||
const int K = (int) w->ne[0];
|
||||
const int pad = K - 1;
|
||||
const int pad = (K - 1) * dilation;
|
||||
|
||||
ggml_tensor * x_full = x;
|
||||
if (pad > 0) {
|
||||
ggml_tensor * left = state_in.at(state_name); // [pad, C]
|
||||
x_full = ggml_concat(ctx0, left, x, 0);
|
||||
}
|
||||
ggml_tensor * y = ggml_conv_1d_dw(ctx0, w, x_full, 1, 0, 1); // [T, C, 1]
|
||||
ggml_tensor * y = ggml_conv_1d_dw(ctx0, w, x_full, 1, 0, dilation); // [T, C, 1]
|
||||
y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]);
|
||||
if (b) {
|
||||
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
#include "models.h"
|
||||
|
||||
#include <cmath>
|
||||
|
||||
// linspace(1, 0) with sway sampling (coef 1), see UnifiedCFM.forward()
|
||||
std::vector<float> mtmd_graph_voxcpm2_gen::t_span(int n_steps) {
|
||||
std::vector<float> t(n_steps + 1);
|
||||
for (int i = 0; i <= n_steps; i++) {
|
||||
const double x = 1.0 - (double) i / n_steps;
|
||||
t[i] = (float) (x + std::cos(M_PI / 2 * x) - 1.0 + x);
|
||||
}
|
||||
return t;
|
||||
}
|
||||
|
||||
// SinusoidalPosEmb with scale 1000: [sin, cos], the frequency step divides by half - 1
|
||||
std::vector<float> mtmd_graph_voxcpm2_gen::time_embd(int n_steps, int n_embd) {
|
||||
const std::vector<float> t = t_span(n_steps);
|
||||
const int half = n_embd / 2;
|
||||
std::vector<float> out((size_t) n_embd * n_steps);
|
||||
for (int s = 0; s < n_steps; s++) {
|
||||
for (int j = 0; j < half; j++) {
|
||||
const double a = 1000.0 * t[s] * std::exp(-std::log(10000.0) * j / (half - 1));
|
||||
out[(size_t) s * n_embd + j] = (float) std::sin(a);
|
||||
out[(size_t) s * n_embd + half + j] = (float) std::cos(a);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
ggml_tensor * mtmd_graph_voxcpm2_gen::inp_pos(const char * name, int n) const {
|
||||
ggml_tensor * pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n);
|
||||
ggml_set_name(pos, name);
|
||||
ggml_set_input(pos);
|
||||
return pos;
|
||||
}
|
||||
|
||||
ggml_tensor * mtmd_graph_voxcpm2_gen::tfm(ggml_tensor * x, const std::vector<mtmd_layer> & layers, ggml_tensor * norm_w, ggml_tensor * pos) const {
|
||||
const int64_t n_tok = x->ne[1];
|
||||
const int64_t n_seq = x->ne[2];
|
||||
const int n_head = hparams.n_head;
|
||||
const int n_head_kv = hparams.n_head_kv;
|
||||
const float kq_scale = 1.0f / sqrtf((float) head_dim);
|
||||
ggml_tensor * factors = model.voxcpm2.rope_factors;
|
||||
|
||||
for (size_t il = 0; il < layers.size(); il++) {
|
||||
const auto & layer = layers[il];
|
||||
ggml_tensor * cur = build_norm(x, layer.ln_1_w, nullptr, NORM_TYPE_RMS, hparams.eps, (int) il);
|
||||
|
||||
ggml_tensor * q = ggml_reshape_4d(ctx0, build_mm(layer.q_w, cur), head_dim, n_head, n_tok, n_seq);
|
||||
ggml_tensor * k = ggml_reshape_4d(ctx0, build_mm(layer.k_w, cur), head_dim, n_head_kv, n_tok, n_seq);
|
||||
ggml_tensor * v = ggml_reshape_4d(ctx0, build_mm(layer.v_w, cur), head_dim, n_head_kv, n_tok, n_seq);
|
||||
q = ggml_rope_ext(ctx0, q, pos, factors, head_dim, GGML_ROPE_TYPE_NEOX, rope_n_ctx_orig, rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
k = ggml_rope_ext(ctx0, k, pos, factors, head_dim, GGML_ROPE_TYPE_NEOX, rope_n_ctx_orig, rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
|
||||
cur = build_attn(layer.o_w, nullptr, q, k, v, nullptr, kq_scale, (int) il);
|
||||
x = ggml_add(ctx0, x, ggml_reshape_3d(ctx0, cur, cur->ne[0], n_tok, n_seq));
|
||||
|
||||
cur = build_norm(x, layer.ln_2_w, nullptr, NORM_TYPE_RMS, hparams.eps, (int) il);
|
||||
cur = build_ffn(cur, layer.ff_up_w, nullptr, layer.ff_gate_w, nullptr, layer.ff_down_w, nullptr, FFN_SILU, (int) il);
|
||||
x = ggml_add(ctx0, x, cur);
|
||||
}
|
||||
return build_norm(x, norm_w, nullptr, NORM_TYPE_RMS, hparams.eps, -1);
|
||||
}
|
||||
|
||||
// each patch gets the special token in front, its output is the summary of the patch
|
||||
ggml_tensor * mtmd_graph_voxcpm2_gen::loc_enc(ggml_tensor * latents, int n) const {
|
||||
const auto & m = model.voxcpm2;
|
||||
const int64_t n_embd = hparams.n_embd;
|
||||
|
||||
ggml_tensor * x = ggml_add(ctx0, build_mm(m.enc_in_w, latents), m.enc_in_b);
|
||||
x = ggml_reshape_3d(ctx0, x, n_embd, n_patch, n);
|
||||
ggml_tensor * special = ggml_repeat(ctx0, ggml_reshape_3d(ctx0, m.enc_special, n_embd, 1, 1),
|
||||
ggml_view_3d(ctx0, x, n_embd, 1, n, x->nb[1], x->nb[2], 0));
|
||||
x = ggml_concat(ctx0, special, x, 1); // [n_embd, 1 + n_patch, n]
|
||||
x = tfm(x, m.enc_layers, m.enc_norm_w, inp_pos("inp_pos_enc", 1 + n_patch));
|
||||
x = ggml_cont(ctx0, ggml_view_2d(ctx0, x, n_embd, n, x->nb[2], 0));
|
||||
cb(x, "loc_enc_out", -1);
|
||||
return ggml_add(ctx0, build_mm(m.enc_to_lm_w, x), m.enc_to_lm_b);
|
||||
}
|
||||
|
||||
void mtmd_graph_voxcpm2_gen::build_res_inp() {
|
||||
const auto & m = model.voxcpm2;
|
||||
const int64_t n_text = m.fusion_b->ne[0];
|
||||
|
||||
ggml_tensor * h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_text, n_rows);
|
||||
ggml_set_name(h, "inp_raw");
|
||||
ggml_set_input(h);
|
||||
ggml_tensor * x = nullptr;
|
||||
if (audio_row) {
|
||||
x = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_text, n_rows);
|
||||
ggml_set_name(x, "inp_feats");
|
||||
ggml_set_input(x);
|
||||
// FSQ: tanh, then round to the grid of 1/9
|
||||
ggml_tensor * q = ggml_tanh(ctx0, ggml_add(ctx0, build_mm(m.fsq_in_w, h), m.fsq_in_b));
|
||||
q = ggml_scale(ctx0, ggml_round(ctx0, ggml_scale(ctx0, q, fsq_scale)), 1.0f / fsq_scale);
|
||||
h = ggml_add(ctx0, build_mm(m.fsq_out_w, q), m.fsq_out_b);
|
||||
} else {
|
||||
x = ggml_scale(ctx0, h, 0.0f); // a text row has no input row
|
||||
}
|
||||
|
||||
// logit(stop) - logit(continue), only the row of the next frame needs it
|
||||
if (n_rows == 1) {
|
||||
ggml_tensor * s = ggml_silu(ctx0, ggml_add(ctx0, build_mm(m.stop_proj_w, h), m.stop_proj_b));
|
||||
s = build_mm(m.stop_head_w, s);
|
||||
s = ggml_sub(ctx0, ggml_view_1d(ctx0, s, 1, ggml_element_size(s)), ggml_view_1d(ctx0, s, 1, 0));
|
||||
ggml_set_name(s, "out_eos_score");
|
||||
ggml_set_output(s);
|
||||
ggml_build_forward_expand(gf, s);
|
||||
}
|
||||
|
||||
// residual LM input rows, then the (FSQ'd) hidden states
|
||||
ggml_tensor * r = ggml_add(ctx0, build_mm(m.fusion_w, ggml_concat(ctx0, h, x, 0)), m.fusion_b);
|
||||
ggml_tensor * out = ggml_concat(ctx0, r, h, 1);
|
||||
ggml_set_name(out, "out_feats");
|
||||
ggml_set_output(out);
|
||||
ggml_build_forward_expand(gf, out);
|
||||
}
|
||||
|
||||
// Euler solver from noise (t = 1) to the patch (t = 0), see UnifiedCFM.solve_euler()
|
||||
void mtmd_graph_voxcpm2_gen::build_flow() {
|
||||
const auto & m = model.voxcpm2;
|
||||
const int64_t n_text = m.fusion_b->ne[0];
|
||||
const int64_t n_embd = hparams.n_embd;
|
||||
|
||||
// FSQ'd hidden state, then the residual LM output
|
||||
ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_text, 2);
|
||||
ggml_set_name(inp, "inp_raw");
|
||||
ggml_set_input(inp);
|
||||
ggml_tensor * cond = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, n_patch);
|
||||
ggml_set_name(cond, "inp_cond");
|
||||
ggml_set_input(cond);
|
||||
ggml_tensor * noise = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, n_patch);
|
||||
ggml_set_name(noise, "inp_noise");
|
||||
ggml_set_input(noise);
|
||||
ggml_tensor * temb = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_steps);
|
||||
ggml_set_name(temb, "inp_temb");
|
||||
ggml_set_input(temb);
|
||||
ggml_tensor * pos = inp_pos("inp_pos_dit", n_dit_tok);
|
||||
|
||||
ggml_tensor * mu_lm = ggml_add(ctx0, build_mm(m.lm_to_dit_w, ggml_view_1d(ctx0, inp, n_text, 0)), m.lm_to_dit_b);
|
||||
ggml_tensor * mu_res = ggml_add(ctx0, build_mm(m.res_to_dit_w, ggml_view_1d(ctx0, inp, n_text, inp->nb[1])), m.res_to_dit_b);
|
||||
ggml_tensor * mu = ggml_concat(ctx0, ggml_reshape_2d(ctx0, mu_lm, n_embd, 1), ggml_reshape_2d(ctx0, mu_res, n_embd, 1), 1);
|
||||
ggml_tensor * mu_0 = ggml_scale(ctx0, mu, 0.0f); // the unconditional branch has mu = 0
|
||||
|
||||
temb = ggml_silu(ctx0, ggml_add(ctx0, build_mm(m.time0_w, temb), m.time0_b));
|
||||
temb = ggml_add(ctx0, build_mm(m.time1_w, temb), m.time1_b);
|
||||
ggml_tensor * cond_tok = ggml_add(ctx0, build_mm(m.dit_cond_w, cond), m.dit_cond_b);
|
||||
|
||||
const std::vector<float> t = t_span(n_steps);
|
||||
// CFG-Zero*: the first steps keep x as is
|
||||
const int n_zero = std::max(1, (int) ((n_steps + 1) * 0.04));
|
||||
|
||||
ggml_tensor * x = noise;
|
||||
for (int s = n_zero; s < n_steps; s++) {
|
||||
ggml_tensor * t_tok = ggml_view_2d(ctx0, temb, n_embd, 1, temb->nb[1], (size_t) s * temb->nb[1]);
|
||||
ggml_tensor * x_tok = ggml_add(ctx0, build_mm(m.dit_in_w, x), m.dit_in_b);
|
||||
ggml_tensor * tail = ggml_concat(ctx0, ggml_concat(ctx0, t_tok, cond_tok, 1), x_tok, 1);
|
||||
ggml_tensor * h = ggml_concat(ctx0,
|
||||
ggml_reshape_3d(ctx0, ggml_concat(ctx0, mu, tail, 1), n_embd, n_dit_tok, 1),
|
||||
ggml_reshape_3d(ctx0, ggml_concat(ctx0, mu_0, tail, 1), n_embd, n_dit_tok, 1), 2);
|
||||
h = tfm(h, model.layers, model.post_ln_w, pos);
|
||||
h = ggml_cont(ctx0, ggml_view_3d(ctx0, h, n_embd, n_patch, 2, h->nb[1], h->nb[2], (size_t) (n_dit_tok - n_patch) * h->nb[1]));
|
||||
h = ggml_add(ctx0, build_mm(m.dit_out_w, h), m.dit_out_b); // [n_latent, n_patch, 2]
|
||||
|
||||
// guided velocity, the unconditional one is scaled by its projection on the conditional one
|
||||
ggml_tensor * v_c = ggml_view_2d(ctx0, h, n_latent, n_patch, h->nb[1], 0);
|
||||
ggml_tensor * v_u = ggml_view_2d(ctx0, h, n_latent, n_patch, h->nb[1], h->nb[2]);
|
||||
ggml_tensor * st = ggml_div(ctx0, ggml_sum(ctx0, ggml_mul(ctx0, v_c, v_u)),
|
||||
ggml_scale_bias(ctx0, ggml_sum(ctx0, ggml_sqr(ctx0, v_u)), 1.0f, 1e-8f));
|
||||
ggml_tensor * v_s = ggml_mul(ctx0, v_u, st);
|
||||
ggml_tensor * v = ggml_add(ctx0, v_s, ggml_scale(ctx0, ggml_sub(ctx0, v_c, v_s), cfg_scale));
|
||||
x = ggml_add(ctx0, x, ggml_scale(ctx0, v, t[s + 1] - t[s]));
|
||||
}
|
||||
GGML_ASSERT(x != noise && "voxcpm2: not enough flow steps");
|
||||
ggml_set_name(x, "out_feats");
|
||||
ggml_set_output(x);
|
||||
ggml_build_forward_expand(gf, x);
|
||||
|
||||
// next backbone input row, the last node
|
||||
ggml_build_forward_expand(gf, loc_enc(x, 1));
|
||||
}
|
||||
|
||||
void mtmd_graph_voxcpm2_gen::build_latent_in() {
|
||||
ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, n_patch * n_rows);
|
||||
ggml_set_name(inp, "inp_feats");
|
||||
ggml_set_input(inp);
|
||||
ggml_tensor * out = loc_enc(inp, n_rows);
|
||||
ggml_set_name(out, "out_feats");
|
||||
ggml_set_output(out);
|
||||
ggml_build_forward_expand(gf, out);
|
||||
}
|
||||
|
||||
std::vector<c2w_state_slot> list_voxcpm2_state_slots(const mtmd_hparams & /*hparams*/, const mtmd_model & model) {
|
||||
const auto & m = model.voxcpm2;
|
||||
std::vector<c2w_state_slot> slots;
|
||||
slots.push_back({"in_dw", m.dec_in_dw_w->ne[0] - 1, m.dec_in_dw_w->ne[2]});
|
||||
for (size_t ib = 0; ib < m.dec.size(); ib++) {
|
||||
const auto & blk = m.dec[ib];
|
||||
const std::string name = "up" + std::to_string(ib);
|
||||
slots.push_back({name, blk.conv_w->ne[0] / 2, blk.conv_w->ne[1]});
|
||||
for (size_t ir = 0; ir < blk.res.size(); ir++) {
|
||||
const auto & ru = blk.res[ir];
|
||||
slots.push_back({name + "_res" + std::to_string(ir),
|
||||
(ru.conv1_w->ne[0] - 1) * mtmd_graph_voxcpm2_gen::res_dilations[ir], ru.conv1_w->ne[2]});
|
||||
}
|
||||
}
|
||||
slots.push_back({"out", m.dec_out_w->ne[0] - 1, m.dec_out_w->ne[1]});
|
||||
return slots;
|
||||
}
|
||||
|
||||
// causal AudioVAE decoder, the conv states carry the left context between calls
|
||||
void mtmd_graph_voxcpm2_gen::build_dec() {
|
||||
const auto & m = model.voxcpm2;
|
||||
|
||||
ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, n_patch * n_rows);
|
||||
ggml_set_name(inp, "inp_feats");
|
||||
ggml_set_input(inp);
|
||||
|
||||
mtmd_graph_qwen3tts_gen::code2wav dec(*this);
|
||||
for (const auto & slot : list_voxcpm2_state_slots(hparams, model)) {
|
||||
ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
|
||||
ggml_set_name(t, ("state_in_" + slot.name).c_str());
|
||||
ggml_set_input(t);
|
||||
dec.state_in[slot.name] = t;
|
||||
}
|
||||
|
||||
ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [T, n_latent]
|
||||
x = dec.causal_conv1d_dw(x, m.dec_in_dw_w, m.dec_in_dw_b, "in_dw");
|
||||
x = dec.causal_conv1d(x, m.dec_in_w, m.dec_in_b, 1, "");
|
||||
for (size_t ib = 0; ib < m.dec.size(); ib++) {
|
||||
const auto & blk = m.dec[ib];
|
||||
const std::string name = "up" + std::to_string(ib);
|
||||
const int64_t n_ch = m.dec_sr_scale[ib]->ne[0];
|
||||
|
||||
// sample-rate conditioning, the row of the output rate is picked at conversion
|
||||
x = ggml_add(ctx0, ggml_mul(ctx0, x, ggml_reshape_2d(ctx0, m.dec_sr_scale[ib], 1, n_ch)),
|
||||
ggml_reshape_2d(ctx0, m.dec_sr_bias[ib], 1, n_ch));
|
||||
x = dec.snake(x, blk.snake_alpha, blk.snake_beta);
|
||||
x = dec.causal_conv_transpose1d(x, blk.conv_w, blk.conv_b, (int) (blk.conv_w->ne[0] / 2), name);
|
||||
for (size_t ir = 0; ir < blk.res.size(); ir++) {
|
||||
const auto & ru = blk.res[ir];
|
||||
ggml_tensor * h = dec.snake(x, ru.act1_alpha, ru.act1_beta);
|
||||
h = dec.causal_conv1d_dw(h, ru.conv1_w, ru.conv1_b, name + "_res" + std::to_string(ir), res_dilations[ir]);
|
||||
h = dec.snake(h, ru.act2_alpha, ru.act2_beta);
|
||||
h = dec.causal_conv1d(h, ru.conv2_w, ru.conv2_b, 1, "");
|
||||
x = ggml_add(ctx0, x, h);
|
||||
}
|
||||
cb(x, "vae_block", (int) ib);
|
||||
}
|
||||
x = dec.snake(x, m.dec_post_alpha, m.dec_post_beta);
|
||||
x = dec.causal_conv1d(x, m.dec_out_w, m.dec_out_b, 1, "out");
|
||||
x = ggml_tanh(ctx0, ggml_reshape_1d(ctx0, x, x->ne[0]));
|
||||
ggml_set_name(x, "out_audio");
|
||||
ggml_set_output(x);
|
||||
ggml_build_forward_expand(gf, x);
|
||||
|
||||
for (auto & slot : dec.state_out) {
|
||||
ggml_set_name(slot.second, ("state_out_" + slot.first).c_str());
|
||||
ggml_set_output(slot.second);
|
||||
ggml_build_forward_expand(gf, slot.second);
|
||||
}
|
||||
}
|
||||
|
||||
ggml_cgraph * mtmd_graph_voxcpm2_gen::build() {
|
||||
switch (gen_process) {
|
||||
case MTMD_SUBCTX_GEN_PROCESS_RES_INP: build_res_inp(); break;
|
||||
case MTMD_SUBCTX_GEN_PROCESS_GEN_CODE: build_flow(); break;
|
||||
case MTMD_SUBCTX_GEN_PROCESS_LATENT_IN: build_latent_in(); break;
|
||||
case MTMD_SUBCTX_GEN_PROCESS_GEN_WAV: build_dec(); break;
|
||||
default: GGML_ABORT("voxcpm2: unknown gen_process");
|
||||
}
|
||||
return gf;
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
#include "models.h"
|
||||
|
||||
ggml_tensor * mtmd_graph_voxcpm2_spkenc::conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation, int pad, bool depthwise) const {
|
||||
if (pad > 0) {
|
||||
x = ggml_pad_ext(ctx0, x, pad, 0, 0, 0, 0, 0, 0, 0);
|
||||
}
|
||||
// im2col in F32: the reference AudioVAE runs in FP32
|
||||
ggml_tensor * y = nullptr;
|
||||
if (depthwise) {
|
||||
ggml_tensor * col = ggml_im2col(ctx0, w, ggml_reshape_4d(ctx0, x, x->ne[0], 1, x->ne[1], 1), stride, 0, 0, 0, dilation, 0, false, GGML_TYPE_F32);
|
||||
y = ggml_mul_mat(ctx0, col, w); // [T, 1, C]
|
||||
y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[2]);
|
||||
} else {
|
||||
ggml_tensor * col = ggml_im2col(ctx0, w, x, stride, 0, 0, 0, dilation, 0, false, GGML_TYPE_F32); // [IC * K, T]
|
||||
y = ggml_mul_mat(ctx0,
|
||||
ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[1] * col->ne[2]),
|
||||
ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2])); // [T, OC]
|
||||
}
|
||||
if (b) {
|
||||
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
||||
}
|
||||
return y;
|
||||
}
|
||||
|
||||
// causal AudioVAE encoder: waveform -> mu of the latents, 4 latents per output row
|
||||
ggml_cgraph * mtmd_graph_voxcpm2_spkenc::build() {
|
||||
using g = mtmd_graph_voxcpm2_gen;
|
||||
const auto & m = model.voxcpm2;
|
||||
mtmd_graph_qwen3tts_gen::code2wav act(*this);
|
||||
|
||||
// the preprocessor hands over the waveform as a single-row "mel"
|
||||
ggml_tensor * inp_raw = build_inp_raw(1);
|
||||
ggml_tensor * x = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], 1); // [T, 1]
|
||||
|
||||
x = conv(x, m.enc_conv_in_w, m.enc_conv_in_b, 1, 1, (int) m.enc_conv_in_w->ne[0] - 1, false);
|
||||
for (size_t ib = 0; ib < m.enc.size(); ib++) {
|
||||
const auto & blk = m.enc[ib];
|
||||
for (size_t ir = 0; ir < blk.res.size(); ir++) {
|
||||
const auto & ru = blk.res[ir];
|
||||
const int dil = g::res_dilations[ir];
|
||||
ggml_tensor * h = act.snake(x, ru.act1_alpha, ru.act1_beta);
|
||||
h = conv(h, ru.conv1_w, ru.conv1_b, 1, dil, (int) (ru.conv1_w->ne[0] - 1) * dil, true);
|
||||
h = act.snake(h, ru.act2_alpha, ru.act2_beta);
|
||||
h = conv(h, ru.conv2_w, ru.conv2_b, 1, 1, 0, false);
|
||||
x = ggml_add(ctx0, x, h);
|
||||
}
|
||||
x = act.snake(x, blk.snake_alpha, blk.snake_beta);
|
||||
// kernel 2 * stride, the causal padding is one stride
|
||||
const int stride = (int) (blk.conv_w->ne[0] / 2);
|
||||
x = conv(x, blk.conv_w, blk.conv_b, stride, 1, stride, false);
|
||||
cb(x, "vae_enc_block", (int) ib);
|
||||
}
|
||||
x = conv(x, m.enc_mu_w, m.enc_mu_b, 1, 1, (int) m.enc_mu_w->ne[0] - 1, false); // [T / hop, n_latent]
|
||||
|
||||
x = ggml_cont(ctx0, ggml_transpose(ctx0, x));
|
||||
x = ggml_reshape_2d(ctx0, x, g::n_latent * g::n_patch, x->ne[1] / g::n_patch);
|
||||
ggml_build_forward_expand(gf, x);
|
||||
return gf;
|
||||
}
|
||||
@@ -2250,11 +2250,11 @@ bool mtmd_audio_preprocessor_pockettts::process(const float * sa
|
||||
return true;
|
||||
}
|
||||
|
||||
bool mtmd_audio_preprocessor_fish_s2::process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const {
|
||||
const size_t max_samples = (size_t) mtmd_hparams::fish_s2_max_spk_seconds * hparams.audio_sample_rate;
|
||||
bool mtmd_audio_preprocessor_spk_ref::process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const {
|
||||
const size_t max_samples = (size_t) max_seconds * hparams.audio_sample_rate;
|
||||
if (n_samples > max_samples) {
|
||||
LOG_WRN("%s: speaker reference is %.1f s, truncating to the first %d s\n", __func__,
|
||||
(double) n_samples / hparams.audio_sample_rate, mtmd_hparams::fish_s2_max_spk_seconds);
|
||||
(double) n_samples / hparams.audio_sample_rate, max_seconds);
|
||||
n_samples = max_samples;
|
||||
}
|
||||
return mtmd_audio_preprocessor_muse_spark_audio::process(samples, n_samples, output, shape_only);
|
||||
|
||||
@@ -134,10 +134,13 @@ struct mtmd_audio_preprocessor_muse_spark_audio : mtmd_audio_preprocessor {
|
||||
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
|
||||
};
|
||||
|
||||
// fish_s2 speaker reference: same as muse spark (raw waveform, whole frames), truncated to fish_s2_max_spk_seconds
|
||||
struct mtmd_audio_preprocessor_fish_s2 : mtmd_audio_preprocessor_muse_spark_audio {
|
||||
using mtmd_audio_preprocessor_muse_spark_audio::mtmd_audio_preprocessor_muse_spark_audio;
|
||||
// speaker reference (fish_s2, voxcpm2): same as muse spark (raw waveform, whole frames), truncated to max_seconds
|
||||
struct mtmd_audio_preprocessor_spk_ref : mtmd_audio_preprocessor_muse_spark_audio {
|
||||
mtmd_audio_preprocessor_spk_ref(const mtmd_model * model, int32_t max_seconds)
|
||||
: mtmd_audio_preprocessor_muse_spark_audio(model), max_seconds(max_seconds) {}
|
||||
bool process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const override;
|
||||
|
||||
const int32_t max_seconds;
|
||||
};
|
||||
|
||||
// vibevoice: gain to -25 dBFS RMS, without clipping the peak
|
||||
|
||||
@@ -977,6 +977,10 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
|
||||
{
|
||||
builder = std::make_unique<mtmd_graph_fish_s2_spkenc>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VOXCPM2_SPKENC:
|
||||
{
|
||||
builder = std::make_unique<mtmd_graph_voxcpm2_spkenc>(ctx, img);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_POCKETTTS_GEN:
|
||||
{
|
||||
const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
|
||||
@@ -1038,6 +1042,24 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
|
||||
? (int) (params->codes->size() / n_cb) : 1;
|
||||
builder = std::make_unique<mtmd_graph_zonos3_gen>(ctx, img, gen_process, all_known, n_frames);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VOXCPM2_GEN:
|
||||
{
|
||||
using g = mtmd_graph_voxcpm2_gen;
|
||||
// the default (warmup) graph is one frame of the flow
|
||||
const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_GEN_CODE;
|
||||
int n_rows = 1;
|
||||
if (gen_process == MTMD_SUBCTX_GEN_PROCESS_RES_INP) {
|
||||
// hidden states in img, the input rows of audio rows in feats
|
||||
n_rows = img.ny();
|
||||
} else if (params && params->feats && gen_process != MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) {
|
||||
GGML_ASSERT(!params->feats->empty() && params->feats->size() % (g::n_latent * g::n_patch) == 0);
|
||||
n_rows = (int) (params->feats->size() / (g::n_latent * g::n_patch));
|
||||
}
|
||||
const bool audio_row = gen_process == MTMD_SUBCTX_GEN_PROCESS_RES_INP && params->feats != nullptr;
|
||||
const int n_steps = params && params->n_steps > 0 ? params->n_steps : g::n_steps_default;
|
||||
const float cfg_scale = params && params->cfg_scale > 0.0f ? params->cfg_scale : g::cfg_default;
|
||||
builder = std::make_unique<mtmd_graph_voxcpm2_gen>(ctx, img, gen_process, n_rows, audio_row, n_steps, cfg_scale);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_YUE2_GEN:
|
||||
{
|
||||
const auto gen_process = params ? params->gen_process : MTMD_SUBCTX_GEN_PROCESS_LATENT_IN;
|
||||
|
||||
@@ -4914,6 +4914,433 @@ private:
|
||||
bool wav_header_sent = false;
|
||||
};
|
||||
|
||||
// VoxCPM2: the backbone emits no token, mtmd turns each hidden state into a patch of AudioVAE latents
|
||||
// every backbone row also goes through the residual LM inside mtmd, see mtmd_gen_code_voxcpm2()
|
||||
class voxcpm2_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
|
||||
public:
|
||||
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
|
||||
|
||||
// step_gen() stops at the limit of the reference, which depends on the text
|
||||
int32_t max_frames() const override { return 0; }
|
||||
bool supports_cfg() const override { return true; }
|
||||
bool supports_flow_steps() const override { return true; }
|
||||
|
||||
void reset() override {
|
||||
seq_id = 0;
|
||||
pos = 0;
|
||||
rows.clear();
|
||||
voice.clear();
|
||||
prompt_pos = 0;
|
||||
n_frames = 0;
|
||||
max_len = 0;
|
||||
h_state_buf.clear();
|
||||
embd_in.clear();
|
||||
latents.clear();
|
||||
n_decoded = 0;
|
||||
gen_state.reset();
|
||||
audio_pcm.clear();
|
||||
out_buf.clear();
|
||||
pcm_sent = 0;
|
||||
wav_header_sent = false;
|
||||
}
|
||||
|
||||
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
|
||||
reset();
|
||||
seq_id = inp->seq_id;
|
||||
out_type = inp->out_type;
|
||||
stream = inp->stream;
|
||||
temp = inp->temp;
|
||||
cfg_scale = inp->cfg_scale;
|
||||
flow_steps = inp->flow_steps;
|
||||
if (flow_steps != 0 && (flow_steps < 2 || flow_steps > 32)) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: voxcpm2 needs flow_steps in [2, 32] (0 = model default)\n");
|
||||
return 2;
|
||||
}
|
||||
if (inp->speaker_ref_text) {
|
||||
LOG_WRN("mtmd_helper_gen_audio: voxcpm2 clones the voice without its transcript, the speaker text is ignored\n");
|
||||
}
|
||||
if (!ensure_cache()) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// re.sub(r"\s+", " ", text) as in VoxCPM._generate() of core.py, a leading or trailing space is kept
|
||||
std::string text;
|
||||
bool space = false;
|
||||
for (char c : std::string(inp->prompt, inp->prompt_len)) {
|
||||
const bool is_space = std::isspace((unsigned char) c);
|
||||
if (!is_space || !space) {
|
||||
text += is_space ? ' ' : c;
|
||||
}
|
||||
space = is_space;
|
||||
}
|
||||
std::vector<llama_token> ids = tokenize(text);
|
||||
if (ids.empty()) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: empty prompt\n");
|
||||
return 1;
|
||||
}
|
||||
// retry_badcase_ratio_threshold * text length + 10, capped by max_len of the reference
|
||||
max_len = std::min((int) ids.size() * 6 + 10, 4096);
|
||||
|
||||
// [<|audio_prompt_start|>, voice rows, <|audio_prompt_end|>,] text, <|audio_start|>
|
||||
if (inp->speaker_ref) {
|
||||
if (!encode_speaker(inp->speaker_ref, voice)) {
|
||||
return 1;
|
||||
}
|
||||
rows.push_back({ tok_ref_start, -1 });
|
||||
for (int i = 0; i < (int) (voice.size() / n_embd); i++) {
|
||||
rows.push_back({ LLAMA_TOKEN_NULL, i });
|
||||
}
|
||||
rows.push_back({ tok_ref_end, -1 });
|
||||
}
|
||||
for (llama_token t : ids) {
|
||||
rows.push_back({ t, -1 });
|
||||
}
|
||||
rows.push_back({ tok_audio_start, -1 });
|
||||
|
||||
// the residual LM reads the prompt rows, then one row per frame after the first
|
||||
max_len = std::min(max_len, n_ctx_res + 1 - (int) rows.size());
|
||||
if (max_len <= 0) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: the prompt is too long for the residual LM (%d rows)\n", n_ctx_res);
|
||||
return 1;
|
||||
}
|
||||
if (rows.size() + (size_t) max_len >= llama_n_ctx_seq(lctx)) {
|
||||
LOG_WRN("mtmd_helper_gen_audio: the context may be too small for the prompt and %d frames\n", max_len);
|
||||
}
|
||||
if (!llama_memory_seq_rm(llama_get_memory(lctx), seq_id, 0, -1)) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: cannot reset the sequence\n");
|
||||
return 1;
|
||||
}
|
||||
return init_gen_state(inp->seed) ? 0 : 1;
|
||||
}
|
||||
|
||||
// the residual LM reads every prompt row but the last, which makes the first frame
|
||||
int32_t step_prompt(int32_t n_batch) override {
|
||||
GGML_ASSERT(n_batch > 0);
|
||||
const int n_prompt = (int) rows.size();
|
||||
if (prompt_pos >= n_prompt) {
|
||||
return 0;
|
||||
}
|
||||
// a batch is either text tokens or voice rows; small batches, every row outputs its hidden state
|
||||
const bool is_voice = rows[prompt_pos].voice >= 0;
|
||||
int n = 0;
|
||||
while (n < std::min(n_batch, 64) && prompt_pos + n < n_prompt && (rows[prompt_pos + n].voice >= 0) == is_voice) {
|
||||
n++;
|
||||
}
|
||||
|
||||
if (is_voice) {
|
||||
decode_embd_batch batch(voice.data() + (size_t) rows[prompt_pos].voice * n_embd, n, 1, n_embd);
|
||||
batch.set_position_normal(pos, seq_id);
|
||||
std::fill(batch.logits.begin(), batch.logits.end(), 1);
|
||||
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, n)) != 0) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
std::vector<llama_token> toks(n);
|
||||
for (int i = 0; i < n; i++) {
|
||||
toks[i] = rows[prompt_pos + i].tok;
|
||||
}
|
||||
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
|
||||
if (!batch_set_tokens(batch.get(), toks.data(), n, pos, seq_id)) {
|
||||
return -1;
|
||||
}
|
||||
for (int i = 0; i < n; i++) {
|
||||
llama_batch_ext_set_output_logits(batch.get(), i, true);
|
||||
}
|
||||
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
// the rows go to the residual LM together, the first frame comes from the last prompt row
|
||||
const bool last = prompt_pos + n == n_prompt;
|
||||
const int n_res = last ? n - 1 : n;
|
||||
std::vector<float> h_rows((size_t) n_res * n_embd);
|
||||
for (int i = 0; i < n_res; i++) {
|
||||
const float * h = llama_get_embeddings_ith(lctx, i);
|
||||
std::copy(h, h + n_embd, h_rows.begin() + (size_t) i * n_embd);
|
||||
}
|
||||
if (last) {
|
||||
const float * h = llama_get_embeddings_ith(lctx, n - 1);
|
||||
h_state_buf.assign(h, h + n_embd);
|
||||
}
|
||||
const float * x = is_voice ? voice.data() + (size_t) rows[prompt_pos].voice * n_embd : nullptr;
|
||||
if (n_res > 0 && !run_row(h_rows.data(), x, -1, nullptr, n_res)) {
|
||||
return -1;
|
||||
}
|
||||
pos += n;
|
||||
prompt_pos += n;
|
||||
return n_prompt - prompt_pos;
|
||||
}
|
||||
|
||||
// the hidden state of the caller is not read, the server may decode other sequences in between
|
||||
int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
|
||||
GGML_UNUSED(sampled);
|
||||
GGML_UNUSED(h_state_in);
|
||||
*h_state_out = nullptr;
|
||||
*out_stop = false;
|
||||
if (n_frames >= max_len || (uint32_t) pos >= llama_n_ctx_seq(lctx)) {
|
||||
*out_stop = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
mtmd_gen_out out{};
|
||||
if (!run_row(h_state_buf.data(), embd_in.empty() ? nullptr : embd_in.data(), 0, &out)) {
|
||||
return 1;
|
||||
}
|
||||
latents.insert(latents.end(), out.feats, out.feats + out.n_feats);
|
||||
n_frames++;
|
||||
*h_state_out = h_state_buf.data();
|
||||
|
||||
// the frame of the stopping step is kept, see the loop of VoxCPM2Model._inference()
|
||||
if ((out.is_eos && n_frames > min_len + 1) || n_frames >= max_len) {
|
||||
*out_stop = true;
|
||||
return 0;
|
||||
}
|
||||
|
||||
embd_in.assign(out.embd, out.embd + n_embd);
|
||||
decode_embd_batch batch(embd_in.data(), 1, 1, n_embd);
|
||||
batch.set_position_normal(pos, seq_id);
|
||||
batch.logits[0] = 1;
|
||||
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
|
||||
return 1;
|
||||
}
|
||||
pos++;
|
||||
const float * h = llama_get_embeddings_ith(lctx, -1);
|
||||
h_state_buf.assign(h, h + n_embd);
|
||||
|
||||
if (stream && latents.size() / n_frame - n_decoded >= stream_frames) {
|
||||
return decode_wav() ? 0 : 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
|
||||
*out_sample_rate = info.sample_rate;
|
||||
|
||||
if (!stream) {
|
||||
if (flush() != 0) {
|
||||
return 1;
|
||||
}
|
||||
if (out_n_samples) {
|
||||
*out_n_samples = (int64_t) audio_pcm.size();
|
||||
}
|
||||
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
|
||||
*out_data = (const char *) audio_pcm.data();
|
||||
*out_data_len = audio_pcm.size() * sizeof(float);
|
||||
return 0;
|
||||
}
|
||||
out_buf.clear();
|
||||
if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
|
||||
return 1;
|
||||
}
|
||||
*out_data = out_buf.data();
|
||||
*out_data_len = out_buf.size();
|
||||
return 0;
|
||||
}
|
||||
|
||||
// streaming: only the audio since the previous call
|
||||
const size_t n_new = audio_pcm.size() - pcm_sent;
|
||||
if (out_n_samples) {
|
||||
*out_n_samples = (int64_t) n_new;
|
||||
}
|
||||
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
|
||||
*out_data = (const char *) (audio_pcm.data() + pcm_sent);
|
||||
*out_data_len = n_new * sizeof(float);
|
||||
pcm_sent = audio_pcm.size();
|
||||
return 0;
|
||||
}
|
||||
out_buf.clear();
|
||||
if (!wav_header_sent) {
|
||||
write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
|
||||
wav_header_sent = true;
|
||||
}
|
||||
append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
|
||||
pcm_sent = audio_pcm.size();
|
||||
*out_data = out_buf.data();
|
||||
*out_data_len = out_buf.size();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t flush() override {
|
||||
return decode_wav() ? 0 : 1;
|
||||
}
|
||||
|
||||
private:
|
||||
struct prompt_row {
|
||||
llama_token tok; // text row
|
||||
int voice; // row of the voice reference, -1 for a text row
|
||||
};
|
||||
|
||||
bool ensure_cache() {
|
||||
if (cache_ready) {
|
||||
return true;
|
||||
}
|
||||
tok_audio_start = find_special_token(vocab, "<|audio_start|>");
|
||||
tok_ref_start = find_special_token(vocab, "<|audio_prompt_start|>");
|
||||
tok_ref_end = find_special_token(vocab, "<|audio_prompt_end|>");
|
||||
if (tok_audio_start == LLAMA_TOKEN_NULL || tok_ref_start == LLAMA_TOKEN_NULL || tok_ref_end == LLAMA_TOKEN_NULL) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: missing voxcpm2 audio tokens in vocab\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
// multi-character CJK tokens are split into single characters, see mask_multichar_chinese_tokens()
|
||||
// a token is split when its text without the SPM space (U+2581) is a CJK token of the vocab, each char maps to its plain token
|
||||
const std::string spm_space = "\xe2\x96\x81";
|
||||
const int32_t n_vocab = llama_vocab_n_tokens(vocab);
|
||||
std::unordered_map<std::string, llama_token> pieces;
|
||||
for (llama_token id = 0; id < n_vocab; id++) {
|
||||
pieces.emplace(llama_vocab_get_text(vocab, id), id);
|
||||
}
|
||||
for (llama_token id = 0; id < n_vocab; id++) {
|
||||
std::string clean = llama_vocab_get_text(vocab, id);
|
||||
for (size_t i; (i = clean.find(spm_space)) != std::string::npos;) {
|
||||
clean.erase(i, spm_space.size());
|
||||
}
|
||||
if (clean.size() < 6 || !is_cjk(clean) || pieces.find(clean) == pieces.end()) {
|
||||
continue;
|
||||
}
|
||||
std::vector<llama_token> ids;
|
||||
for (size_t i = 0; i < clean.size(); i += 3) {
|
||||
auto it = pieces.find(clean.substr(i, 3));
|
||||
if (it == pieces.end()) {
|
||||
break;
|
||||
}
|
||||
ids.push_back(it->second);
|
||||
}
|
||||
if (ids.size() * 3 == clean.size()) {
|
||||
split_cjk.emplace(id, std::move(ids));
|
||||
}
|
||||
}
|
||||
cache_ready = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
// chars of U+4E00..U+9FFF only, each is 3 bytes in UTF-8
|
||||
static bool is_cjk(const std::string & s) {
|
||||
if (s.empty() || s.size() % 3 != 0) {
|
||||
return false;
|
||||
}
|
||||
for (size_t i = 0; i < s.size(); i += 3) {
|
||||
const auto b0 = (unsigned char) s[i];
|
||||
const auto b1 = (unsigned char) s[i + 1];
|
||||
const auto b2 = (unsigned char) s[i + 2];
|
||||
if ((b0 & 0xF0) != 0xE0 || (b1 & 0xC0) != 0x80 || (b2 & 0xC0) != 0x80) {
|
||||
return false;
|
||||
}
|
||||
const uint32_t cp = ((b0 & 0x0F) << 12) | ((b1 & 0x3F) << 6) | (b2 & 0x3F);
|
||||
if (cp < 0x4E00 || cp > 0x9FFF) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// no BOS, special tokens in the text are plain text, like tokenize() of the reference
|
||||
std::vector<llama_token> tokenize(const std::string & text) const {
|
||||
std::vector<llama_token> ids(text.size() + 16);
|
||||
const int32_t n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), ids.data(), (int32_t) ids.size(), false, false);
|
||||
if (n < 0) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
|
||||
return {};
|
||||
}
|
||||
std::vector<llama_token> out;
|
||||
for (int32_t i = 0; i < n; i++) {
|
||||
auto it = split_cjk.find(ids[i]);
|
||||
if (it != split_cjk.end()) {
|
||||
out.insert(out.end(), it->second.begin(), it->second.end());
|
||||
} else {
|
||||
out.push_back(ids[i]);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// backbone rows through mtmd: code0 < 0 for n_rows prompt rows, else one row and the next frame goes to out
|
||||
bool run_row(const float * h, const float * x, int32_t code0, mtmd_gen_out * out = nullptr, int n_rows = 1) {
|
||||
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
|
||||
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
|
||||
inp.code0 = code0;
|
||||
inp.embd = const_cast<float *>(h);
|
||||
inp.feats = x;
|
||||
inp.n_feats = x ? (size_t) n_rows * n_embd : 0;
|
||||
inp.n_codes = (size_t) n_rows;
|
||||
if (temp >= 0.0f) {
|
||||
inp.temp = temp;
|
||||
}
|
||||
if (flow_steps > 0) {
|
||||
inp.n_steps = flow_steps;
|
||||
}
|
||||
if (cfg_scale > 0.0f) {
|
||||
inp.cfg_scale = cfg_scale;
|
||||
}
|
||||
mtmd_gen_out tmp{};
|
||||
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, out ? out : &tmp) != 0) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: voxcpm2 gen_code failed\n");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// the AudioVAE decoder is causal and keeps its state, so the latents can be decoded in pieces
|
||||
bool decode_wav() {
|
||||
const size_t n_total = latents.size() / n_frame;
|
||||
while (n_decoded < n_total) {
|
||||
const size_t n = std::min(n_total - n_decoded, (size_t) 16);
|
||||
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
|
||||
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
|
||||
inp.feats = latents.data() + n_decoded * n_frame;
|
||||
inp.n_feats = n * n_frame;
|
||||
mtmd_gen_out out{};
|
||||
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: AudioVAE decode failed\n");
|
||||
return false;
|
||||
}
|
||||
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
|
||||
n_decoded += n;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static constexpr int min_len = 2; // min_len of the reference
|
||||
static constexpr size_t n_frame = 64 * 4; // latents of one frame, n_latent * n_patch
|
||||
static constexpr size_t stream_frames = 2; // frames per decoder call when streaming (0.32 s)
|
||||
static constexpr int n_ctx_res = 8192; // rows of the residual LM, see mtmd_graph_voxcpm2_gen::n_ctx_res
|
||||
|
||||
llama_token tok_audio_start = LLAMA_TOKEN_NULL;
|
||||
llama_token tok_ref_start = LLAMA_TOKEN_NULL;
|
||||
llama_token tok_ref_end = LLAMA_TOKEN_NULL;
|
||||
std::unordered_map<llama_token, std::vector<llama_token>> split_cjk;
|
||||
bool cache_ready = false;
|
||||
|
||||
float temp = -1.0f;
|
||||
float cfg_scale = 0.0f;
|
||||
int32_t flow_steps = 0;
|
||||
|
||||
llama_seq_id seq_id = 0;
|
||||
llama_pos pos = 0;
|
||||
std::vector<prompt_row> rows;
|
||||
std::vector<float> voice; // backbone rows of the voice reference
|
||||
int prompt_pos = 0;
|
||||
int n_frames = 0;
|
||||
int max_len = 0;
|
||||
|
||||
std::vector<float> h_state_buf; // hidden state of the last backbone row
|
||||
std::vector<float> embd_in; // input row of the last backbone row, empty for the text row
|
||||
std::vector<float> latents; // frame-major
|
||||
size_t n_decoded = 0; // frames already through the decoder
|
||||
|
||||
std::vector<float> audio_pcm;
|
||||
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
|
||||
std::vector<char> out_buf;
|
||||
bool stream = false;
|
||||
size_t pcm_sent = 0; // samples already returned by get_output()
|
||||
bool wav_header_sent = false;
|
||||
};
|
||||
|
||||
static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
|
||||
switch (mtmd_gen_audio_get_info(mctx).type) {
|
||||
case MTMD_GEN_AUDIO_TYPE_QWEN3TTS:
|
||||
@@ -4940,6 +5367,8 @@ static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lc
|
||||
return std::unique_ptr<mtmd_gen_audio_pipeline>(new music3_gen_audio_pipeline(lctx, mctx));
|
||||
case MTMD_GEN_AUDIO_TYPE_FISH_S2:
|
||||
return std::unique_ptr<mtmd_gen_audio_pipeline>(new fish_s2_gen_audio_pipeline(lctx, mctx));
|
||||
case MTMD_GEN_AUDIO_TYPE_VOXCPM2:
|
||||
return std::unique_ptr<mtmd_gen_audio_pipeline>(new voxcpm2_gen_audio_pipeline(lctx, mctx));
|
||||
default:
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
@@ -622,6 +622,8 @@ enum projector_type {
|
||||
PROJECTOR_TYPE_ZONOS3_GEN,
|
||||
PROJECTOR_TYPE_ZONOS3_SPKENC,
|
||||
PROJECTOR_TYPE_YUE2_GEN,
|
||||
PROJECTOR_TYPE_VOXCPM2_GEN,
|
||||
PROJECTOR_TYPE_VOXCPM2_SPKENC,
|
||||
PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN,
|
||||
PROJECTOR_TYPE_FISH_S2_GEN,
|
||||
PROJECTOR_TYPE_FISH_S2_SPKENC,
|
||||
@@ -720,6 +722,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
|
||||
{ PROJECTOR_TYPE_ZONOS3_GEN, "zonos3_gen"},
|
||||
{ PROJECTOR_TYPE_ZONOS3_SPKENC, "zonos3_spkenc"},
|
||||
{ PROJECTOR_TYPE_YUE2_GEN, "yue2_gen"},
|
||||
{ PROJECTOR_TYPE_VOXCPM2_GEN, "voxcpm2_gen"},
|
||||
{ PROJECTOR_TYPE_VOXCPM2_SPKENC, "voxcpm2_spkenc"},
|
||||
{ PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN, "minimax_music3_gen"},
|
||||
{ PROJECTOR_TYPE_FISH_S2_GEN, "fish_s2_gen"},
|
||||
{ PROJECTOR_TYPE_FISH_S2_SPKENC, "fish_s2_spkenc"},
|
||||
|
||||
+150
-1
@@ -1105,6 +1105,12 @@ struct mtmd_model_loader {
|
||||
{
|
||||
hparams.audio_sample_rate = 48000;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VOXCPM2_GEN:
|
||||
{
|
||||
hparams.audio_sample_rate = 48000;
|
||||
// the flow graph unrolls up to n_steps_max LocDiT passes
|
||||
model.max_nodes = 65536;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
{
|
||||
hparams.audio_sample_rate = 44100;
|
||||
@@ -1145,6 +1151,14 @@ struct mtmd_model_loader {
|
||||
hparams.flow_n_step = 1;
|
||||
hparams.gen_eos_threshold = -4.0f;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VOXCPM2_SPKENC:
|
||||
{
|
||||
// AudioVAE encoder, one backbone row per patch of latents (checked against the strides at load)
|
||||
hparams.audio_sample_rate = 16000;
|
||||
hparams.audio_patch_sizes = { 2, 5, 8, 8, mtmd_graph_voxcpm2_gen::n_patch };
|
||||
// whole patches, about 10 s like a typical reference clip
|
||||
hparams.warmup_audio_size = hparams.audio_samples_per_token() * 63;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_FISH_S2_SPKENC:
|
||||
{
|
||||
// Fish DAC encoder, codec.pth has no config
|
||||
@@ -1411,7 +1425,8 @@ struct mtmd_model_loader {
|
||||
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
|
||||
model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
|
||||
model.proj_type != PROJECTOR_TYPE_POCKETTTS_SPKENC &&
|
||||
model.proj_type != PROJECTOR_TYPE_FISH_S2_SPKENC;
|
||||
model.proj_type != PROJECTOR_TYPE_FISH_S2_SPKENC &&
|
||||
model.proj_type != PROJECTOR_TYPE_VOXCPM2_SPKENC;
|
||||
|
||||
// Validate audio hparams loaded from GGUF metadata
|
||||
if (hparams.n_mel_bins <= 0 || (fft_based && hparams.n_mel_bins > 256)) {
|
||||
@@ -1691,6 +1706,20 @@ struct mtmd_model_loader {
|
||||
}
|
||||
}
|
||||
|
||||
// DAC-style residual unit: Snake, dilated conv, Snake, conv k = 1 (voxcpm2 AudioVAE)
|
||||
auto load_dac_res = [&](const std::string & q) {
|
||||
mtmd_code2wav::dac_res ru;
|
||||
ru.act1_alpha = get_tensor(q + ".snake1.alpha");
|
||||
ru.act1_beta = get_tensor(q + ".snake1.beta");
|
||||
ru.conv1_w = get_tensor(q + ".conv1.weight");
|
||||
ru.conv1_b = get_tensor(q + ".conv1.bias");
|
||||
ru.act2_alpha = get_tensor(q + ".snake2.alpha");
|
||||
ru.act2_beta = get_tensor(q + ".snake2.beta");
|
||||
ru.conv2_w = get_tensor(q + ".conv2.weight");
|
||||
ru.conv2_b = get_tensor(q + ".conv2.bias");
|
||||
return ru;
|
||||
};
|
||||
|
||||
// code2wav: RVQ codes -> raw PCM, drama3 has no quant/pre_conv/tfm projections
|
||||
auto load_code2wav = [&](bool with_proj) {
|
||||
auto & c2w = model.c2w;
|
||||
@@ -3032,6 +3061,119 @@ struct mtmd_model_loader {
|
||||
throw std::runtime_error(mtmd_format("%s: yue2: unexpected VAE decoder layout\n", __func__));
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VOXCPM2_GEN:
|
||||
{
|
||||
// generic a.gen.code.blk.* tensors are the LocDiT layers, LocEnc has the same hparams
|
||||
using g = mtmd_graph_voxcpm2_gen;
|
||||
auto & v = model.voxcpm2;
|
||||
v.rope_factors = get_tensor("a.gen.rope_factors");
|
||||
|
||||
v.enc_layers.resize(hparams.n_layer);
|
||||
for (int il = 0; il < hparams.n_layer; il++) {
|
||||
auto & layer = v.enc_layers[il];
|
||||
const char * p = "a.gen.enc";
|
||||
layer.ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight"));
|
||||
layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight"));
|
||||
layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight"));
|
||||
layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight"));
|
||||
layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight"));
|
||||
layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight"));
|
||||
layer.ff_gate_w = get_tensor(mtmd_format(TN_FFN_GATE, p, il, "weight"));
|
||||
layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight"));
|
||||
layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight"));
|
||||
}
|
||||
v.enc_special = get_tensor("a.gen.enc.special_token");
|
||||
v.enc_in_w = get_tensor("a.gen.enc.in_proj.weight");
|
||||
v.enc_in_b = get_tensor("a.gen.enc.in_proj.bias");
|
||||
v.enc_norm_w = get_tensor("a.gen.enc.post_ln.weight");
|
||||
v.enc_to_lm_w = get_tensor("a.gen.enc_to_lm.weight");
|
||||
v.enc_to_lm_b = get_tensor("a.gen.enc_to_lm.bias");
|
||||
|
||||
v.dit_in_w = get_tensor("a.gen.dit.in_proj.weight");
|
||||
v.dit_in_b = get_tensor("a.gen.dit.in_proj.bias");
|
||||
v.dit_cond_w = get_tensor("a.gen.dit.cond_proj.weight");
|
||||
v.dit_cond_b = get_tensor("a.gen.dit.cond_proj.bias");
|
||||
v.dit_out_w = get_tensor("a.gen.dit.out_proj.weight");
|
||||
v.dit_out_b = get_tensor("a.gen.dit.out_proj.bias");
|
||||
v.time0_w = get_tensor("a.gen.dit.time_mlp.0.weight");
|
||||
v.time0_b = get_tensor("a.gen.dit.time_mlp.0.bias");
|
||||
v.time1_w = get_tensor("a.gen.dit.time_mlp.1.weight");
|
||||
v.time1_b = get_tensor("a.gen.dit.time_mlp.1.bias");
|
||||
v.lm_to_dit_w = get_tensor("a.gen.lm_to_dit.weight");
|
||||
v.lm_to_dit_b = get_tensor("a.gen.lm_to_dit.bias");
|
||||
v.res_to_dit_w = get_tensor("a.gen.res_to_dit.weight");
|
||||
v.res_to_dit_b = get_tensor("a.gen.res_to_dit.bias");
|
||||
|
||||
v.fsq_in_w = get_tensor("a.gen.fsq.in.weight");
|
||||
v.fsq_in_b = get_tensor("a.gen.fsq.in.bias");
|
||||
v.fsq_out_w = get_tensor("a.gen.fsq.out.weight");
|
||||
v.fsq_out_b = get_tensor("a.gen.fsq.out.bias");
|
||||
v.fusion_w = get_tensor("a.gen.fusion.weight");
|
||||
v.fusion_b = get_tensor("a.gen.fusion.bias");
|
||||
v.stop_proj_w = get_tensor("a.gen.stop_proj.weight");
|
||||
v.stop_proj_b = get_tensor("a.gen.stop_proj.bias");
|
||||
v.stop_head_w = get_tensor("a.gen.stop_head.weight");
|
||||
|
||||
const std::string p = "a.gen.vae";
|
||||
v.dec_in_dw_w = get_tensor(p + ".conv_in_dw.weight");
|
||||
v.dec_in_dw_b = get_tensor(p + ".conv_in_dw.bias");
|
||||
v.dec_in_w = get_tensor(p + ".conv_in.weight");
|
||||
v.dec_in_b = get_tensor(p + ".conv_in.bias");
|
||||
int64_t ratio = 1;
|
||||
for (int b = 0; gguf_find_tensor(ctx_gguf.get(), mtmd_format("%s.blk.%d.up.weight", p.c_str(), b).c_str()) >= 0; b++) {
|
||||
const std::string q = p + ".blk." + std::to_string(b);
|
||||
mtmd_code2wav::dac_block blk;
|
||||
blk.snake_alpha = get_tensor(q + ".snake.alpha");
|
||||
blk.snake_beta = get_tensor(q + ".snake.beta");
|
||||
blk.conv_w = get_tensor(q + ".up.weight");
|
||||
blk.conv_b = get_tensor(q + ".up.bias");
|
||||
for (int r = 0; r < 3; r++) {
|
||||
blk.res.push_back(load_dac_res(q + ".res." + std::to_string(r)));
|
||||
}
|
||||
ratio *= blk.conv_w->ne[0] / 2;
|
||||
v.dec_sr_scale.push_back(get_tensor(q + ".sr_scale"));
|
||||
v.dec_sr_bias.push_back(get_tensor(q + ".sr_bias"));
|
||||
v.dec.push_back(std::move(blk));
|
||||
}
|
||||
v.dec_post_alpha = get_tensor(p + ".post_snake.alpha");
|
||||
v.dec_post_beta = get_tensor(p + ".post_snake.beta");
|
||||
v.dec_out_w = get_tensor(p + ".conv_out.weight");
|
||||
v.dec_out_b = get_tensor(p + ".conv_out.bias");
|
||||
|
||||
// the graphs hard-code these shapes
|
||||
if (v.rope_factors->ne[0] != g::head_dim / 2 || v.dit_in_w->ne[0] != g::n_latent ||
|
||||
v.dec_in_dw_w->ne[2] != g::n_latent || ratio != g::dec_ratio || v.dec_out_w->ne[2] != 1 ||
|
||||
model.layers.empty() || model.layers[0].q_w->ne[1] != (int64_t) g::head_dim * hparams.n_head) {
|
||||
throw std::runtime_error(mtmd_format("%s: voxcpm2: unexpected tensor shapes\n", __func__));
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VOXCPM2_SPKENC:
|
||||
{
|
||||
using g = mtmd_graph_voxcpm2_gen;
|
||||
auto & v = model.voxcpm2;
|
||||
const std::string p = "a.vae.enc";
|
||||
v.enc_conv_in_w = get_tensor(p + ".conv_in.weight");
|
||||
v.enc_conv_in_b = get_tensor(p + ".conv_in.bias");
|
||||
int64_t hop = 1;
|
||||
for (int b = 0; gguf_find_tensor(ctx_gguf.get(), mtmd_format("%s.blk.%d.down.weight", p.c_str(), b).c_str()) >= 0; b++) {
|
||||
const std::string q = p + ".blk." + std::to_string(b);
|
||||
mtmd_code2wav::dac_block blk;
|
||||
for (int r = 0; r < 3; r++) {
|
||||
blk.res.push_back(load_dac_res(q + ".res." + std::to_string(r)));
|
||||
}
|
||||
blk.snake_alpha = get_tensor(q + ".snake.alpha");
|
||||
blk.snake_beta = get_tensor(q + ".snake.beta");
|
||||
blk.conv_w = get_tensor(q + ".down.weight");
|
||||
blk.conv_b = get_tensor(q + ".down.bias");
|
||||
hop *= blk.conv_w->ne[0] / 2;
|
||||
v.enc.push_back(std::move(blk));
|
||||
}
|
||||
v.enc_mu_w = get_tensor(p + ".fc_mu.weight");
|
||||
v.enc_mu_b = get_tensor(p + ".fc_mu.bias");
|
||||
if (hop * g::n_patch != hparams.audio_samples_per_token() || v.enc_mu_w->ne[2] != g::n_latent) {
|
||||
throw std::runtime_error(mtmd_format("%s: voxcpm2: unexpected AudioVAE encoder layout\n", __func__));
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
{
|
||||
// generic a.gen.code.* tensors (depth decoder layers, position_embd, post_ln) are loaded above
|
||||
@@ -4855,11 +4997,13 @@ int mtmd_model::n_output_tokens(const mtmd_image_f32 * img) const {
|
||||
case PROJECTOR_TYPE_YUE2_GEN:
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
case PROJECTOR_TYPE_FISH_S2_GEN:
|
||||
case PROJECTOR_TYPE_VOXCPM2_GEN:
|
||||
{
|
||||
// one hidden-state vector fed back to the talker per call
|
||||
n_patches = 1;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_FISH_S2_SPKENC:
|
||||
case PROJECTOR_TYPE_VOXCPM2_SPKENC:
|
||||
{
|
||||
// one backbone row per codec frame, the preprocessor pads to whole frames
|
||||
n_patches = (int) (img->nx() / hparams.audio_samples_per_token());
|
||||
@@ -5055,6 +5199,11 @@ int mtmd_model::n_mmproj_embd() const {
|
||||
return gen_code_out_embd_w->ne[0];
|
||||
case PROJECTOR_TYPE_YUE2_GEN:
|
||||
return yue2.inp_w->ne[1];
|
||||
case PROJECTOR_TYPE_VOXCPM2_GEN:
|
||||
return voxcpm2.enc_to_lm_w->ne[1];
|
||||
case PROJECTOR_TYPE_VOXCPM2_SPKENC:
|
||||
// the rows come from LocEnc of the gen model, see voxcpm2_ref_encode()
|
||||
return hparams.projection_dim;
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
return gen_code_out_embd_w->ne[0];
|
||||
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
|
||||
|
||||
@@ -217,6 +217,8 @@ struct mtmd_hparams {
|
||||
static constexpr int32_t qwen3tts_max_ref_seconds = 30;
|
||||
// fish_s2: the codec encoder runs at 44.1 kHz, memory grows with the clip
|
||||
static constexpr int32_t fish_s2_max_spk_seconds = 30;
|
||||
// voxcpm2: the AudioVAE encoder runs at 16 kHz, memory grows with the clip
|
||||
static constexpr int32_t voxcpm2_max_spk_seconds = 30;
|
||||
int32_t seanet_n_stage = 0;
|
||||
std::vector<int32_t> seanet_ratios; // encoder order (reversed compared to the config)
|
||||
int32_t mimi_downsample = 0; // encoder frame rate / model frame rate
|
||||
@@ -1067,6 +1069,68 @@ struct mtmd_music3 {
|
||||
std::vector<mtmd_layer> dit;
|
||||
};
|
||||
|
||||
// voxcpm2: LocEnc, LocDiT projections, FSQ, residual LM fusion, stop head and the causal AudioVAE
|
||||
// the LocDiT layers are model.layers, the residual LM layers are loaded by libllama, see mtmd_voxcpm2_res_load()
|
||||
struct mtmd_voxcpm2 {
|
||||
ggml_tensor * rope_factors = nullptr; // LongRoPE factors of LocEnc and LocDiT
|
||||
|
||||
// LocEnc: [special token, patch latents] -> output of the special token -> backbone input row
|
||||
std::vector<mtmd_layer> enc_layers;
|
||||
ggml_tensor * enc_special = nullptr;
|
||||
ggml_tensor * enc_in_w = nullptr;
|
||||
ggml_tensor * enc_in_b = nullptr;
|
||||
ggml_tensor * enc_norm_w = nullptr;
|
||||
ggml_tensor * enc_to_lm_w = nullptr;
|
||||
ggml_tensor * enc_to_lm_b = nullptr;
|
||||
|
||||
// LocDiT: [lm, res, time, cond patch, noisy patch] -> velocity of the noisy patch
|
||||
ggml_tensor * dit_in_w = nullptr;
|
||||
ggml_tensor * dit_in_b = nullptr;
|
||||
ggml_tensor * dit_cond_w = nullptr;
|
||||
ggml_tensor * dit_cond_b = nullptr;
|
||||
ggml_tensor * dit_out_w = nullptr;
|
||||
ggml_tensor * dit_out_b = nullptr;
|
||||
ggml_tensor * time0_w = nullptr; // time_mlp, the constant dt term is folded into time1_b
|
||||
ggml_tensor * time0_b = nullptr;
|
||||
ggml_tensor * time1_w = nullptr;
|
||||
ggml_tensor * time1_b = nullptr;
|
||||
ggml_tensor * lm_to_dit_w = nullptr;
|
||||
ggml_tensor * lm_to_dit_b = nullptr;
|
||||
ggml_tensor * res_to_dit_w = nullptr;
|
||||
ggml_tensor * res_to_dit_b = nullptr;
|
||||
|
||||
// backbone hidden state: FSQ (audio rows), residual LM input, stop head
|
||||
ggml_tensor * fsq_in_w = nullptr;
|
||||
ggml_tensor * fsq_in_b = nullptr;
|
||||
ggml_tensor * fsq_out_w = nullptr;
|
||||
ggml_tensor * fsq_out_b = nullptr;
|
||||
ggml_tensor * fusion_w = nullptr;
|
||||
ggml_tensor * fusion_b = nullptr;
|
||||
ggml_tensor * stop_proj_w = nullptr;
|
||||
ggml_tensor * stop_proj_b = nullptr;
|
||||
ggml_tensor * stop_head_w = nullptr;
|
||||
|
||||
// AudioVAE decoder (gen): depthwise conv, conv k = 1, upsample blocks with a sample-rate scale and bias, Snake, conv
|
||||
ggml_tensor * dec_in_dw_w = nullptr;
|
||||
ggml_tensor * dec_in_dw_b = nullptr;
|
||||
ggml_tensor * dec_in_w = nullptr;
|
||||
ggml_tensor * dec_in_b = nullptr;
|
||||
std::vector<ggml_tensor *> dec_sr_scale;
|
||||
std::vector<ggml_tensor *> dec_sr_bias;
|
||||
std::vector<mtmd_code2wav::dac_block> dec;
|
||||
ggml_tensor * dec_post_alpha = nullptr;
|
||||
ggml_tensor * dec_post_beta = nullptr;
|
||||
ggml_tensor * dec_out_w = nullptr;
|
||||
ggml_tensor * dec_out_b = nullptr;
|
||||
|
||||
// AudioVAE encoder (spkenc): conv, downsample blocks (residual units, Snake, strided conv), fc_mu
|
||||
ggml_tensor * enc_conv_in_w = nullptr;
|
||||
ggml_tensor * enc_conv_in_b = nullptr;
|
||||
std::vector<mtmd_code2wav::dac_block> enc;
|
||||
ggml_tensor * enc_mu_w = nullptr;
|
||||
ggml_tensor * enc_mu_b = nullptr;
|
||||
};
|
||||
|
||||
struct mtmd_vocos {
|
||||
ggml_tensor * input_w = nullptr;
|
||||
ggml_tensor * input_b = nullptr;
|
||||
@@ -1362,6 +1426,7 @@ struct mtmd_model {
|
||||
std::vector<mtmd_drama3_layer> drama3_layers;
|
||||
mtmd_zonos3_wav zonos3;
|
||||
mtmd_yue2 yue2;
|
||||
mtmd_voxcpm2 voxcpm2;
|
||||
mtmd_music3 music3;
|
||||
mtmd_snake_vae snake_vae; // yue2, minimax_music3
|
||||
mtmd_tiny_voca tiny_voca;
|
||||
|
||||
@@ -394,6 +394,7 @@ static std::vector<c2w_state_slot> list_gen_state_slots(const mtmd_hparams & hpa
|
||||
case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model);
|
||||
case PROJECTOR_TYPE_LFM3A_GEN: return list_lfm3a_gen_state_slots(hparams, model);
|
||||
case PROJECTOR_TYPE_ZONOS3_GEN: return list_zonos3_gen_state_slots(hparams, model);
|
||||
case PROJECTOR_TYPE_VOXCPM2_GEN: return list_voxcpm2_state_slots(hparams, model);
|
||||
default: return {};
|
||||
}
|
||||
}
|
||||
@@ -1600,6 +1601,44 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
{
|
||||
set_input_f32("inp_feats", *params->feats);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_VOXCPM2_GEN:
|
||||
{
|
||||
auto set_positions = [&](const char * name) {
|
||||
std::vector<int32_t> pos(ggml_nelements(get_inp_tensor(name)));
|
||||
for (size_t i = 0; i < pos.size(); i++) {
|
||||
pos[i] = (int32_t) i;
|
||||
}
|
||||
set_input_i32(name, pos);
|
||||
};
|
||||
if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE) {
|
||||
// feats: the patch of the previous frame
|
||||
set_input_f32("inp_cond", *params->feats);
|
||||
// flow matching starts from gaussian noise, std = temp
|
||||
ggml_tensor * t = get_inp_tensor("inp_noise");
|
||||
std::vector<float> noise(ggml_nelements(t), 0.0f);
|
||||
if (params->temp > 0.0f) {
|
||||
std::normal_distribution<float> dist(0.0f, params->temp);
|
||||
for (auto & v : noise) {
|
||||
v = dist(params->gen_state->rng);
|
||||
}
|
||||
}
|
||||
set_input_f32("inp_noise", noise);
|
||||
const int n_steps = (int) (ggml_nelements(get_inp_tensor("inp_temb")) / hparams.n_embd);
|
||||
set_input_f32("inp_temb", mtmd_graph_voxcpm2_gen::time_embd(n_steps, hparams.n_embd));
|
||||
set_positions("inp_pos_dit");
|
||||
set_positions("inp_pos_enc");
|
||||
} else if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_RES_INP) {
|
||||
if (params->feats) {
|
||||
set_input_f32("inp_feats", *params->feats);
|
||||
}
|
||||
} else if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_LATENT_IN) {
|
||||
set_input_f32("inp_feats", *params->feats);
|
||||
set_positions("inp_pos_enc");
|
||||
} else if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
|
||||
set_input_f32("inp_feats", *params->feats);
|
||||
set_gen_state_in();
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_KOKORO:
|
||||
{
|
||||
const auto * io = params->kokoro;
|
||||
@@ -1830,6 +1869,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
|
||||
case PROJECTOR_TYPE_ZONOS3_SPKENC:
|
||||
case PROJECTOR_TYPE_FISH_S2_SPKENC:
|
||||
case PROJECTOR_TYPE_VOXCPM2_SPKENC:
|
||||
{
|
||||
// do nothing
|
||||
} break;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user