mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-17 16:27:39 -06:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
744c68cd14 | ||
|
|
307e3f1551 | ||
|
|
9048bad8d6 | ||
|
|
a9d43ecc30 | ||
|
|
8c16babba5 | ||
|
|
ad9d9e8f50 | ||
|
|
955c4e8ee5 | ||
|
|
10152bc4a0 | ||
|
|
8777c0c833 | ||
|
|
918076a635 | ||
|
|
ebc9fb933e | ||
|
|
8b8ca4c98e | ||
|
|
08a9b35893 | ||
|
|
acad5e405c | ||
|
|
d3064fddce |
@@ -0,0 +1,201 @@
|
||||
name: UI Lib Publish
|
||||
|
||||
# Publishes @ggml-org/llama-ui, the library in ui/lib/ui, to npm.
|
||||
#
|
||||
# The version in ui/lib/ui/package.json is the release. A commit that reaches master with a
|
||||
# version npm does not have builds the library and publishes it; a commit that changes the
|
||||
# library without bumping it publishes nothing and says so on the run, which is the drift that
|
||||
# let 0.3.0 sit a day behind master. Bump the version in the PR whose change should ship, and
|
||||
# this workflow does the rest.
|
||||
#
|
||||
# Auth: no npm token is stored here. The job exchanges its OIDC identity for one (npm trusted
|
||||
# publishing) and signs the tarball with a provenance attestation. release-publish.yml has done
|
||||
# the same exchange on every push to master since it landed, its actions/attest step being the
|
||||
# proof that this repository can be issued a token.
|
||||
#
|
||||
# One-time setup, before this can publish: on the package, Publishing settings, Trusted
|
||||
# Publisher, GitHub Actions, with organization ggml-org, repository llama.cpp, workflow file
|
||||
# ui-lib-publish.yml, and Environment left blank - this job names no environment, since creating
|
||||
# one needs repository admin. Merging this workflow before that exists leaves the publish step
|
||||
# red on master: the release is late rather than wrong, and re-running the job publishes it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: 'Build and pack, but do not publish'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-lib-publish.yml',
|
||||
'ui/lib/ui/**.*'
|
||||
]
|
||||
# the advisory half, so a library change without a bump is annotated on the PR rather than
|
||||
# discovered after the merge: the path filter keeps it off PRs that touch only Llama Chat
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ui-lib-publish.yml',
|
||||
'ui/lib/ui/**.*'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
# never cancel a run mid-publish: a version half-published cannot be taken back. GitHub holds
|
||||
# one pending run per group, so a newer push replaces an older queued one - no release is lost,
|
||||
# as each run reads the version from its own commit.
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
LIB_DIR: ui/lib/ui
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish @ggml-org/llama-ui
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: ubuntu-26.04-arm
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Check npm can publish with an OIDC token
|
||||
run: |
|
||||
npm -v
|
||||
node -e '
|
||||
const v = process.argv[1];
|
||||
const [major, minor] = v.split(".").map(Number);
|
||||
if (major < 11 || (major === 11 && minor < 5)) {
|
||||
console.error(`npm ${v} cannot publish with trusted publishing: 11.5.1 or newer is needed`);
|
||||
process.exit(1);
|
||||
}
|
||||
' "$(npm -v)"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Run type checking
|
||||
run: npm run check
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Build the package and lint it
|
||||
run: npm run prepack
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Compare the version with npm
|
||||
id: version
|
||||
# on a PR this is advisory and must not fail the job: a registry outage is not the
|
||||
# contributor's mistake. On master the publish decision rests on the answer, so it fails.
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node scripts/npm-version-check.mjs
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Note a library change that ships no new version
|
||||
if: github.event_name == 'pull_request' && always() && steps.version.outcome == 'success' && steps.version.outputs.should_publish != 'true'
|
||||
env:
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
run: echo "::warning::$NOTE"
|
||||
|
||||
- name: Publish
|
||||
# master only. npm's trusted publisher matches on the repository and the workflow file,
|
||||
# not the branch, so this condition is the only thing stopping a manual run from a PR
|
||||
# branch - one that anyone with write access can start - from publishing code that no
|
||||
# one reviewed. A dispatch on master remains the way to re-run a release.
|
||||
if: github.ref == 'refs/heads/master' && steps.version.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
# --ignore-scripts publishes the tarball prepack built and publint passed rather than
|
||||
# rebuilding it here; --provenance is implied by trusted publishing and named to be
|
||||
# explicit. A package published with nothing in it cannot be taken back, so refuse first.
|
||||
env:
|
||||
TAG: ${{ steps.version.outputs.tag }}
|
||||
run: |
|
||||
test -d dist || { echo "dist/ is missing: nothing was built, so nothing is published"; exit 1; }
|
||||
npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Verify it landed
|
||||
if: github.ref == 'refs/heads/master' && steps.version.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
NAME: ${{ steps.version.outputs.name }}
|
||||
VERSION: ${{ steps.version.outputs.version }}
|
||||
run: |
|
||||
# the registry can serve a new version some time after publish returns, so retry rather
|
||||
# than call a release that worked a failure
|
||||
for try in 1 2 3 4 5 6; do
|
||||
seen=$(npm view "$NAME@$VERSION" version 2>/dev/null || true)
|
||||
if [ "$seen" = "$VERSION" ]; then
|
||||
echo "$NAME@$VERSION is on npm"
|
||||
npm view "$NAME@$VERSION" dist-tags version dist.integrity
|
||||
exit 0
|
||||
fi
|
||||
echo "not visible yet (attempt $try of 6)"
|
||||
sleep 10
|
||||
done
|
||||
echo "::error::$NAME@$VERSION published without error but is not visible on the registry after 6 attempts. The publish did not fail: check the package on npm before publishing again."
|
||||
exit 1
|
||||
|
||||
- name: Report
|
||||
if: always()
|
||||
# every value reaches the shell through env, not through the script, so a package name
|
||||
# from a PR's package.json cannot inject into this step
|
||||
env:
|
||||
NAME: ${{ steps.version.outputs.name || '@ggml-org/llama-ui' }}
|
||||
VERSION: ${{ steps.version.outputs.version || 'unknown' }}
|
||||
LATEST: ${{ steps.version.outputs.latest }}
|
||||
VERDICT: ${{ steps.version.outputs.verdict || 'not reached' }}
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
SHOULD: ${{ steps.version.outputs.should_publish }}
|
||||
DRY: ${{ inputs.dry_run }}
|
||||
run: |
|
||||
if [ "$DRY" = "true" ]; then published="no (dry run)"
|
||||
elif [ "$SHOULD" = "true" ]; then published=yes
|
||||
else published=no
|
||||
fi
|
||||
{
|
||||
echo "### $NAME $VERSION"
|
||||
echo
|
||||
echo "| | |"
|
||||
echo "| --- | --- |"
|
||||
echo "| npm latest was | ${LATEST:-nothing published yet} |"
|
||||
echo "| verdict | \`$VERDICT\` |"
|
||||
echo "| published | $published |"
|
||||
echo
|
||||
echo "> $NOTE"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Mind the verdict
|
||||
if: always() && github.event_name != 'pull_request' && steps.version.outcome == 'success'
|
||||
# a version going backwards is a mistake; a version not bumped is usually one too, but it
|
||||
# is also what a stories-only commit looks like, so that only fails the run when the
|
||||
# repository variable UI_LIB_PUBLISH_FAIL_ON_UNBUMPED is set to true
|
||||
env:
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
VERDICT: ${{ steps.version.outputs.verdict }}
|
||||
FAIL_ON_UNBUMPED: ${{ vars.UI_LIB_PUBLISH_FAIL_ON_UNBUMPED }}
|
||||
run: |
|
||||
case "$VERDICT" in
|
||||
stale)
|
||||
echo "::error::$NOTE"
|
||||
exit 1 ;;
|
||||
published)
|
||||
if [ "$FAIL_ON_UNBUMPED" = "true" ]; then
|
||||
echo "::error::$NOTE"
|
||||
exit 1
|
||||
fi
|
||||
echo "::warning::$NOTE" ;;
|
||||
esac
|
||||
@@ -912,6 +912,10 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
|
||||
throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");
|
||||
}
|
||||
|
||||
if (params.sampling.ignore_eos && !common_grammar_value(params.sampling.grammar).empty()) {
|
||||
throw std::invalid_argument("error: --ignore-eos is not supported with --grammar or --json-schema\n");
|
||||
}
|
||||
|
||||
const bool skip_model_download =
|
||||
// server will call common_params_handle_models() later, so we skip it here
|
||||
ctx_arg.ex == LLAMA_EXAMPLE_SERVER ||
|
||||
|
||||
@@ -203,6 +203,13 @@ struct common_sampler * common_sampler_init(
|
||||
throw std::invalid_argument("penalty_present must be finite");
|
||||
}
|
||||
const llama_vocab * vocab = llama_model_get_vocab(model);
|
||||
if (!common_grammar_value(params.grammar).empty()) {
|
||||
for (const auto & b : params.logit_bias) {
|
||||
if (b.bias == -INFINITY && llama_vocab_is_eog(vocab, b.token)) {
|
||||
throw std::invalid_argument("EOG tokens cannot be banned when a grammar is set");
|
||||
}
|
||||
}
|
||||
}
|
||||
llama_sampler_chain_params lparams = llama_sampler_chain_default_params();
|
||||
|
||||
lparams.no_perf = params.no_perf;
|
||||
|
||||
@@ -283,6 +283,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Qwen3NextForCausalLM": "qwen",
|
||||
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
|
||||
"PocketTTSModel": "pockettts",
|
||||
"KokoroModel": "kokoro",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
@@ -423,6 +424,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"Qwen3ASRForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
|
||||
"PocketTTSModel": "pockettts",
|
||||
"KokoroModel": "kokoro",
|
||||
"SopranoModel": "soprano",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
|
||||
@@ -653,6 +653,28 @@ class ModelBase:
|
||||
raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported")
|
||||
elif quant_method in ("modelopt", "modelopt_mixed"):
|
||||
dequant_fp8()
|
||||
# FP8_PB_WO: one scale per block, the block size follows from the two shapes
|
||||
for name in self.model_tensors.keys():
|
||||
if not name.endswith(".weight_scale_inv"):
|
||||
continue
|
||||
weight_name = name.removesuffix("_scale_inv")
|
||||
if weight_name not in self.model_tensors:
|
||||
tensors_to_remove.append(name)
|
||||
continue
|
||||
w = self.model_tensors[weight_name]
|
||||
s = self.model_tensors[name]
|
||||
if w().dtype != torch.float8_e4m3fn:
|
||||
continue
|
||||
w_shape, s_shape = w().shape, s().shape
|
||||
block_size = None
|
||||
if prod(s_shape) != 1:
|
||||
if len(s_shape) != len(w_shape) or any(ws % ss for ws, ss in zip(w_shape, s_shape)):
|
||||
raise ValueError(f"FP8 weight {weight_name!r} {list(w_shape)} is not tiled by its scale {list(s_shape)}")
|
||||
block_size = [ws // ss for ws, ss in zip(w_shape, s_shape)]
|
||||
self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)
|
||||
tensors_to_remove.append(name)
|
||||
if self._fp8_as_q8:
|
||||
self._fp8_dequantized.add(weight_name)
|
||||
elif quant_method is not None:
|
||||
raise NotImplementedError(f"Quant method is not yet supported: {quant_method!r}")
|
||||
|
||||
|
||||
@@ -0,0 +1,419 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable
|
||||
|
||||
import torch
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, MmprojModel, gguf, logger
|
||||
|
||||
# Kokoro (StyleTTS2 with an iSTFTNet decoder), https://huggingface.co/hexgrad/Kokoro-82M
|
||||
# text model: the PL-BERT phoneme encoder (ALBERT), one token per phoneme character
|
||||
# mmproj: bert_encoder, prosody predictor, text encoder, decoder and the voices
|
||||
# the text to phonemes step is the "tts" chat template, its dictionaries come from the misaki and nltk python packages
|
||||
|
||||
DEFAULT_VOICE = "af_heart"
|
||||
|
||||
MODULE_PREFIX = {
|
||||
"bert_encoder": "a.gen.bert_enc.",
|
||||
"predictor": "a.gen.pred.",
|
||||
"text_encoder": "a.gen.text.",
|
||||
"decoder": "a.gen.dec.",
|
||||
}
|
||||
|
||||
|
||||
def _checkpoint(dir_model: Path) -> Path | None:
|
||||
paths = sorted(dir_model.glob("kokoro-*.pth"))
|
||||
return paths[0] if paths else None
|
||||
|
||||
|
||||
def _is_kokoro_checkpoint(dir_model: Path) -> bool:
|
||||
if _checkpoint(dir_model) is None or not (dir_model / "config.json").is_file():
|
||||
return False
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
return "plbert" in config and "istftnet" in config and "vocab" in config
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_kokoro_checkpoint)
|
||||
def _load_kokoro_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected Kokoro checkpoint")
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
plbert = config["plbert"]
|
||||
return {
|
||||
"architectures": ["KokoroModel"],
|
||||
"model_type": "kokoro",
|
||||
"num_hidden_layers": plbert["num_hidden_layers"],
|
||||
"hidden_size": plbert["hidden_size"],
|
||||
"intermediate_size": plbert["intermediate_size"],
|
||||
"num_attention_heads": plbert["num_attention_heads"],
|
||||
"max_position_embeddings": plbert["max_position_embeddings"],
|
||||
"vocab_size": config["n_token"],
|
||||
"layer_norm_eps": 1e-12, # AlbertConfig default
|
||||
"kokoro": config,
|
||||
# the mmproj tensors are named explicitly, they do not go through the block tensor map
|
||||
"audio_config": {"num_hidden_layers": 0},
|
||||
}
|
||||
|
||||
|
||||
def _load_modules(dir_model: Path, modules: Iterable[str]) -> dict[str, dict[str, Tensor]]:
|
||||
path = _checkpoint(dir_model)
|
||||
assert path is not None
|
||||
state = torch.load(path, map_location="cpu", mmap=True, weights_only=True)
|
||||
# every module was saved from a DataParallel wrapper
|
||||
return {m: {k.removeprefix("module."): v for k, v in state[m].items()} for m in modules}
|
||||
|
||||
|
||||
def _g2p_fallback(lexicons: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
# model for the words missing from the lexicon: n-grams over units "letter:sounds", learned from the lexicon
|
||||
# key "<0 to 4 units before> <letter>" -> {sounds: probability}; "<s>" pads the start of a word, the letter "$" is its end
|
||||
# the "tts" template decodes it with a beam search
|
||||
order, min_count, stress = 5, 3, "\u02c8\u02cc"
|
||||
vowels = set("AIOQWYaiu\u00e6\u0251\u0252\u0254\u0259\u025b\u025c\u026a\u028a\u028c\u1d7b\u1d4a")
|
||||
|
||||
def sounds(ps: str) -> list[str]:
|
||||
# a stress mark goes with the sound after it, a length mark with the one before
|
||||
out: list[str] = []
|
||||
pre = ""
|
||||
for c in ps:
|
||||
if c in stress:
|
||||
pre += c
|
||||
elif c == "\u02d0" and out:
|
||||
out[-1] += c
|
||||
else:
|
||||
out.append(pre + c)
|
||||
pre = ""
|
||||
return out
|
||||
|
||||
def penalty(letter: str, chunk: str) -> float:
|
||||
# a consonant letter says consonants only; a vowel letter says a vowel, with at most a glide
|
||||
flat = [c for c in chunk if c not in stress and c != "\u02d0"]
|
||||
n_vowel = sum(c in vowels for c in flat)
|
||||
if letter in "bcdfgjkmnpqstvxz" and n_vowel > 0:
|
||||
return -8.0
|
||||
if letter in "aeiou" and flat and (n_vowel == 0 or any(c not in vowels and c not in "jw" for c in flat)):
|
||||
return -8.0
|
||||
return 0.0
|
||||
|
||||
words: dict[str, list[str]] = {}
|
||||
for lexicon in lexicons:
|
||||
for word, v in lexicon.items():
|
||||
ps = v["DEFAULT"] if isinstance(v, dict) else v
|
||||
# same spelling normalization as the template: doubled consonants and "ck" say one sound
|
||||
w = re.sub(r"([b-df-hj-np-tv-z])\1", r"\1", word.lower()).replace("ck", "k")
|
||||
if ps and " " not in ps and len(w) >= 2 and re.fullmatch(r"[a-z']+", w) and w not in words:
|
||||
t = sounds(ps)
|
||||
if len(t) <= 2 * len(w):
|
||||
words[w] = t
|
||||
|
||||
def align(w: str, t: list[str], cost: dict[str, dict[str, float]]) -> list[str] | None:
|
||||
# best split of the sounds over the letters, each letter takes 0, 1 or 2 of them
|
||||
n_l, n_t = len(w), len(t)
|
||||
two = [t[j] + t[j + 1] for j in range(n_t - 1)]
|
||||
best = [[-1e18] * (n_t + 1) for _ in range(n_l + 1)]
|
||||
back = [[0] * (n_t + 1) for _ in range(n_l + 1)]
|
||||
best[0][0] = 0.0
|
||||
for i in range(n_l):
|
||||
letter = w[i]
|
||||
row = cost.setdefault(letter, {})
|
||||
cur, nxt, bk = best[i], best[i + 1], back[i + 1]
|
||||
# the sounds left must fit in the letters left
|
||||
for j in range(max(0, n_t - 2 * (n_l - i)), n_t + 1):
|
||||
base = cur[j]
|
||||
if base < -1e17:
|
||||
continue
|
||||
for k, chunk in ((0, ""), (1, t[j] if j < n_t else None), (2, two[j] if j + 1 < n_t else None)):
|
||||
if chunk is None:
|
||||
break
|
||||
c = row.get(chunk)
|
||||
if c is None:
|
||||
# not seen yet for this letter
|
||||
c = row[chunk] = -12.0 + penalty(letter, chunk)
|
||||
if base + c > nxt[j + k]:
|
||||
nxt[j + k] = base + c
|
||||
bk[j + k] = k
|
||||
if best[n_l][n_t] < -1e17:
|
||||
return None
|
||||
out: list[str] = []
|
||||
j = n_t
|
||||
for i in range(n_l, 0, -1):
|
||||
k = back[i][j]
|
||||
out.append("".join(t[j - k:j]))
|
||||
j -= k
|
||||
return out[::-1]
|
||||
|
||||
# hard EM: align with the current letter -> sounds costs, count, repeat
|
||||
# the costs are learned on a sample of the words, then every word is aligned once
|
||||
def step(items: Iterable[tuple[str, list[str]]], cost: dict[str, dict[str, float]]) -> list[tuple[str, list[str]]]:
|
||||
return [(w, a) for w, a in ((w, align(w, t, cost)) for w, t in items) if a is not None]
|
||||
|
||||
sample = list(words.items())[::max(1, len(words) // 20000)]
|
||||
cost: dict[str, dict[str, float]] = {}
|
||||
for _ in range(5):
|
||||
counts: dict[str, Counter[str]] = defaultdict(Counter)
|
||||
for w, a in step(sample, cost):
|
||||
for c, o in zip(w, a):
|
||||
counts[c][o] += 1
|
||||
# small cost for a silent letter, so that a sound stays on its own letter
|
||||
cost = {c: {o: math.log(n / sum(cnt.values())) - (0.5 if o == "" else 0.0) + penalty(c, o) for o, n in cnt.items()}
|
||||
for c, cnt in counts.items()}
|
||||
aligned = step(words.items(), cost)
|
||||
|
||||
# counts of a unit after each of its contexts, from 4 units down to none
|
||||
ngram: dict[str, Counter[str]] = defaultdict(Counter)
|
||||
n_ctx: Counter[str] = Counter()
|
||||
for w, a in aligned:
|
||||
units = ["<s>"] * (order - 1) + [c + ":" + o for c, o in zip(w, a)] + ["$:"]
|
||||
for i in range(order - 1, len(units)):
|
||||
letter, _, o = units[i].partition(":")
|
||||
for n in range(order):
|
||||
ctx = " ".join(units[i - n:i])
|
||||
ngram[(ctx + " " if ctx else "") + letter][o] += 1
|
||||
n_ctx[ctx] += 1
|
||||
|
||||
# the probability is over all the units after the context, not only the ones of this letter
|
||||
# this tells the search how likely the letter is after the sounds it chose, so it can drop a bad choice
|
||||
model: dict[str, Any] = {}
|
||||
for key, cnt in ngram.items():
|
||||
ctx, _, letter = key.rpartition(" ")
|
||||
# the two shortest contexts keep all the readings that the letter has min_count times or more
|
||||
kept = {o: float(f"{n / n_ctx[ctx]:.4g}") for o, n in cnt.most_common()
|
||||
if n >= min_count or (key.count(" ") < 2 and ngram[letter][o] >= min_count)}
|
||||
if kept:
|
||||
model[key] = kept
|
||||
return model
|
||||
|
||||
|
||||
def _g2p_dicts() -> dict[str, dict[str, Any]] | None:
|
||||
# dictionaries of the "tts" template: the misaki lexicon and the nltk averaged perceptron tagger
|
||||
spec = importlib.util.find_spec("misaki")
|
||||
if spec is None or spec.origin is None:
|
||||
logger.warning("misaki is not installed, the model will take phonemes as prompt (pip install misaki nltk)")
|
||||
return None
|
||||
data = Path(spec.origin).parent / "data"
|
||||
try:
|
||||
nltk = importlib.import_module("nltk")
|
||||
base = Path(str(nltk.data.find("taggers/averaged_perceptron_tagger_eng"))) / "averaged_perceptron_tagger_eng"
|
||||
except (ImportError, LookupError):
|
||||
logger.warning("the nltk tagger is not available, the model will take phonemes as prompt "
|
||||
"(pip install nltk && python -m nltk.downloader averaged_perceptron_tagger_eng)")
|
||||
return None
|
||||
|
||||
def load(path: Path) -> Any:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
# tagger: classes sorted by label, feature -> {"c<class index>": weight}
|
||||
# (index keys: jinja2's map(attribute=...) splits on "." and some labels are punctuation)
|
||||
classes = sorted(load(Path(f"{base}.classes.json")))
|
||||
cid = {c: f"c{i}" for i, c in enumerate(classes)}
|
||||
weights = load(Path(f"{base}.weights.json"))
|
||||
lex = {
|
||||
"gold": load(data / "us_gold.json"),
|
||||
"silver": load(data / "us_silver.json"),
|
||||
"gold_gb": load(data / "gb_gold.json"),
|
||||
"silver_gb": load(data / "gb_silver.json"),
|
||||
}
|
||||
logger.info("gguf: learning the pronunciation of unknown words from the lexicon, this takes a few minutes")
|
||||
# one process per variant
|
||||
with ProcessPoolExecutor(max_workers=2) as pool:
|
||||
lts, lts_gb = pool.map(_g2p_fallback, [[lex["gold"], lex["silver"]], [lex["gold_gb"], lex["silver_gb"]]])
|
||||
return {
|
||||
**lex,
|
||||
"lts": lts,
|
||||
"lts_gb": lts_gb,
|
||||
"pos_weights": {f: {cid[t]: w for t, w in tw.items()} for f, tw in weights.items()},
|
||||
"pos_tagdict": load(Path(f"{base}.tagdict.json")),
|
||||
"pos_classes": {"": classes},
|
||||
}
|
||||
|
||||
|
||||
@ModelBase.register("KokoroModel")
|
||||
@ModelBase.example("hexgrad/Kokoro-82M")
|
||||
class KokoroTextModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.KOKORO
|
||||
|
||||
_LAYER = "encoder.albert_layer_groups.0.albert_layers.0."
|
||||
_TENSOR_MAP = {
|
||||
"embeddings.LayerNorm": gguf.MODEL_TENSOR.TOKEN_EMBD_NORM,
|
||||
"encoder.embedding_hidden_mapping_in": gguf.MODEL_TENSOR.TOKEN_EMBD_PROJ,
|
||||
}
|
||||
# the layers share their weights, stored as block 0
|
||||
_LAYER_TENSOR_MAP = {
|
||||
"attention.query": gguf.MODEL_TENSOR.ATTN_Q,
|
||||
"attention.key": gguf.MODEL_TENSOR.ATTN_K,
|
||||
"attention.value": gguf.MODEL_TENSOR.ATTN_V,
|
||||
"attention.dense": gguf.MODEL_TENSOR.ATTN_OUT,
|
||||
"attention.LayerNorm": gguf.MODEL_TENSOR.ATTN_OUT_NORM,
|
||||
"ffn": gguf.MODEL_TENSOR.FFN_UP,
|
||||
"ffn_output": gguf.MODEL_TENSOR.FFN_DOWN,
|
||||
"full_layer_layer_norm": gguf.MODEL_TENSOR.LAYER_OUT_NORM,
|
||||
}
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
del remote_hf_model_id
|
||||
bert = _load_modules(self.dir_model, ["bert"])["bert"]
|
||||
return {name: (lambda t=t: t) for name, t in bert.items()}
|
||||
|
||||
def set_vocab(self):
|
||||
config = self.hparams["kokoro"]
|
||||
n_vocab = config["n_token"]
|
||||
tokens: list[str] = [f"[UNUSED_{i}]" for i in range(n_vocab)]
|
||||
toktypes = [gguf.TokenType.UNUSED] * n_vocab
|
||||
for text, i in config["vocab"].items():
|
||||
tokens[i] = text
|
||||
toktypes[i] = gguf.TokenType.NORMAL
|
||||
# id 0 pads both ends of the phoneme sequence
|
||||
tokens[0] = "<pad>"
|
||||
toktypes[0] = gguf.TokenType.CONTROL
|
||||
|
||||
# the phonemes are mapped to tokens one character at a time by the audio pipeline
|
||||
self.gguf_writer.add_tokenizer_model("llama")
|
||||
self.gguf_writer.add_tokenizer_pre("default")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores([0.0] * n_vocab)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
self.gguf_writer.add_bos_token_id(0)
|
||||
self.gguf_writer.add_eos_token_id(0)
|
||||
self.gguf_writer.add_pad_token_id(0)
|
||||
self.gguf_writer.add_add_bos_token(False)
|
||||
self.gguf_writer.add_add_eos_token(False)
|
||||
|
||||
dicts = _g2p_dicts()
|
||||
if dicts is not None:
|
||||
template_path = Path(__file__).parent.parent / "models" / "templates" / "hexgrad-Kokoro-82M-tts.jinja"
|
||||
with open(template_path, encoding="utf-8") as f:
|
||||
self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}])
|
||||
self.gguf_writer.add_tts_dicts(dicts)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
self.gguf_writer.add_causal_attention(False)
|
||||
self.gguf_writer.add_pooling_type(gguf.PoolingType.NONE)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
del bid
|
||||
n_embd = self.hparams["hidden_size"]
|
||||
|
||||
if name == "embeddings.word_embeddings.weight":
|
||||
# zero-pad the rows to n_embd, llama.cpp wants token and embedding inputs of the same width
|
||||
data_torch = torch.nn.functional.pad(data_torch, (0, n_embd - data_torch.shape[1]))
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), data_torch)
|
||||
return
|
||||
if name == "embeddings.position_embeddings.weight":
|
||||
# every token has type 0, fold its embedding into the positions
|
||||
type0 = self.model_tensors["embeddings.token_type_embeddings.weight"]()[0]
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.POS_EMBD), data_torch + type0)
|
||||
return
|
||||
if name.startswith(("embeddings.token_type_embeddings.", "pooler.")):
|
||||
return
|
||||
|
||||
key, suffix = name.rsplit(".", 1)
|
||||
if key in self._TENSOR_MAP:
|
||||
yield (self.format_tensor_name(self._TENSOR_MAP[key], suffix="." + suffix), data_torch)
|
||||
return
|
||||
if key.startswith(self._LAYER):
|
||||
tensor = self._LAYER_TENSOR_MAP[key.removeprefix(self._LAYER)]
|
||||
yield (self.format_tensor_name(tensor, 0, suffix="." + suffix), data_torch)
|
||||
return
|
||||
raise ValueError(f"unexpected tensor {name}")
|
||||
|
||||
|
||||
@ModelBase.register("KokoroModel")
|
||||
@ModelBase.example("hexgrad/Kokoro-82M")
|
||||
class KokoroMmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._wnorm_g: dict[str, Tensor] = {}
|
||||
self._wnorm_v: dict[str, Tensor] = {}
|
||||
self._lstm_bias: dict[str, Tensor] = {}
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
return self.global_config.get("audio_config")
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
del remote_hf_model_id
|
||||
tensors: dict[str, Callable[[], Tensor]] = {}
|
||||
for module, state in _load_modules(self.dir_model, MODULE_PREFIX.keys()).items():
|
||||
for name, t in state.items():
|
||||
tensors[MODULE_PREFIX[module] + name] = (lambda t=t: t)
|
||||
return tensors
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
# the names are final already
|
||||
return item
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
config = self.global_config["kokoro"]
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.KOKORO)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(config["hidden_dim"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(config["hidden_dim"])
|
||||
self.gguf_writer.add_gen_audio_block_count(config["n_layer"])
|
||||
self.gguf_writer.add_gen_audio_head_count(1)
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5)
|
||||
if (self.dir_model / "voices" / f"{DEFAULT_VOICE}.pt").is_file():
|
||||
self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
del bid
|
||||
# torch weight_norm: weight = g * v / |v| over dims 1..n
|
||||
if name.endswith((".weight_g", ".weight_v")):
|
||||
base = name[:-len(".weight_g")]
|
||||
(self._wnorm_g if name.endswith("_g") else self._wnorm_v)[base] = data_torch
|
||||
if base in self._wnorm_g and base in self._wnorm_v:
|
||||
g = self._wnorm_g.pop(base).float()
|
||||
v = self._wnorm_v.pop(base).float()
|
||||
w = g * v / v.norm(dim=tuple(range(1, v.dim())), keepdim=True)
|
||||
if base.endswith(".pool"):
|
||||
# depthwise transposed conv [C, 1, 3], stored as one row per tap
|
||||
w = w.squeeze(1).T.contiguous()
|
||||
yield (base + ".weight", w)
|
||||
return
|
||||
# the two LSTM biases are always added
|
||||
if ".bias_ih_l0" in name or ".bias_hh_l0" in name:
|
||||
base = name.replace(".bias_ih_l0", ".bias_l0").replace(".bias_hh_l0", ".bias_l0")
|
||||
if base in self._lstm_bias:
|
||||
yield (base, self._lstm_bias.pop(base).float() + data_torch.float())
|
||||
else:
|
||||
self._lstm_bias[base] = data_torch
|
||||
return
|
||||
# snake alphas are per channel
|
||||
if ".alpha1." in name or ".alpha2." in name:
|
||||
data_torch = data_torch.reshape(-1)
|
||||
yield (name, data_torch)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
del name, bid
|
||||
# tensors read on the host stay F32
|
||||
if "lstm" in new_name or ".shared." in new_name or new_name.startswith(("a.gen.bert_enc.", "voice.")) \
|
||||
or ".duration_proj." in new_name or ".m_source." in new_name or ".pool." in new_name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# CONV_TRANSPOSE_1D takes only F32 weights on some backends
|
||||
if ".ups." in new_name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# the conv ops have no BF16 kernels
|
||||
if n_dims >= 2:
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# a voice is a table of style vectors, the row is picked by the number of phonemes
|
||||
for path in sorted((self.dir_model / "voices").glob("*.pt")):
|
||||
pack = torch.load(path, map_location="cpu", weights_only=True)
|
||||
yield (f"voice.{path.stem}", pack.squeeze(1).float())
|
||||
+72
-2
@@ -275,6 +275,14 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
|
||||
or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight")))
|
||||
):
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
# the reference encoder's convs run with an F32 im2col, see f32_im2col in pockettts-seanet.cpp
|
||||
if new_name.startswith("a.seanet.") or new_name == "a.downsample.conv.weight":
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# codes are picked by the nearest codebook entry: Q8_0 changes about 8% of them, F16 none
|
||||
if new_name.startswith("a.gen.enc.tfm.blk.") and new_name.endswith(".weight") and n_dims == 2:
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
if new_name.startswith("a.gen.enc.quant."):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# ConvTranspose1d kernels: only F16/F32 are implemented, no BF16
|
||||
if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
@@ -298,8 +306,8 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# code2wav tensors are already named by generate_extra_tensors(), pass them through
|
||||
if name.startswith("a.gen.wav."):
|
||||
# code2wav and reference encoder tensors are already named by generate_extra_tensors(), pass them through
|
||||
if name.startswith(("a.gen.wav.", "a.gen.enc.", "a.seanet.", "a.downsample.")):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
@@ -356,6 +364,68 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
yield from self._generate_code2wav_tensors()
|
||||
yield from self._generate_ref_encoder_tensors()
|
||||
|
||||
# mimi encoder of the speech tokenizer: reference clip -> codes, for voice cloning with a transcript (ICL)
|
||||
def _generate_ref_encoder_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
from safetensors.torch import load_file
|
||||
|
||||
state_dict = load_file(self.dir_model / "speech_tokenizer" / "model.safetensors")
|
||||
if "encoder.downsample.conv.weight" not in state_dict:
|
||||
return
|
||||
|
||||
with open(self.dir_model / "speech_tokenizer" / "config.json", "r", encoding="utf-8") as f:
|
||||
enc_config = json.load(f)["encoder_config"]
|
||||
n_stage = len(enc_config["upsampling_ratios"])
|
||||
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
# encoder.encoder.layers: conv_in, then per stage a residual unit, an ELU and a strided conv, then an ELU and conv_out
|
||||
def seanet(idx: int, suffix: str) -> Tensor:
|
||||
return state_dict[f"encoder.encoder.layers.{idx}.{suffix}"]
|
||||
|
||||
for suffix in ("weight", "bias"):
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_CONV_IN, suffix=f".{suffix}"), seanet(0, f"conv.{suffix}"))
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_CONV_OUT, suffix=f".{suffix}"), seanet(3 * n_stage + 2, f"conv.{suffix}"))
|
||||
for stage in range(n_stage):
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_RES_CONV1, stage, suffix=f".{suffix}"), seanet(3 * stage + 1, f"block.1.conv.{suffix}"))
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_RES_CONV2, stage, suffix=f".{suffix}"), seanet(3 * stage + 1, f"block.3.conv.{suffix}"))
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_SCALE_CONV, stage, suffix=f".{suffix}"), seanet(3 * stage + 3, f"conv.{suffix}"))
|
||||
|
||||
yield (self.format_tensor_name(T.A_ENC_DOWNSAMPLE_CONV), state_dict["encoder.downsample.conv.weight"])
|
||||
|
||||
tfm_layer_map = {
|
||||
"input_layernorm.weight": (T.A_GEN_ENC_TFM_ATTN_NORM, ".weight"),
|
||||
"input_layernorm.bias": (T.A_GEN_ENC_TFM_ATTN_NORM, ".bias"),
|
||||
"self_attn.q_proj.weight": (T.A_GEN_ENC_TFM_ATTN_Q, ".weight"),
|
||||
"self_attn.k_proj.weight": (T.A_GEN_ENC_TFM_ATTN_K, ".weight"),
|
||||
"self_attn.v_proj.weight": (T.A_GEN_ENC_TFM_ATTN_V, ".weight"),
|
||||
"self_attn.o_proj.weight": (T.A_GEN_ENC_TFM_ATTN_OUT, ".weight"),
|
||||
"self_attn_layer_scale.scale": (T.A_GEN_ENC_TFM_ATTN_SCALE, ".weight"),
|
||||
"post_attention_layernorm.weight": (T.A_GEN_ENC_TFM_FFN_NORM, ".weight"),
|
||||
"post_attention_layernorm.bias": (T.A_GEN_ENC_TFM_FFN_NORM, ".bias"),
|
||||
"mlp.fc1.weight": (T.A_GEN_ENC_TFM_FFN_UP, ".weight"),
|
||||
"mlp.fc2.weight": (T.A_GEN_ENC_TFM_FFN_DOWN, ".weight"),
|
||||
"mlp_layer_scale.scale": (T.A_GEN_ENC_TFM_FFN_SCALE, ".weight"),
|
||||
}
|
||||
for bid in range(enc_config["num_hidden_layers"]):
|
||||
for key, (tensor_id, suffix) in tfm_layer_map.items():
|
||||
yield (self.format_tensor_name(tensor_id, bid, suffix=suffix), state_dict[f"encoder.encoder_transformer.layers.{bid}.{key}"])
|
||||
|
||||
def rvq_codebook(prefix: str, idx: int) -> Tensor:
|
||||
# checkpoint has EMA accumulators, so codebook = embed_sum / cluster_usage
|
||||
embed_sum = state_dict[f"{prefix}.layers.{idx}.codebook.embed_sum"]
|
||||
cluster_usage = state_dict[f"{prefix}.layers.{idx}.codebook.cluster_usage"]
|
||||
return embed_sum / cluster_usage.clamp_min(1e-5).unsqueeze(-1)
|
||||
|
||||
# the tokenizer keeps the semantic codebook and the first acoustic ones, see encoder_valid_num_quantizers
|
||||
semantic = "encoder.quantizer.semantic_residual_vector_quantizer"
|
||||
acoustic = "encoder.quantizer.acoustic_residual_vector_quantizer"
|
||||
yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_FIRST_IN), state_dict[f"{semantic}.input_proj.weight"].squeeze(-1))
|
||||
yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_FIRST_CB), rvq_codebook(semantic, 0))
|
||||
yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_REST_IN), state_dict[f"{acoustic}.input_proj.weight"].squeeze(-1))
|
||||
yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_REST_CB),
|
||||
torch.stack([rvq_codebook(acoustic, i) for i in range(self._CODE_GEN_N_CODEBOOKS)], dim=0))
|
||||
|
||||
def _generate_code2wav_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# code2wav weights live in speech_tokenizer/model.safetensors, not the main safetensors
|
||||
|
||||
+30
-8
@@ -8,7 +8,7 @@ from torch import Tensor
|
||||
import gguf
|
||||
import numpy as np
|
||||
|
||||
from .base import ModelBase
|
||||
from .base import LazyTorchTensor, ModelBase
|
||||
from .qwen import _LinearAttentionVReorderBase, _Qwen35MRopeMixin
|
||||
from .qwen3vl import Qwen3VLVisionModel
|
||||
|
||||
@@ -41,6 +41,18 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
self._ple_shards: dict[int, str] = {}
|
||||
self._ple_row_dim: int | None = None
|
||||
self._mtp_fc: dict[str, Tensor] = {}
|
||||
self._ple_scale: float | None = None
|
||||
|
||||
def dequant_model(self):
|
||||
# FP8 checkpoints: all n-gram shards share one scale, stored under the parent module name
|
||||
# there is no "ngram_embedding.weight", so the generic FP8 pass drops it or cannot map it
|
||||
for name in list(self.model_tensors):
|
||||
if name.endswith(".ngram_embedding.weight_scale"):
|
||||
scale = LazyTorchTensor.to_eager(self.model_tensors.pop(name)()).float().flatten()
|
||||
if scale.numel() != 1:
|
||||
raise ValueError(f"PLE scale {name!r} must be a scalar, got {scale.numel()} values")
|
||||
self._ple_scale = float(scale[0])
|
||||
super().dequant_model()
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
@@ -203,21 +215,31 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
f"PLE shard {shard} has row dim {int(shape[-1])}, expected {self._ple_row_dim}")
|
||||
rows += int(shape[0])
|
||||
|
||||
# an FP8 table keeps its bytes and gets a scale tensor, any other type has the scale folded in
|
||||
keep_fp8 = name in self._fp8_e4m3_preserved
|
||||
table = gguf.LazyChunkedTensor(
|
||||
[self._load_ple_shard(shard) for shard in shards],
|
||||
[self._load_ple_shard(shard, keep_fp8) for shard in shards],
|
||||
shape=(rows, self._ple_row_dim),
|
||||
dtype=np.float32,
|
||||
dtype=np.uint8 if keep_fp8 else np.float32,
|
||||
)
|
||||
gguf_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD]
|
||||
return [(gguf_name + ".weight", cast(Tensor, table))]
|
||||
out = [(gguf_name + ".weight", cast(Tensor, table))]
|
||||
if keep_fp8 and self._ple_scale is not None:
|
||||
out.append((gguf_name + ".scale", torch.tensor([self._ple_scale], dtype=torch.float32)))
|
||||
return out
|
||||
|
||||
def _load_ple_shard(self, name: str):
|
||||
def _load_ple_shard(self, name: str, keep_fp8: bool):
|
||||
def load() -> np.ndarray:
|
||||
from .base import LazyTorchTensor
|
||||
|
||||
# a fresh lazy tensor every call, or to_eager() memoizes every shard
|
||||
eager = LazyTorchTensor.to_eager(self.model_tensors[name]())
|
||||
return eager.to(torch.float32).contiguous().numpy()
|
||||
if keep_fp8:
|
||||
if eager.dtype != torch.float8_e4m3fn:
|
||||
raise ValueError(f"PLE shard {name} is {eager.dtype}, expected FP8 like the other shards")
|
||||
return eager.contiguous().view(torch.uint8).numpy()
|
||||
data = eager.to(torch.float32)
|
||||
if self._ple_scale is not None:
|
||||
data = data * self._ple_scale
|
||||
return data.contiguous().numpy()
|
||||
return load
|
||||
|
||||
def prepare_tensors(self):
|
||||
|
||||
@@ -15,6 +15,7 @@ Supported models:
|
||||
- [Fish Audio S2 Pro](#fish-audio-s2-pro)
|
||||
- [YuE2](#yue2) (songs)
|
||||
- [MiniMax Music 3](#minimax-music-3) (songs)
|
||||
- [Kokoro](#kokoro)
|
||||
|
||||
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
|
||||
|
||||
@@ -23,6 +24,7 @@ The sections below list the params supported by each model, with `llama-tts` exa
|
||||
Available params:
|
||||
- `--tts-lang` can be `zh`, `en`, `de`, `it`, `pt`, `es`, `ja`, `ko`, `fr`, `ru` (default: `en`)
|
||||
- `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3)
|
||||
- `--tts-speaker-text` is the transcript of the speaker file. With it, the speech continues from the reference codes (the "ICL" mode of the reference), which clones the voice more closely. It needs a clip of 30 s or less and an mmproj converted with the speech tokenizer encoder, otherwise it falls back to the speaker embedding only
|
||||
|
||||
Example usage:
|
||||
|
||||
@@ -34,6 +36,16 @@ llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Voice cloning with the transcript of the reference:
|
||||
|
||||
```sh
|
||||
llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \
|
||||
-p "Hello world" \
|
||||
--tts-speaker-file speaker.wav \
|
||||
--tts-speaker-text "The words spoken in speaker.wav." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
## Pocket TTS
|
||||
|
||||
Available params:
|
||||
@@ -352,3 +364,62 @@ python convert_hf_to_gguf.py MiniMax-Music3 --mmproj --outfile mmproj-minimax-mu
|
||||
```
|
||||
|
||||
The mmproj holds the depth decoder and the DiT layers (in the `--outtype` type), and the condition encoder, the small DiT projections and the DAC decoder, always in F32.
|
||||
|
||||
## Kokoro
|
||||
|
||||
Available params:
|
||||
- `--tts-voice` selects one of the voices shipped with the weights (default: `af_heart`); `llama-tts` lists them when the name is unknown. The first letter of a voice is its language: `a` American English, `b` British English, `e` Spanish, `f` French, `h` Hindi, `i` Italian, `p` Brazilian Portuguese, `j` Japanese, `z` Mandarin
|
||||
- `--tts-lang` can be `en-us` or `en-gb`; when it is not given, the voices named `b*` speak British English and those named `a*` American English
|
||||
- Note: `--tts-speaker-file` is not supported, the model has no voice cloning
|
||||
- Note: sampling params are not used, the model is not autoregressive. `--seed` only changes the noise of the vocoder source
|
||||
- `speed` (`llama-server` only) changes the speaking rate, e.g. `1.5`; `loudness` is not supported
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m kokoro.gguf \
|
||||
-mm mmproj-kokoro.gguf \
|
||||
-p "Hello world" \
|
||||
--tts-voice am_michael \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
The model speaks phonemes. Only English has a text to phonemes step (the model's `tts` chat template, with the pronunciation dictionary stored in the model):
|
||||
- the pronunciation of a word missing from the dictionary (mostly rare names) is predicted from its spelling, by a model that the converter learns from the dictionary. The reference uses espeak-ng for these words; the prediction is about as accurate, but it is not the same, so such a word can sound different from the reference
|
||||
- a text longer than 510 phonemes is split at sentence ends and spoken segment by segment
|
||||
|
||||
**Other languages, or your own phonemes:**
|
||||
|
||||
The prompt of a voice of another language must already be phonemes, in the symbols of the model (see `vocab` in its `config.json`). Run the text to phonemes step in your own code, for example with [espeak-ng](https://github.com/espeak-ng/espeak-ng), which the reference uses for Spanish, French, Hindi, Italian and Brazilian Portuguese:
|
||||
|
||||
```sh
|
||||
PHONEMES=$(echo "El gobierno anunció nuevas medidas." | espeak-ng -q --ipa -v es --stdin | tr '\n' ' ')
|
||||
llama-tts -m kokoro.gguf \
|
||||
-mm mmproj-kokoro.gguf \
|
||||
-p "$PHONEMES" \
|
||||
--tts-voice ef_dora \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Notes:
|
||||
- espeak-ng drops the punctuation, which the model uses for pauses: phonemize the text between the marks and put them back for better prosody
|
||||
- the reference also replaces a few sounds by single symbols (e.g. `tʃ` by `ʧ`), see `espeak.py` of [misaki](https://github.com/hexgrad/misaki)
|
||||
- with `llama-server`, send `"raw_prompt": true` to pass phonemes with any voice, including the English ones
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
Download `config.json`, `kokoro-v1_0.pth` and the `voices/*.pt` files you want from [Kokoro-82M](https://huggingface.co/hexgrad/Kokoro-82M), every voice in the directory is stored in the mmproj:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/Kokoro-82M --outfile kokoro.gguf
|
||||
python convert_hf_to_gguf.py path/to/Kokoro-82M --mmproj --outfile mmproj-kokoro.gguf
|
||||
```
|
||||
|
||||
The pronunciation dictionary and the part-of-speech tagger of the `tts` template come from the `misaki` and `nltk` python packages. The converter also learns the spelling-to-sound model for unknown words from that dictionary, which adds about 15 seconds:
|
||||
|
||||
```sh
|
||||
pip install misaki nltk
|
||||
python -m nltk.downloader averaged_perceptron_tagger_eng
|
||||
```
|
||||
|
||||
Without them the converter writes no `tts` template, and the prompt must be a phoneme string (as produced by misaki).
|
||||
|
||||
@@ -386,7 +386,8 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const int64_t padded_w = IW + 2 * int64_t(PD_X), padded_h = IH + 2 * int64_t(PD_Y);
|
||||
const bool padded_fits = padded_w > 0 && padded_w <= limit && padded_h > 0 && padded_h <= limit &&
|
||||
padded_w * padded_h <= limit && IC * B <= limit / (padded_w * padded_h);
|
||||
if (kernel->type == GGML_TYPE_F16 && (use_mma || use_blas || use_simt) && (use_blas || !small_conv) &&
|
||||
const bool kernel_ok = kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32;
|
||||
if (kernel_ok && (use_mma || use_blas || use_simt) && (use_blas || !small_conv) &&
|
||||
ggml_nelements(input) <= limit && ggml_nelements(kernel) <= limit && total <= limit && padded_fits &&
|
||||
PD_X >= 0 && PD_Y >= 0 && ST_X > 0 && ST_Y > 0 && DL_X > 0 && DL_Y > 0 &&
|
||||
(OW - 1) * ST_X + (KW - 1) * DL_X < padded_w && (OH - 1) * ST_Y + (KH - 1) * DL_Y < padded_h &&
|
||||
@@ -402,6 +403,13 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
conv2d_pad_f16<<<(padded_total + 255) / 256, 256, 0, st>>>(X_D, x_half.get(), int(IW), int(IH), pw, ph,
|
||||
PD_X, PD_Y, padded_total);
|
||||
}
|
||||
// F32 kernels take the same path: stage them once as F16, like the input, and still accumulate in F32.
|
||||
ggml_cuda_pool_alloc<half> k_half(ctx.pool());
|
||||
const half * K_H = (const half *) K_D;
|
||||
if (kernel->type == GGML_TYPE_F32) {
|
||||
K_H = k_half.alloc(ggml_nelements(kernel));
|
||||
ggml_get_to_fp16_cuda(GGML_TYPE_F32)(K_D, k_half.get(), ggml_nelements(kernel), st);
|
||||
}
|
||||
const conv_params padded_params = { pw, ph, OW, OH, KW, KH, ST_X, ST_Y, 0, 0, DL_X, DL_Y, IC, OC, B, total };
|
||||
if (use_blas) {
|
||||
const float alpha = 1.0f, beta = 0.0f;
|
||||
@@ -409,7 +417,7 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
cublasHandle_t cublas_h = ctx.cublas_handle();
|
||||
for (int n = 0; n < B; ++n) {
|
||||
CUBLAS_CHECK(cublasGemmEx(cublas_h, CUBLAS_OP_N, CUBLAS_OP_N, positions, int(OC), int(IC), &alpha,
|
||||
x_half.get() + int64_t(n) * IC * positions, CUDA_R_16F, positions, K_D,
|
||||
x_half.get() + int64_t(n) * IC * positions, CUDA_R_16F, positions, K_H,
|
||||
CUDA_R_16F, int(IC), &beta, Y_D + int64_t(n) * OC * positions, CUDA_R_32F,
|
||||
positions, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP));
|
||||
}
|
||||
@@ -426,11 +434,9 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const dim3 block(device.warp_size, 4);
|
||||
const dim3 grid(unsigned((OW * OH + 63) / 64), unsigned((OC + 63) / 64), unsigned(B * split_k));
|
||||
if (use_mma) {
|
||||
conv2d_launch_implicit_gemm<true>(x_half.get(), (const half *) K_D, result, padded_params, split_k, grid,
|
||||
block, st);
|
||||
conv2d_launch_implicit_gemm<true>(x_half.get(), K_H, result, padded_params, split_k, grid, block, st);
|
||||
} else {
|
||||
conv2d_launch_implicit_gemm<false>(x_half.get(), (const half *) K_D, result, padded_params, split_k, grid,
|
||||
block, st);
|
||||
conv2d_launch_implicit_gemm<false>(x_half.get(), K_H, result, padded_params, split_k, grid, block, st);
|
||||
}
|
||||
if (split_k > 1) {
|
||||
conv2d_reduce_split_k<<<(total + 255) / 256, 256, 0, st>>>(result, Y_D, int(total), int(OC * OW * OH),
|
||||
|
||||
@@ -2263,7 +2263,7 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) {
|
||||
src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;
|
||||
|
||||
// fusion is not universally faster on Pascal
|
||||
if (cc <= GGML_CUDA_CC_PASCAL) {
|
||||
if (cc < GGML_CUDA_CC_VOLTA) {
|
||||
return false;
|
||||
}
|
||||
//we only support fusion for ncols_dst = 1
|
||||
@@ -2275,6 +2275,11 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// do not fuse the matmuls that few-row MMQ takes in ggml_cuda_mul_mat_id
|
||||
if (tensor->op == GGML_OP_MUL_MAT_ID && ggml_cuda_should_use_mmq_few(src0, src1, tensor->src[2], cc)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return use_mul_mat_vec_q;
|
||||
}
|
||||
|
||||
@@ -2426,13 +2431,13 @@ static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_t
|
||||
const ggml_tensor * y = up->src[1];
|
||||
const int cc = ggml_cuda_info().devices[device].cc;
|
||||
|
||||
// For MUL_MAT_ID: same kernel choice as ggml_cuda_mul_mat_id, with the nominal tiles of all experts (the per-expert token counts are only known on the GPU).
|
||||
// For MUL_MAT_ID: the MMVQ/MMQ choice of ggml_cuda_mul_mat_id, with the nominal tiles of all experts (the per-expert token counts are only known on the GPU).
|
||||
const bool is_id = up->op == GGML_OP_MUL_MAT_ID;
|
||||
const int64_t ncols = is_id ? y->ne[2] : y->ne[1];
|
||||
const int64_t nchannels = is_id ? x_up->ne[2] : y->ne[2]*y->ne[3];
|
||||
const bool use_mmvq = is_id ? ncols <= get_mmvq_mmid_max_batch(x_up->type, cc) : ggml_cuda_should_use_mmvq(x_up->type, cc, ncols);
|
||||
// do not fuse the matmuls that few-row MMQ takes in ggml_cuda_mul_mat
|
||||
if (!is_id && ggml_cuda_should_use_mmq_few(x_up, y, cc)) {
|
||||
// do not fuse the matmuls that few-row MMQ takes in ggml_cuda_mul_mat; for MUL_MAT_ID the fused MMQ keeps the expert gate and up
|
||||
if (!is_id && ggml_cuda_should_use_mmq_few(x_up, y, nullptr, cc)) {
|
||||
return false;
|
||||
}
|
||||
if (use_mmvq || !ggml_cuda_should_use_mmq(x_up->type, cc, ncols, /*n_experts =*/ is_id ? x_up->ne[2] : 0)) {
|
||||
@@ -2488,13 +2493,13 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
|
||||
return;
|
||||
}
|
||||
// from a per-type number of src1 columns up to 24, few-row MMQ is faster than MMVQ and MMQ
|
||||
const bool use_mmq_few = ggml_cuda_should_use_mmq_few(src0, src1, cc);
|
||||
const bool use_mmq_few = ggml_cuda_should_use_mmq_few(src0, src1, nullptr, cc);
|
||||
if (!use_mmq_few && ggml_cuda_should_use_mmvq(src0->type, cc, ne11)) {
|
||||
ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst);
|
||||
return;
|
||||
}
|
||||
if (use_mmq_few) {
|
||||
ggml_cuda_mul_mat_q_few(ctx, src0, src1, dst);
|
||||
ggml_cuda_mul_mat_q_few(ctx, src0, src1, nullptr, dst);
|
||||
return;
|
||||
}
|
||||
if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) {
|
||||
@@ -2531,6 +2536,10 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c
|
||||
}
|
||||
}
|
||||
|
||||
if (ggml_cuda_should_use_mmq_few(src0, src1, dst->src[2], cc)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[2], /*n_experts=*/src0->ne[2])) {
|
||||
return false;
|
||||
}
|
||||
@@ -2557,8 +2566,11 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
|
||||
|
||||
// [TAG_MUL_MAT_ID_CUDA_GRAPHS]
|
||||
if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
|
||||
// up to 24 tokens few-row MMQ is faster than MMQ, on DGX Spark also faster than MMVQ
|
||||
const bool use_mmq_few = ggml_cuda_should_use_mmq_few(src0, src1, ids, cc);
|
||||
|
||||
static_assert(MMVQ_MAX_BATCH_SIZE == MMVF_MAX_BATCH_SIZE);
|
||||
if (ne2 <= MMVQ_MAX_BATCH_SIZE) {
|
||||
if (!use_mmq_few && ne2 <= MMVQ_MAX_BATCH_SIZE) {
|
||||
if (ggml_is_quantized(src0->type)) {
|
||||
const int mmvq_mmid_max = get_mmvq_mmid_max_batch(src0->type, cc);
|
||||
if (ne2 <= mmvq_mmid_max) {
|
||||
@@ -2578,6 +2590,11 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
|
||||
}
|
||||
}
|
||||
|
||||
if (use_mmq_few) {
|
||||
ggml_cuda_mul_mat_q_few(ctx, src0, src1, ids, dst);
|
||||
return;
|
||||
}
|
||||
|
||||
if (ggml_cuda_should_use_mmq(src0->type, cc, ne12, /*n_experts=*/ne02)) {
|
||||
ggml_cuda_mul_mat_q(ctx, src0, src1, ids, dst);
|
||||
return;
|
||||
|
||||
+134
-67
@@ -1,6 +1,7 @@
|
||||
#include "mmq-few.cuh"
|
||||
#include "cp-async.cuh"
|
||||
#include "mma.cuh"
|
||||
#include "mmvq.cuh"
|
||||
#include "quantize.cuh"
|
||||
#include "vecdotq.cuh"
|
||||
|
||||
@@ -13,6 +14,7 @@ using namespace ggml_cuda_mma;
|
||||
|
||||
// Few-row MMQ: int8 MMA mat-mul for 2-24 src1 columns. A CUDA block has 16 src0 rows (32 for 2 column groups on large matrices).
|
||||
// Its warps split K in units of 256 values and copy raw src0 rows into their own cp.async ring, src1 is quantized to the MMA fragment layout.
|
||||
// MUL_MAT_ID: one column group per expert and CUDA block, for short rows the warps split the rows instead of K (64 rows per block).
|
||||
|
||||
static constexpr int MMQ_FEW_MAX_NCG = 3; // max. number of column groups
|
||||
static constexpr int MMQ_FEW_NROWS = 16; // src0 rows of one MMA tile
|
||||
@@ -80,6 +82,11 @@ static constexpr __host__ __device__ bool mmq_few_tall_supported(const ggml_type
|
||||
type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ4_NL || type == GGML_TYPE_IQ4_XS;
|
||||
}
|
||||
|
||||
// MUL_MAT_ID with fewer than 2 units of 256 values per warp: the warps split the rows, each warp has its own 16 rows.
|
||||
static constexpr __host__ __device__ bool mmq_few_ksplit(const bool has_ids, const int nunits) {
|
||||
return !has_ids || nunits >= 2*MMQ_FEW_NWARPS;
|
||||
}
|
||||
|
||||
#if defined(CP_ASYNC_AVAILABLE)
|
||||
|
||||
// The src1 fragments of a lane for one unit of 256 values, see quantize.cuh.
|
||||
@@ -484,17 +491,19 @@ static __device__ __forceinline__ void mmq_few_vec_dot(const char * const (&x)[2
|
||||
|
||||
#endif // defined(CP_ASYNC_AVAILABLE)
|
||||
|
||||
template <ggml_type type, int ncg, int nmt>
|
||||
// has_ids: MUL_MAT_ID, blockIdx.y is a pair of quantize_mmq_few_id_q8_1, its expert is the src0 channel and dst dims 1 and 2 are slot and token.
|
||||
template <ggml_type type, int ncg, int nmt, bool has_ids>
|
||||
__launch_bounds__(MMQ_FEW_NWARPS*ggml_cuda_get_physical_warp_size(), 1)
|
||||
static __global__ void mul_mat_q_few(
|
||||
const char * x, const int4 * y_ptr, float * dst,
|
||||
const int nunits, const int nrows_x, const int ncols_dst, const int64_t stride_row_x, const int stride_col_dst,
|
||||
const int channel_ratio, const int64_t stride_channel_x, const int stride_channel_dst,
|
||||
const int sample_ratio, const int64_t stride_sample_x, const int stride_sample_dst, const bool x_is_constant) {
|
||||
const int sample_ratio, const int64_t stride_sample_x, const int stride_sample_dst, const bool x_is_constant, const int * pairs) {
|
||||
#if defined(CP_ASYNC_AVAILABLE)
|
||||
constexpr int nwarps = MMQ_FEW_NWARPS;
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
static_assert(warp_size == WARP_SIZE, "unsupported warp size");
|
||||
static_assert(!has_ids || (ncg == 1 && nmt == 1), "unsupported MUL_MAT_ID config");
|
||||
|
||||
constexpr int unit_size = mmq_few_unit_size(type);
|
||||
constexpr int nrows = nmt*MMQ_FEW_NROWS;
|
||||
@@ -510,13 +519,30 @@ static __global__ void mul_mat_q_few(
|
||||
extern __shared__ char data_mmq_few[];
|
||||
char * ring = data_mmq_few + threadIdx.y*nstages*nbytes_stage;
|
||||
|
||||
const int row0 = blockIdx.x*nrows;
|
||||
const bool ksplit = mmq_few_ksplit(has_ids, nunits);
|
||||
const int row0_blk = blockIdx.x*(ksplit ? nrows : nwarps*nrows);
|
||||
const int row0 = ksplit ? row0_blk : row0_blk + threadIdx.y*nrows;
|
||||
const int iu_begin = ksplit ? nunits_stage*threadIdx.y : 0;
|
||||
const int iu_step = ksplit ? nunits_stage*nwarps : nunits_stage;
|
||||
|
||||
const int channel_dst = blockIdx.y;
|
||||
const int sample_dst = blockIdx.z;
|
||||
const int nchannels = gridDim.y;
|
||||
|
||||
x += int64_t(sample_dst/sample_ratio)*stride_sample_x + int64_t(channel_dst/channel_ratio)*stride_channel_x;
|
||||
dst += int64_t(sample_dst)*stride_sample_dst + int64_t(channel_dst)*stride_channel_dst;
|
||||
int ncols = ncols_dst;
|
||||
[[maybe_unused]] const int * pair = nullptr;
|
||||
if constexpr (has_ids) {
|
||||
ggml_cuda_pdl_sync(); // the pairs come from the preceding kernel
|
||||
pair = pairs + channel_dst*MMQ_FEW_PAIR_INTS;
|
||||
ncols = pair[1];
|
||||
if (ncols == 0) {
|
||||
return;
|
||||
}
|
||||
x += int64_t(pair[0])*stride_channel_x;
|
||||
} else {
|
||||
x += int64_t(sample_dst/sample_ratio)*stride_sample_x + int64_t(channel_dst/channel_ratio)*stride_channel_x;
|
||||
dst += int64_t(sample_dst)*stride_sample_dst + int64_t(channel_dst)*stride_channel_dst;
|
||||
}
|
||||
const int4 * GGML_CUDA_RESTRICT y = y_ptr + (int64_t(sample_dst)*nchannels + channel_dst)*nunits*ncg*MMQ_FEW_Q8_1_INT4;
|
||||
|
||||
// No bounds check for the units, the gate makes nunits a multiple of nunits_stage.
|
||||
@@ -562,11 +588,11 @@ static __global__ void mul_mat_q_few(
|
||||
// DGX Spark DRAM serves the short row pieces of the stage copies slowly, so the rows of the block are first prefetched into L2 in address order.
|
||||
{
|
||||
const int tid = threadIdx.y*warp_size + threadIdx.x;
|
||||
const int nrows_blk = min(nrows, nrows_x - row0);
|
||||
const int nrows_blk = min(ksplit ? nrows : nwarps*nrows, nrows_x - row0_blk);
|
||||
const int64_t row_bytes = int64_t(nunits)*unit_size;
|
||||
const bool contig = stride_row_x == row_bytes;
|
||||
for (int ir = 0; ir < (contig ? 1 : nrows_blk); ++ir) {
|
||||
const uintptr_t p0 = (uintptr_t) (x + int64_t(row0 + ir)*stride_row_x);
|
||||
const uintptr_t p0 = (uintptr_t) (x + int64_t(row0_blk + ir)*stride_row_x);
|
||||
const uintptr_t p1 = p0 + (contig ? nrows_blk*row_bytes : row_bytes);
|
||||
for (uintptr_t p = (p0 & ~uintptr_t(127)) + 128*tid; p < p1; p += 128*nwarps*warp_size) {
|
||||
ggml_cuda_prefetch_l2((const void *) p);
|
||||
@@ -575,11 +601,9 @@ static __global__ void mul_mat_q_few(
|
||||
}
|
||||
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
|
||||
const int iu_step = nunits_stage*nwarps;
|
||||
|
||||
#pragma unroll
|
||||
for (int stage = 0; stage < nstages - 1; ++stage) {
|
||||
const int iu0 = nunits_stage*threadIdx.y + stage*iu_step;
|
||||
const int iu0 = iu_begin + stage*iu_step;
|
||||
if (iu0 < nunits) {
|
||||
load_stage(stage, iu0);
|
||||
}
|
||||
@@ -598,16 +622,16 @@ static __global__ void mul_mat_q_few(
|
||||
|
||||
// The next src1 fragments are loaded right after the MMAs that use the current ones, an earlier load is slower.
|
||||
mmq_few_y_frag yf[nunits_stage][ncg];
|
||||
if (nunits_stage*threadIdx.y < nunits) {
|
||||
if (iu_begin < nunits) {
|
||||
#pragma unroll
|
||||
for (int k = 0; k < nunits_stage; ++k) {
|
||||
load_y(yf[k], nunits_stage*threadIdx.y + k);
|
||||
load_y(yf[k], iu_begin + k);
|
||||
}
|
||||
}
|
||||
|
||||
float sum[nmt][ncg][4] = {{{0.0f}}};
|
||||
|
||||
for (int iter = 0, iu0 = nunits_stage*threadIdx.y; iu0 < nunits; ++iter, iu0 += iu_step) {
|
||||
for (int iter = 0, iu0 = iu_begin; iu0 < nunits; ++iter, iu0 += iu_step) {
|
||||
const int iu_load = iu0 + (nstages - 1)*iu_step;
|
||||
if (iu_load < nunits) {
|
||||
load_stage((iter + nstages - 1) % nstages, iu_load);
|
||||
@@ -643,26 +667,28 @@ static __global__ void mul_mat_q_few(
|
||||
ggml_cuda_pdl_lc();
|
||||
|
||||
// reduce the sums over the warps as one flat array
|
||||
constexpr int nsum = nmt*ncg*4;
|
||||
float * sum_flat = &sum[0][0][0];
|
||||
if (ksplit) {
|
||||
constexpr int nsum = nmt*ncg*4;
|
||||
float * sum_flat = &sum[0][0][0];
|
||||
|
||||
float * buf_sum = (float *) data_mmq_few;
|
||||
__syncthreads();
|
||||
if (threadIdx.y > 0) {
|
||||
float * buf_sum = (float *) data_mmq_few;
|
||||
__syncthreads();
|
||||
if (threadIdx.y > 0) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < nsum; ++i) {
|
||||
buf_sum[((threadIdx.y - 1)*nsum + i)*warp_size + threadIdx.x] = sum_flat[i];
|
||||
for (int i = 0; i < nsum; ++i) {
|
||||
buf_sum[((threadIdx.y - 1)*nsum + i)*warp_size + threadIdx.x] = sum_flat[i];
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (threadIdx.y > 0) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (threadIdx.y > 0) {
|
||||
return;
|
||||
}
|
||||
#pragma unroll
|
||||
for (int w = 0; w < nwarps - 1; ++w) {
|
||||
for (int w = 0; w < nwarps - 1; ++w) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < nsum; ++i) {
|
||||
sum_flat[i] += buf_sum[(w*nsum + i)*warp_size + threadIdx.x];
|
||||
for (int i = 0; i < nsum; ++i) {
|
||||
sum_flat[i] += buf_sum[(w*nsum + i)*warp_size + threadIdx.x];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -674,8 +700,13 @@ static __global__ void mul_mat_q_few(
|
||||
for (int l = 0; l < 4; ++l) {
|
||||
const int i = row0 + MMQ_FEW_NROWS*m + tile<16, 8, int>::get_i(l);
|
||||
const int j = c*MMQ_FEW_NCOLS + tile<16, 8, int>::get_j(l);
|
||||
if (i < nrows_x && j < ncols_dst) {
|
||||
dst[int64_t(j)*stride_col_dst + i] = sum[m][c][l];
|
||||
if (i < nrows_x && j < ncols) {
|
||||
if constexpr (has_ids) {
|
||||
const int e = pair[2 + j]; // token*n_expert_used + slot
|
||||
dst[int64_t(e / ncols_dst)*stride_channel_dst + int64_t(e % ncols_dst)*stride_col_dst + i] = sum[m][c][l];
|
||||
} else {
|
||||
dst[int64_t(j)*stride_col_dst + i] = sum[m][c][l];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -683,7 +714,7 @@ static __global__ void mul_mat_q_few(
|
||||
#else
|
||||
GGML_UNUSED_VARS(x, y_ptr, dst, nunits, nrows_x, ncols_dst, stride_row_x, stride_col_dst,
|
||||
channel_ratio, stride_channel_x, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_dst, x_is_constant);
|
||||
sample_ratio, stride_sample_x, stride_sample_dst, x_is_constant, pairs);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(CP_ASYNC_AVAILABLE)
|
||||
}
|
||||
@@ -695,9 +726,10 @@ static bool mmq_few_is_constant(const ggml_tensor * src0) {
|
||||
!(x_root->flags & GGML_TENSOR_FLAG_PARAM);
|
||||
}
|
||||
|
||||
template <ggml_type type, int ncg, int nmt>
|
||||
// MUL_MAT_ID: grid y is the pairs, the kernel takes the expert from the pair and does not use the channel and sample ratios.
|
||||
template <ggml_type type, int ncg, int nmt, bool has_ids = false>
|
||||
static void launch_mul_mat_q_few(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
|
||||
const int4 * src1_q, cudaStream_t stream) {
|
||||
const int4 * src1_q, cudaStream_t stream, const int * pairs = nullptr, const int npairs_max = 0) {
|
||||
GGML_TENSOR_BINARY_OP_LOCALS;
|
||||
|
||||
const int nunits = ne00 / QK_K;
|
||||
@@ -705,18 +737,19 @@ static void launch_mul_mat_q_few(const ggml_tensor * src0, const ggml_tensor * s
|
||||
|
||||
const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;
|
||||
const int nbytes_shared = mmq_few_nbytes_shared(type, ncg, nmt);
|
||||
CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q_few<type, ncg, nmt>), nbytes_shared);
|
||||
CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q_few<type, ncg, nmt, has_ids>), nbytes_shared);
|
||||
|
||||
const bool x_is_constant = mmq_few_is_constant(src0);
|
||||
|
||||
const dim3 block_nums((ne01 + nmt*MMQ_FEW_NROWS - 1) / (nmt*MMQ_FEW_NROWS), ne12, ne13);
|
||||
const int nrows_blk = (mmq_few_ksplit(has_ids, nunits) ? 1 : MMQ_FEW_NWARPS)*nmt*MMQ_FEW_NROWS;
|
||||
const dim3 block_nums((ne01 + nrows_blk - 1) / nrows_blk, has_ids ? npairs_max : ne12, ne13);
|
||||
const dim3 block_dims(warp_size, MMQ_FEW_NWARPS, 1);
|
||||
const ggml_cuda_kernel_launch_params params(block_nums, block_dims, nbytes_shared, stream);
|
||||
ggml_cuda_kernel_launch(mul_mat_q_few<type, ncg, nmt>, params,
|
||||
ggml_cuda_kernel_launch(mul_mat_q_few<type, ncg, nmt, has_ids>, params,
|
||||
(const char *) src0->data, src1_q, (float *) dst->data,
|
||||
nunits, (int) ne01, (int) ne1, (int64_t) nb01, (int) (nb1/sizeof(float)),
|
||||
(int) (ne12/ne02), (int64_t) nb02, (int) (nb2/sizeof(float)),
|
||||
(int) (ne13/ne03), (int64_t) nb03, (int) (nb3/sizeof(float)), x_is_constant);
|
||||
(int) (ne13/ne03), (int64_t) nb03, (int) (nb3/sizeof(float)), x_is_constant, pairs);
|
||||
}
|
||||
|
||||
static int mmq_few_ncg(const int64_t ncols) {
|
||||
@@ -743,7 +776,11 @@ static void launch_mul_mat_q_few_tall(const ggml_tensor * src0, const ggml_tenso
|
||||
|
||||
template <ggml_type type>
|
||||
static void launch_mul_mat_q_few_switch_ncg(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
|
||||
const int4 * src1_q, const int ncg, cudaStream_t stream) {
|
||||
const int4 * src1_q, const int * pairs, const int npairs_max, const int ncg, cudaStream_t stream) {
|
||||
if (pairs) {
|
||||
launch_mul_mat_q_few<type, 1, 1, true>(src0, src1, dst, src1_q, stream, pairs, npairs_max);
|
||||
return;
|
||||
}
|
||||
switch (ncg) {
|
||||
case 1:
|
||||
launch_mul_mat_q_few<type, 1, 1>(src0, src1, dst, src1_q, stream);
|
||||
@@ -759,7 +796,7 @@ static void launch_mul_mat_q_few_switch_ncg(const ggml_tensor * src0, const ggml
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
||||
void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {
|
||||
GGML_TENSOR_BINARY_OP_LOCALS;
|
||||
|
||||
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
||||
@@ -767,51 +804,73 @@ void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor
|
||||
GGML_ASSERT(nb00 == ggml_type_size(src0->type));
|
||||
GGML_ASSERT(nb10 == sizeof(float));
|
||||
GGML_ASSERT(nb0 == sizeof(float));
|
||||
GGML_ASSERT(ne11 <= MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS);
|
||||
GGML_ASSERT(ne00 % QK_K == 0);
|
||||
GGML_ASSERT(ne12 % ne02 == 0 && ne13 % ne03 == 0);
|
||||
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
const int nunits = ne00 / QK_K;
|
||||
const int ncg = mmq_few_ncg(ne11);
|
||||
const int ncg = ids ? 1 : mmq_few_ncg(ne11);
|
||||
|
||||
ggml_cuda_pool_alloc<int4> src1_q(ctx.pool(), int64_t(ne12)*ne13*nunits*ncg*MMQ_FEW_Q8_1_INT4);
|
||||
quantize_mmq_few_q8_1_cuda((const float *) src1->data, src1_q.get(), mmq_few_ds_layout(src0->type), ne10, nb11/sizeof(float), nb12/sizeof(float), nb13/sizeof(float),
|
||||
ne11, ne12, ne13, stream);
|
||||
ggml_cuda_pool_alloc<int> pairs(ctx.pool());
|
||||
ggml_cuda_pool_alloc<int4> src1_q(ctx.pool());
|
||||
int npairs_max = 0;
|
||||
if (ids) {
|
||||
GGML_ASSERT(ids->type == GGML_TYPE_I32);
|
||||
GGML_ASSERT(ids->nb[0] == ggml_type_size(ids->type));
|
||||
GGML_ASSERT(ne1 == ids->ne[0] && ne13 == 1 && ne3 == 1);
|
||||
|
||||
const int64_t n_expert_used = ids->ne[0];
|
||||
const int64_t n_tokens = ids->ne[1];
|
||||
const int64_t nentries = n_tokens*n_expert_used;
|
||||
const int64_t nused = std::min(ne02, nentries);
|
||||
// an expert with c > 0 entries (at most one per token) has ceil(c/MMQ_FEW_NCOLS) pairs
|
||||
npairs_max = (int) std::min(nused + (nentries - nused)/MMQ_FEW_NCOLS, ne02*((n_tokens + MMQ_FEW_NCOLS - 1) / MMQ_FEW_NCOLS));
|
||||
|
||||
pairs.alloc(int64_t(npairs_max)*MMQ_FEW_PAIR_INTS);
|
||||
src1_q.alloc(int64_t(npairs_max)*nunits*MMQ_FEW_Q8_1_INT4);
|
||||
quantize_mmq_few_id_q8_1_cuda((const float *) src1->data, (const int32_t *) ids->data, src1_q.get(), pairs.get(), mmq_few_ds_layout(src0->type),
|
||||
ne00, nb11/sizeof(float), nb12/sizeof(float), ne11, ne02, n_expert_used, n_tokens, ids->nb[1]/sizeof(int32_t), npairs_max, stream);
|
||||
} else {
|
||||
GGML_ASSERT(ne11 <= MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS);
|
||||
GGML_ASSERT(ne12 % ne02 == 0 && ne13 % ne03 == 0);
|
||||
|
||||
src1_q.alloc(int64_t(ne12)*ne13*nunits*ncg*MMQ_FEW_Q8_1_INT4);
|
||||
quantize_mmq_few_q8_1_cuda((const float *) src1->data, src1_q.get(), mmq_few_ds_layout(src0->type), ne10, nb11/sizeof(float), nb12/sizeof(float), nb13/sizeof(float),
|
||||
ne11, ne12, ne13, stream);
|
||||
}
|
||||
|
||||
switch (src0->type) {
|
||||
case GGML_TYPE_Q8_0:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q8_0>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q8_0>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q3_K:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q3_K>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q3_K>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q4_K:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q4_K>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q4_K>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q5_K:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q5_K>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q5_K>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q6_K:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q6_K>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q6_K>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_IQ3_S:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ3_S>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ3_S>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_IQ4_NL:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_NL>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_NL>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_IQ4_XS:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_XS>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_XS>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
default:
|
||||
GGML_ABORT("unsupported type for few-row MMQ");
|
||||
}
|
||||
}
|
||||
|
||||
// Min. number of src1 columns for the few-row MMQ, 0 if it is not used for this type, GPU and row length.
|
||||
static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const int64_t ne00) {
|
||||
// Min. number of src1 columns (MUL_MAT_ID: tokens) for the few-row MMQ, 0 if it is not used for this type, GPU and row length.
|
||||
static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const int64_t ne00, const bool has_ids) {
|
||||
#ifdef GGML_CUDA_FORCE_CUBLAS
|
||||
return 0;
|
||||
#endif // GGML_CUDA_FORCE_CUBLAS
|
||||
@@ -824,13 +883,16 @@ static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const
|
||||
if (cc == GGML_CUDA_CC_DGX_SPARK && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_DGX_SPARK) {
|
||||
return 0;
|
||||
}
|
||||
// with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there
|
||||
if (mmq_few_unit_size(type) == 0 || ne00 < 8*QK_K) {
|
||||
// with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there (MUL_MAT_ID splits the rows instead)
|
||||
if (mmq_few_unit_size(type) == 0 || (!has_ids && ne00 < 8*QK_K)) {
|
||||
return 0;
|
||||
}
|
||||
if (cc == GGML_CUDA_CC_DGX_SPARK) { // tuned on DGX Spark GB10
|
||||
if (cc == GGML_CUDA_CC_DGX_SPARK) { // tuned on DGX Spark GB10, also for MUL_MAT_ID
|
||||
return type == GGML_TYPE_IQ4_XS || type == GGML_TYPE_IQ4_NL ? 8 : 2;
|
||||
}
|
||||
if (has_ids) { // after MMVQ, which is faster up to its max. batch size
|
||||
return get_mmvq_mmid_max_batch(type, cc) + 1;
|
||||
}
|
||||
switch (type) { // all GPUs except DGX Spark, tuned on RTX 5090
|
||||
case GGML_TYPE_Q3_K: return 3;
|
||||
case GGML_TYPE_Q5_K: return 3;
|
||||
@@ -844,14 +906,19 @@ static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const
|
||||
}
|
||||
}
|
||||
|
||||
bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const int cc) {
|
||||
const int min_ncols = ggml_cuda_mmq_few_min_ncols(src0->type, cc, src0->ne[0]);
|
||||
if (min_ncols == 0 || src1->ne[1] < min_ncols || src1->ne[1] > MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS) {
|
||||
bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, const int cc) {
|
||||
const int64_t ncols = ids ? ids->ne[1] : src1->ne[1];
|
||||
const int min_ncols = ggml_cuda_mmq_few_min_ncols(src0->type, cc, src0->ne[0], ids != nullptr);
|
||||
if (min_ncols == 0 || ncols < min_ncols || ncols > MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS) {
|
||||
return false;
|
||||
}
|
||||
// MUL_MAT_ID: the pairs are built in shared memory
|
||||
if (ids && (2*src0->ne[2] + 1)*sizeof(int) > ggml_cuda_info().devices[ggml_cuda_get_device()].smpb) {
|
||||
return false;
|
||||
}
|
||||
// the row must hold a whole number of ring stages of the launched kernel
|
||||
const int ncg = mmq_few_ncg(src1->ne[1]);
|
||||
const int nmt = mmq_few_nmt(src0->type, ncg, src0->ne[1]);
|
||||
const int ncg = ids ? 1 : mmq_few_ncg(ncols);
|
||||
const int nmt = ids ? 1 : mmq_few_nmt(src0->type, ncg, src0->ne[1]);
|
||||
const int nunits_stage = mmq_few_nunits_stage(src0->type, ncg, nmt);
|
||||
if (src0->ne[0] % (nunits_stage*QK_K) != 0) {
|
||||
return false;
|
||||
@@ -863,13 +930,13 @@ bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor *
|
||||
|
||||
#else // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
||||
GGML_UNUSED_VARS(ctx, src0, src1, dst);
|
||||
void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {
|
||||
GGML_UNUSED_VARS(ctx, src0, src1, ids, dst);
|
||||
GGML_ABORT("few-row MMQ is not supported on HIP or MUSA");
|
||||
}
|
||||
|
||||
bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const int cc) {
|
||||
GGML_UNUSED_VARS(src0, src1, cc);
|
||||
bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, const int cc) {
|
||||
GGML_UNUSED_VARS(src0, src1, ids, cc);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
#include "common.cuh"
|
||||
|
||||
bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, int cc);
|
||||
// ids != nullptr: MUL_MAT_ID
|
||||
bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, int cc);
|
||||
|
||||
void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
|
||||
void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);
|
||||
|
||||
@@ -49,6 +49,36 @@ static __global__ void pool1d_nchw_kernel(
|
||||
o_ptr[cur_ow] = res;
|
||||
}
|
||||
|
||||
// a window covering the whole row gives one output per row: reduce it with a block instead of one thread
|
||||
template <ggml_op_pool op>
|
||||
static __global__ void pool1d_row_kernel(const int iw, const float * src, float * dst) {
|
||||
static_assert(op == GGML_OP_POOL_MAX || op == GGML_OP_POOL_AVG, "unsupported pool op");
|
||||
|
||||
const float * row = src + (int64_t) blockIdx.x*iw;
|
||||
|
||||
float res = op == GGML_OP_POOL_MAX ? -FLT_MAX : 0.0f;
|
||||
for (int i = threadIdx.x; i < iw; i += blockDim.x) {
|
||||
res = op == GGML_OP_POOL_MAX ? max(res, row[i]) : res + row[i];
|
||||
}
|
||||
|
||||
__shared__ float buf[WARP_SIZE];
|
||||
res = block_reduce<op == GGML_OP_POOL_MAX ? block_reduce_method::MAX : block_reduce_method::SUM>(res, buf);
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
dst[blockIdx.x] = op == GGML_OP_POOL_MAX ? res : res/iw;
|
||||
}
|
||||
}
|
||||
|
||||
static void pool1d_row_f32_cuda(const int iw, const int64_t nr, const float * src, float * dst, const enum ggml_op_pool op, cudaStream_t stream) {
|
||||
const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm;
|
||||
const dim3 block_dims(nr/nsm < 2 ? 512 : (iw < 1024 ? 32 : 128));
|
||||
if (op == GGML_OP_POOL_MAX) {
|
||||
pool1d_row_kernel<GGML_OP_POOL_MAX><<<nr, block_dims, 0, stream>>>(iw, src, dst);
|
||||
} else {
|
||||
pool1d_row_kernel<GGML_OP_POOL_AVG><<<nr, block_dims, 0, stream>>>(iw, src, dst);
|
||||
}
|
||||
}
|
||||
|
||||
static void pool1d_nchw_kernel_f32_f32_cuda(
|
||||
const int iw, const int ow,
|
||||
const int kw, const int sw, const int pw,
|
||||
@@ -79,6 +109,12 @@ void ggml_cuda_op_pool1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const int64_t OW = dst->ne[0];
|
||||
const int64_t nr = ggml_nrows(src0);
|
||||
|
||||
// short rows: one thread per row is faster
|
||||
if (OW == 1 && p0 == 0 && k0 == IW && IW >= CUDA_POOL1D_ROW_MIN_IW && (op == GGML_OP_POOL_MAX || op == GGML_OP_POOL_AVG)) {
|
||||
pool1d_row_f32_cuda(IW, nr, src0_d, dst_d, op, stream);
|
||||
return;
|
||||
}
|
||||
|
||||
const int parallel_elements = (int)(nr * OW);
|
||||
|
||||
pool1d_nchw_kernel_f32_f32_cuda(IW, OW, k0, s0, p0, parallel_elements, src0_d, dst_d, op, stream);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "common.cuh"
|
||||
|
||||
#define CUDA_POOL1D_BLOCK_SIZE 256
|
||||
#define CUDA_POOL1D_ROW_MIN_IW 256
|
||||
|
||||
void ggml_cuda_op_pool1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
|
||||
+149
-27
@@ -600,6 +600,40 @@ void quantize_mmq_q8_1_cuda(
|
||||
}
|
||||
}
|
||||
|
||||
// Stores the 4 values xi of a thread, values [iu, iu + 4) of column g in a unit of 256 values, in the column group yg of the unit, see quantize.cuh for the layout.
|
||||
template <mmq_q8_1_ds_layout ds_layout>
|
||||
static __device__ __forceinline__ void quantize_mmq_few_store(const float4 xi, int4 * yg, const int iu, const int g) {
|
||||
// the 32 values of a sub-block are held by 8 threads
|
||||
float amax = fmaxf(fmaxf(fabsf(xi.x), fabsf(xi.y)), fmaxf(fabsf(xi.z), fabsf(xi.w)));
|
||||
amax = warp_reduce_max<8>(amax);
|
||||
|
||||
const float d_inv = 127.0f / amax;
|
||||
char4 q;
|
||||
q.x = roundf(xi.x*d_inv);
|
||||
q.y = roundf(xi.y*d_inv);
|
||||
q.z = roundf(xi.z*d_inv);
|
||||
q.w = roundf(xi.w*d_inv);
|
||||
const float d = 1.0f / d_inv;
|
||||
|
||||
// as in quantize_mmq_q8_1: the sum of the quantized values; the int warp_reduce_sum reduces the whole warp on Ampere+
|
||||
[[maybe_unused]] float sum = 0.0f;
|
||||
if constexpr (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) {
|
||||
sum = d*warp_reduce_sum<8>(float(q.x + q.y + q.z + q.w));
|
||||
}
|
||||
|
||||
const int s = iu / QK8_1; // sub-block
|
||||
const int k = iu % QK8_1; // first value in sub-block
|
||||
((int *) (yg + (s/2)*WARP_SIZE + 4*g + (k % 16)/4))[2*(s % 2) + k/16] = *(const int *) &q;
|
||||
if (k == 0) {
|
||||
int4 * ds = yg + 4*WARP_SIZE + 4*(g/2) + s/2;
|
||||
if constexpr (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {
|
||||
((float *) ds)[2*(s % 2) + g % 2] = d;
|
||||
} else {
|
||||
((half2 *) ds)[2*(s % 2) + g % 2] = make_half2(d, sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Each thread quantizes 4 values of one column, see quantize.cuh for the layout.
|
||||
template <mmq_q8_1_ds_layout ds_layout>
|
||||
static __global__ void quantize_mmq_few_q8_1(
|
||||
@@ -619,37 +653,99 @@ static __global__ void quantize_mmq_few_q8_1(
|
||||
const int64_t i3 = blockIdx.z / ne2;
|
||||
const float4 xi = (int) blockIdx.x < ne1 ? *(const float4 *) (x + i3*s03 + i2*s02 + blockIdx.x*s01 + i0) : make_float4(0.0f, 0.0f, 0.0f, 0.0f);
|
||||
|
||||
// the 32 values of a sub-block are held by 8 threads
|
||||
float amax = fmaxf(fmaxf(fabsf(xi.x), fabsf(xi.y)), fmaxf(fabsf(xi.z), fabsf(xi.w)));
|
||||
amax = warp_reduce_max<8>(amax);
|
||||
|
||||
const float d_inv = 127.0f / amax;
|
||||
char4 q;
|
||||
q.x = roundf(xi.x*d_inv);
|
||||
q.y = roundf(xi.y*d_inv);
|
||||
q.z = roundf(xi.z*d_inv);
|
||||
q.w = roundf(xi.w*d_inv);
|
||||
const float d = 1.0f / d_inv;
|
||||
|
||||
// as in quantize_mmq_q8_1: the sum of the quantized values; the int warp_reduce_sum reduces the whole warp on Ampere+
|
||||
[[maybe_unused]] float sum = 0.0f;
|
||||
if constexpr (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) {
|
||||
sum = d*warp_reduce_sum<8>(float(q.x + q.y + q.z + q.w));
|
||||
}
|
||||
|
||||
const int s = i0 % QK_K / QK8_1; // sub-block
|
||||
const int k = i0 % QK8_1; // first value in sub-block
|
||||
const int g = blockIdx.x % MMQ_FEW_NCOLS; // column in group
|
||||
int4 * yg = vy + ((blockIdx.z*(ne00/QK_K) + i0/QK_K)*(gridDim.x/MMQ_FEW_NCOLS) + blockIdx.x/MMQ_FEW_NCOLS)*MMQ_FEW_Q8_1_INT4;
|
||||
((int *) (yg + (s/2)*WARP_SIZE + 4*g + (k % 16)/4))[2*(s % 2) + k/16] = *(const int *) &q;
|
||||
if (k == 0) {
|
||||
int4 * ds = yg + 4*WARP_SIZE + 4*(g/2) + s/2;
|
||||
if constexpr (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {
|
||||
((float *) ds)[2*(s % 2) + g % 2] = d;
|
||||
} else {
|
||||
((half2 *) ds)[2*(s % 2) + g % 2] = make_half2(d, sum);
|
||||
quantize_mmq_few_store<ds_layout>(xi, yg, i0 % QK_K, g);
|
||||
}
|
||||
|
||||
// MUL_MAT_ID: block x is the (token, slot) entry, ids are as in mm_ids_helper. Each block finds the pair of its entry in shared memory, block (0, 0) also writes all pairs.
|
||||
// Entry rank r = number of earlier tokens with the same expert, pair = first pair of the expert + r/MMQ_FEW_NCOLS, column = r % MMQ_FEW_NCOLS.
|
||||
template <mmq_q8_1_ds_layout ds_layout>
|
||||
static __global__ void quantize_mmq_few_id_q8_1(
|
||||
const float * x_ptr, const int32_t * ids, void * vy_ptr, int * pairs, const int64_t ne00, const int64_t s11, const int64_t s12,
|
||||
const int ne11, const int n_experts, const int n_expert_used, const int64_t si1, const int npairs_max) {
|
||||
// per expert: bit t set if token t uses it and the index of its first pair; then the total number of pairs
|
||||
extern __shared__ unsigned int data_mmq_few_id[];
|
||||
unsigned int * tokens = data_mmq_few_id;
|
||||
int * pair0 = (int *) (tokens + n_experts);
|
||||
int & npairs = pair0[n_experts];
|
||||
|
||||
ggml_cuda_pdl_lc();
|
||||
const float * GGML_CUDA_RESTRICT x = x_ptr;
|
||||
int4 * GGML_CUDA_RESTRICT vy = (int4 *) vy_ptr;
|
||||
ggml_cuda_pdl_sync();
|
||||
|
||||
const int nentries = gridDim.x;
|
||||
auto expert_of = [&](const int i) {
|
||||
const int e = ids[(i / n_expert_used)*si1 + i % n_expert_used];
|
||||
return e >= 0 && e < n_experts ? e : -1; // other ids skip the slot
|
||||
};
|
||||
|
||||
for (int e = threadIdx.x; e < n_experts; e += blockDim.x) {
|
||||
tokens[e] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
for (int i = threadIdx.x; i < nentries; i += blockDim.x) {
|
||||
const int e = expert_of(i);
|
||||
if (e >= 0) {
|
||||
atomicOr(&tokens[e], 1u << (i / n_expert_used));
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (threadIdx.x < WARP_SIZE) {
|
||||
const int per_lane = (n_experts + WARP_SIZE - 1) / WARP_SIZE;
|
||||
const int e0 = min(int(threadIdx.x)*per_lane, n_experts);
|
||||
const int e1 = min(e0 + per_lane, n_experts);
|
||||
int n = 0;
|
||||
for (int e = e0; e < e1; ++e) {
|
||||
n += (__popc(tokens[e]) + MMQ_FEW_NCOLS - 1) / MMQ_FEW_NCOLS;
|
||||
}
|
||||
int p = warp_prefix_inclusive_sum<int>(n) - n;
|
||||
for (int e = e0; e < e1; ++e) {
|
||||
pair0[e] = p;
|
||||
p += (__popc(tokens[e]) + MMQ_FEW_NCOLS - 1) / MMQ_FEW_NCOLS;
|
||||
}
|
||||
if (threadIdx.x == WARP_SIZE - 1) {
|
||||
npairs = p;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
auto rank = [&](const int i, const int e) {
|
||||
return __popc(tokens[e] & ((1u << (i / n_expert_used)) - 1u));
|
||||
};
|
||||
|
||||
if (blockIdx.x == 0 && blockIdx.y == 0) {
|
||||
for (int i = threadIdx.x; i < nentries; i += blockDim.x) {
|
||||
const int e = expert_of(i);
|
||||
if (e < 0) {
|
||||
continue;
|
||||
}
|
||||
const int r = rank(i, e);
|
||||
int * pair = pairs + (pair0[e] + r/MMQ_FEW_NCOLS)*MMQ_FEW_PAIR_INTS;
|
||||
pair[2 + r % MMQ_FEW_NCOLS] = i;
|
||||
if (r % MMQ_FEW_NCOLS == 0) {
|
||||
pair[0] = e;
|
||||
pair[1] = min(__popc(tokens[e]) - r, MMQ_FEW_NCOLS);
|
||||
}
|
||||
}
|
||||
for (int p = npairs + threadIdx.x; p < npairs_max; p += blockDim.x) {
|
||||
pairs[p*MMQ_FEW_PAIR_INTS + 1] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const int e = expert_of(blockIdx.x);
|
||||
const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4;
|
||||
if (e < 0 || i0 >= ne00) {
|
||||
return;
|
||||
}
|
||||
const int it = blockIdx.x / n_expert_used;
|
||||
const int is = blockIdx.x % n_expert_used;
|
||||
const float4 xi = *(const float4 *) (x + it*s12 + (is % ne11)*s11 + i0);
|
||||
|
||||
const int r = rank(blockIdx.x, e);
|
||||
int4 * yg = vy + (int64_t(pair0[e] + r/MMQ_FEW_NCOLS)*(ne00/QK_K) + i0/QK_K)*MMQ_FEW_Q8_1_INT4;
|
||||
quantize_mmq_few_store<ds_layout>(xi, yg, i0 % QK_K, r % MMQ_FEW_NCOLS);
|
||||
}
|
||||
|
||||
void quantize_mmq_few_q8_1_cuda(
|
||||
@@ -675,6 +771,32 @@ void quantize_mmq_few_q8_1_cuda(
|
||||
}
|
||||
}
|
||||
|
||||
void quantize_mmq_few_id_q8_1_cuda(
|
||||
const float * x, const int32_t * ids, void * vy, int * pairs, const mmq_q8_1_ds_layout ds_layout, const int64_t ne00, const int64_t s11, const int64_t s12,
|
||||
const int64_t ne11, const int64_t n_experts, const int64_t n_expert_used, const int64_t n_tokens, const int64_t si1, const int64_t npairs_max, cudaStream_t stream) {
|
||||
GGML_ASSERT(ne00 % QK_K == 0);
|
||||
GGML_ASSERT(n_tokens <= 32); // one bit per token
|
||||
|
||||
const int64_t block_num_y = (ne00 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ);
|
||||
const size_t nbytes_shared = (2*n_experts + 1)*sizeof(int);
|
||||
GGML_ASSERT(nbytes_shared <= ggml_cuda_info().devices[ggml_cuda_get_device()].smpb);
|
||||
const dim3 num_blocks(n_tokens*n_expert_used, block_num_y, 1);
|
||||
const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
|
||||
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, nbytes_shared, stream);
|
||||
switch (ds_layout) {
|
||||
case MMQ_Q8_1_DS_LAYOUT_D4:
|
||||
ggml_cuda_kernel_launch(quantize_mmq_few_id_q8_1<MMQ_Q8_1_DS_LAYOUT_D4>, launch_params,
|
||||
x, ids, vy, pairs, ne00, s11, s12, (int) ne11, (int) n_experts, (int) n_expert_used, si1, (int) npairs_max);
|
||||
break;
|
||||
case MMQ_Q8_1_DS_LAYOUT_DS4:
|
||||
ggml_cuda_kernel_launch(quantize_mmq_few_id_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4>, launch_params,
|
||||
x, ids, vy, pairs, ne00, s11, s12, (int) ne11, (int) n_experts, (int) n_expert_used, si1, (int) npairs_max);
|
||||
break;
|
||||
default:
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
}
|
||||
|
||||
// scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row)
|
||||
void quantize_scatter_mmq_q8_1_cuda(
|
||||
const float * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0,
|
||||
|
||||
@@ -32,6 +32,14 @@ void quantize_mmq_few_q8_1_cuda(
|
||||
const float * x, void * vy, mmq_q8_1_ds_layout ds_layout, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
|
||||
int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
|
||||
|
||||
// MUL_MAT_ID: a pair is an expert and one column group of up to MMQ_FEW_NCOLS of its (token, slot) entries. The src1 of pair p is channel p above, its columns past the entries are not written.
|
||||
// A pair is MMQ_FEW_PAIR_INTS ints: expert, number of entries (0 for an unused pair), then the entries token*n_expert_used + slot.
|
||||
static constexpr int MMQ_FEW_PAIR_INTS = 2 + MMQ_FEW_NCOLS;
|
||||
|
||||
void quantize_mmq_few_id_q8_1_cuda(
|
||||
const float * x, const int32_t * ids, void * vy, int * pairs, mmq_q8_1_ds_layout ds_layout, int64_t ne00, int64_t s11, int64_t s12,
|
||||
int64_t ne11, int64_t n_experts, int64_t n_expert_used, int64_t n_tokens, int64_t si1, int64_t npairs_max, cudaStream_t stream);
|
||||
|
||||
void quantize_mmq_fp4_cuda(const float * x,
|
||||
const int32_t * ids,
|
||||
void * vy,
|
||||
|
||||
@@ -580,13 +580,15 @@ static inline float ggml_f8_e4m3_to_fp32(uint8_t x) {
|
||||
return NAN;
|
||||
}
|
||||
|
||||
const int exp = (ax >> 3) & 0xF;
|
||||
const int man = ax & 0x7;
|
||||
const uint32_t exp = (ax >> 3) & 0xF;
|
||||
const uint32_t man = ax & 0x7;
|
||||
// OCP FP8 rev. 1.1, section 5.1: subnormal magnitude = M * 2^-9; normal magnitude = (1 + M / 8) * 2^(E - 7).
|
||||
// a normal value maps to fp32 exponent E + 120 and the top 3 mantissa bits
|
||||
const float value = exp == 0
|
||||
? ldexpf((float) man, -9)
|
||||
: ldexpf(1.0f + (float) man / 8.0f, exp - 7);
|
||||
return x & 0x80 ? -value : value;
|
||||
? (float) man * (1.0f / 512.0f)
|
||||
: fp32_from_bits(((exp + 120) << 23) | (man << 20));
|
||||
// set the sign bit without a branch, the sign of real data is random
|
||||
return fp32_from_bits(fp32_to_bits(value) | ((uint32_t) (x & 0x80) << 24));
|
||||
}
|
||||
|
||||
static inline int ggml_round_to_nearest_even(float x) {
|
||||
|
||||
@@ -1473,6 +1473,7 @@ struct ggml_backend_opencl_context {
|
||||
cl_kernel kernel_gemv_noshuffle_q5_k_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP)
|
||||
cl_kernel kernel_gemm_noshuffle_q5_k_f32;
|
||||
cl_kernel kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin;
|
||||
cl_kernel kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin;
|
||||
cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin;
|
||||
cl_kernel kernel_gemv_noshuffle_q5_k_f32_32b_trans;
|
||||
cl_kernel kernel_gemv_noshuffle_q5_0_f32;
|
||||
@@ -6845,6 +6846,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
|
||||
backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans = nullptr;
|
||||
backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin = nullptr;
|
||||
backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin = nullptr;
|
||||
backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin = nullptr;
|
||||
{
|
||||
if (backend_ctx->has_vector_subgroup_broadcast) {
|
||||
@@ -6879,6 +6881,17 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q5_k_f32_32b_trans_ila_a7", &bin_size);
|
||||
if (kernel_bin && bin_size > 0) {
|
||||
cl_program bin_prog =
|
||||
build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size);
|
||||
|
||||
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin =
|
||||
clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7", &err), err));
|
||||
CL_CHECK(clReleaseProgram(bin_prog));
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8", &bin_size);
|
||||
if (kernel_bin && bin_size > 0) {
|
||||
cl_program bin_prog =
|
||||
@@ -11850,7 +11863,8 @@ inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
|
||||
inline bool use_q5_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
if (!backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans ||
|
||||
!backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin) {
|
||||
!(backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin ||
|
||||
backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin)) {
|
||||
return false;
|
||||
}
|
||||
return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0);
|
||||
@@ -29557,6 +29571,77 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno_ila(ggml_backend_t backend, const gg
|
||||
const int gemm_tile_n = 64;
|
||||
int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n;
|
||||
|
||||
const bool have_a8 = backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin != nullptr;
|
||||
|
||||
if (!have_a8) {
|
||||
cl_mem b_src_sub = nullptr;
|
||||
cl_mem b_pad_sub = nullptr;
|
||||
cl_mem b_buf = nullptr;
|
||||
region.origin = offset1;
|
||||
region.size = (size_t)K * N * sizeof(float);
|
||||
CL_CHECK((b_src_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
|
||||
if (N_pad == N) {
|
||||
b_buf = b_src_sub;
|
||||
} else {
|
||||
size_t padded_bytes = (size_t)K * N_pad * sizeof(float);
|
||||
backend_ctx->prealloc_act_trans.allocate(context, padded_bytes);
|
||||
region.origin = 0;
|
||||
region.size = padded_bytes;
|
||||
CL_CHECK((b_pad_sub = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
|
||||
|
||||
const float b_pad_zero = 0.0f;
|
||||
CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_pad_sub, &b_pad_zero, sizeof(float),
|
||||
0, padded_bytes, 0, NULL, NULL));
|
||||
CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, b_src_sub, b_pad_sub, 0, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL));
|
||||
b_buf = b_pad_sub;
|
||||
}
|
||||
|
||||
img_fmt = { CL_R, CL_FLOAT };
|
||||
memset(&img_desc, 0, sizeof(img_desc));
|
||||
img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
|
||||
img_desc.image_width = (size_t)K * N_pad;
|
||||
img_desc.buffer = b_buf;
|
||||
cl_mem b_img_a7;
|
||||
CL_CHECK((b_img_a7 = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
|
||||
|
||||
const int c_offset_for_kernel = (int)offsetd;
|
||||
|
||||
int line_stride_B = K * (int)sizeof(float);
|
||||
int line_stride_C = M * (int)sizeof(float);
|
||||
|
||||
kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin;
|
||||
cl_uint k_arg = 0;
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->q_img));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->qh));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->d));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->dm));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->s));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img_a7));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset1));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_C));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N));
|
||||
|
||||
size_t local_work_size[3] = { 64, 2, 4 };
|
||||
size_t m_tiles = (size_t)CEIL_DIV(M, 64);
|
||||
size_t global_y = ((m_tiles + 1) / 2) * 2;
|
||||
size_t global_work_size[3] = { 64, global_y, (size_t)CEIL_DIV(N_pad, 32) };
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
|
||||
|
||||
CL_CHECK(clReleaseMemObject(b_img_a7));
|
||||
if (b_src_sub) {
|
||||
CL_CHECK(clReleaseMemObject(b_src_sub));
|
||||
}
|
||||
if (b_pad_sub) {
|
||||
CL_CHECK(clReleaseMemObject(b_pad_sub));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
cl_mem b_sub_buf = nullptr;
|
||||
cl_mem b_padded = nullptr;
|
||||
cl_mem b_buf = nullptr;
|
||||
|
||||
@@ -770,6 +770,7 @@ class MODEL_ARCH(IntEnum):
|
||||
QWEN3TTS = auto()
|
||||
POCKETTTS = auto()
|
||||
ZONOS3 = auto()
|
||||
KOKORO = auto()
|
||||
K2HORIZON = auto()
|
||||
KOLIBRI1 = auto()
|
||||
LIMITE = auto()
|
||||
@@ -794,6 +795,7 @@ class VISION_PROJECTOR_TYPE(IntEnum):
|
||||
class MODEL_TENSOR(IntEnum):
|
||||
TOKEN_EMBD = auto()
|
||||
TOKEN_EMBD_NORM = auto()
|
||||
TOKEN_EMBD_PROJ = auto()
|
||||
MASKED_EMBD_CENTROIDS= auto()
|
||||
MASKED_EMBD_ORDERING = auto()
|
||||
TOKEN_TYPES = auto()
|
||||
@@ -1360,6 +1362,21 @@ class MODEL_TENSOR(IntEnum):
|
||||
A_CODEC_VQ_FIRST_CB = auto() # semantic VQ codebook, l2-normalized
|
||||
A_CODEC_VQ_REST_IN = auto() # residual VQ input projections, merged 3D
|
||||
A_CODEC_VQ_REST_CB = auto() # residual VQ codebooks, l2-normalized, merged 3D
|
||||
# qwen3tts reference encoder: raw PCM -> RVQ codes, the SEANet stack and the downsample conv use A_ENC_*
|
||||
A_GEN_ENC_TFM_ATTN_NORM = auto()
|
||||
A_GEN_ENC_TFM_ATTN_Q = auto()
|
||||
A_GEN_ENC_TFM_ATTN_K = auto()
|
||||
A_GEN_ENC_TFM_ATTN_V = auto()
|
||||
A_GEN_ENC_TFM_ATTN_OUT = auto()
|
||||
A_GEN_ENC_TFM_ATTN_SCALE = auto()
|
||||
A_GEN_ENC_TFM_FFN_NORM = auto()
|
||||
A_GEN_ENC_TFM_FFN_UP = auto()
|
||||
A_GEN_ENC_TFM_FFN_DOWN = auto()
|
||||
A_GEN_ENC_TFM_FFN_SCALE = auto()
|
||||
A_GEN_ENC_QUANT_FIRST_IN = auto()
|
||||
A_GEN_ENC_QUANT_FIRST_CB = auto()
|
||||
A_GEN_ENC_QUANT_REST_IN = auto()
|
||||
A_GEN_ENC_QUANT_REST_CB = auto()
|
||||
# qwen3tts code2wav: RVQ codes -> raw PCM
|
||||
A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D)
|
||||
A_GEN_WAV_QUANT_FIRST_OUT = auto() # semantic RVQ, out_proj
|
||||
@@ -1740,6 +1757,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
|
||||
MODEL_ARCH.QWEN3TTS: "qwen3tts",
|
||||
MODEL_ARCH.POCKETTTS: "pockettts",
|
||||
MODEL_ARCH.ZONOS3: "zonos3",
|
||||
MODEL_ARCH.KOKORO: "kokoro",
|
||||
MODEL_ARCH.K2HORIZON: "k2-horizon",
|
||||
MODEL_ARCH.KOLIBRI1: "kolibri1",
|
||||
MODEL_ARCH.LIMITE: "limite",
|
||||
@@ -1762,6 +1780,7 @@ VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = {
|
||||
TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
|
||||
MODEL_TENSOR.TOKEN_EMBD: "token_embd",
|
||||
MODEL_TENSOR.TOKEN_EMBD_NORM: "token_embd_norm",
|
||||
MODEL_TENSOR.TOKEN_EMBD_PROJ: "token_embd_proj",
|
||||
MODEL_TENSOR.TOKEN_TYPES: "token_types",
|
||||
MODEL_TENSOR.MASKED_EMBD_CENTROIDS: "masked_embd_centroids",
|
||||
MODEL_TENSOR.MASKED_EMBD_ORDERING: "masked_embd_ordering",
|
||||
@@ -2323,6 +2342,20 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
|
||||
MODEL_TENSOR.A_CODEC_VQ_FIRST_CB: "a.codec.vq.first.codebook",
|
||||
MODEL_TENSOR.A_CODEC_VQ_REST_IN: "a.codec.vq.rest.in_proj",
|
||||
MODEL_TENSOR.A_CODEC_VQ_REST_CB: "a.codec.vq.rest.codebook",
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_NORM: "a.gen.enc.tfm.blk.{bid}.ln1",
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_Q: "a.gen.enc.tfm.blk.{bid}.attn_q",
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_K: "a.gen.enc.tfm.blk.{bid}.attn_k",
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_V: "a.gen.enc.tfm.blk.{bid}.attn_v",
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_OUT: "a.gen.enc.tfm.blk.{bid}.attn_out",
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_SCALE: "a.gen.enc.tfm.blk.{bid}.ls1",
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_FFN_NORM: "a.gen.enc.tfm.blk.{bid}.ln2",
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_FFN_UP: "a.gen.enc.tfm.blk.{bid}.ffn_up",
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_FFN_DOWN: "a.gen.enc.tfm.blk.{bid}.ffn_down",
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_FFN_SCALE: "a.gen.enc.tfm.blk.{bid}.ls2",
|
||||
MODEL_TENSOR.A_GEN_ENC_QUANT_FIRST_IN: "a.gen.enc.quant.first.in_proj",
|
||||
MODEL_TENSOR.A_GEN_ENC_QUANT_FIRST_CB: "a.gen.enc.quant.first.codebook",
|
||||
MODEL_TENSOR.A_GEN_ENC_QUANT_REST_IN: "a.gen.enc.quant.rest.in_proj",
|
||||
MODEL_TENSOR.A_GEN_ENC_QUANT_REST_CB: "a.gen.enc.quant.rest.codebook",
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN: "a.gen.wav.quant.first.in_proj",
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj",
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB: "a.gen.wav.quant.first.codebook",
|
||||
@@ -2778,6 +2811,20 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.A_CODEC_VQ_FIRST_CB,
|
||||
MODEL_TENSOR.A_CODEC_VQ_REST_IN,
|
||||
MODEL_TENSOR.A_CODEC_VQ_REST_CB,
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_NORM,
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_Q,
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_K,
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_V,
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_OUT,
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_ATTN_SCALE,
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_FFN_NORM,
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_FFN_UP,
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_FFN_DOWN,
|
||||
MODEL_TENSOR.A_GEN_ENC_TFM_FFN_SCALE,
|
||||
MODEL_TENSOR.A_GEN_ENC_QUANT_FIRST_IN,
|
||||
MODEL_TENSOR.A_GEN_ENC_QUANT_FIRST_CB,
|
||||
MODEL_TENSOR.A_GEN_ENC_QUANT_REST_IN,
|
||||
MODEL_TENSOR.A_GEN_ENC_QUANT_REST_CB,
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN,
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT,
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB,
|
||||
@@ -6819,6 +6866,20 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.FFN_DOWN,
|
||||
MODEL_TENSOR.FFN_UP,
|
||||
],
|
||||
MODEL_ARCH.KOKORO: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
MODEL_TENSOR.TOKEN_EMBD_NORM,
|
||||
MODEL_TENSOR.TOKEN_EMBD_PROJ,
|
||||
MODEL_TENSOR.POS_EMBD,
|
||||
MODEL_TENSOR.ATTN_Q,
|
||||
MODEL_TENSOR.ATTN_K,
|
||||
MODEL_TENSOR.ATTN_V,
|
||||
MODEL_TENSOR.ATTN_OUT,
|
||||
MODEL_TENSOR.ATTN_OUT_NORM,
|
||||
MODEL_TENSOR.FFN_DOWN,
|
||||
MODEL_TENSOR.FFN_UP,
|
||||
MODEL_TENSOR.LAYER_OUT_NORM,
|
||||
],
|
||||
MODEL_ARCH.ZONOS3: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
MODEL_TENSOR.OUTPUT_NORM,
|
||||
@@ -7294,6 +7355,7 @@ class VisionProjectorType:
|
||||
MINIMAX_MUSIC3_GEN = "minimax_music3_gen" # audio generation: RVQ depth decoder + flow-matching DiT + DAC decoder
|
||||
FISH_S2_GEN = "fish_s2_gen" # audio generation: fast AR code predictor + Fish DAC decoder
|
||||
FISH_S2_SPKENC = "fish_s2_spkenc" # audio: Fish DAC encoder + RVQ, reference clip -> prompt rows
|
||||
KOKORO = "kokoro" # audio generation: StyleTTS2 prosody predictor + iSTFTNet decoder
|
||||
HUNYUANVL = "hunyuanvl"
|
||||
PARAKEET = "parakeet" # audio
|
||||
MINIMAXM3 = "minimax_m3"
|
||||
|
||||
@@ -0,0 +1,585 @@
|
||||
{#- Kokoro "tts" template: english text -> phonemes, a port of the english path of misaki (the reference G2P)
|
||||
functions: tts_norm_scan(), tts_norm_sub(), tts_dict(), see docs/tts-norm-patterns.md
|
||||
dictionaries, written by conversion/kokoro.py:
|
||||
- gold / silver (US), gold_gb / silver_gb (british): the misaki lexicon, a value is a string or {DEFAULT, <tag>...}
|
||||
- pos_weights / pos_tagdict / pos_classes: the nltk averaged perceptron tagger
|
||||
- lts / lts_gb: model for the words missing from the lexicon, see lts_search()
|
||||
a voice of another language takes phonemes, its prompt passes through -#}
|
||||
{%- set unk = unk if unk is defined else '' -%}
|
||||
{#- british english: the voices named b*, or language en-gb; it has its own lexicon and a few suffix rules -#}
|
||||
{%- set lang = (language if language is defined and language else '') | lower | replace('_', '-') -%}
|
||||
{%- set GB = lang in ['en-gb', 'british'] or (not lang and voice is defined and voice and voice[:1] == 'b') -%}
|
||||
{%- set GOLD = 'gold_gb' if GB else 'gold' -%}
|
||||
{%- set SILVER = 'silver_gb' if GB else 'silver' -%}
|
||||
{%- set LTS = 'lts_gb' if GB else 'lts' -%}
|
||||
{%- set V = 'AIOQWYaiuæɑɒɔəɛɜɪʊʌᵻ' -%}
|
||||
{%- set C = 'bdfhjklmnpstvwzðŋɡɹɾʃʒʤʧθ' -%}
|
||||
{%- set NQP = ';:,.!?—…' -%}
|
||||
{%- set TAUS = 'AIOWYiuæɑəɛɪɹʊʌ' -%}
|
||||
{%- set SYMBOLS = {'%': 'percent', '&': 'and', '+': 'plus', '@': 'at'} -%}
|
||||
{#- the lexicon has only ASCII keys: an accented letter is read as its base letter -#}
|
||||
{%- set FOLD = [['[àáâãäåāă]', 'a'], ['[ÀÁÂÃÄÅĀĂ]', 'A'], ['[çćč]', 'c'], ['[ÇĆČ]', 'C'], ['[èéêëēėęě]', 'e'], ['[ÈÉÊËĒĖĘĚ]', 'E'],
|
||||
['[ìíîïīı]', 'i'], ['[ÌÍÎÏĪİ]', 'I'], ['[ñńň]', 'n'], ['[ÑŃŇ]', 'N'], ['[òóôõöøōő]', 'o'], ['[ÒÓÔÕÖØŌŐ]', 'O'],
|
||||
['[ùúûüūůű]', 'u'], ['[ÙÚÛÜŪŮŰ]', 'U'], ['[ýÿ]', 'y'], ['[ÝŸ]', 'Y'], ['[śšș]', 's'], ['[ŚŠȘ]', 'S'], ['[źżž]', 'z'], ['[ŹŻŽ]', 'Z'],
|
||||
['ł', 'l'], ['Ł', 'L'], ['ß', 'ss'], ['æ', 'ae'], ['Æ', 'AE'], ['œ', 'oe'], ['Œ', 'OE']] -%}
|
||||
{%- set CURRENCIES = {'$': ['dollar', 'cent'], '£': ['pound', 'pence'], '€': ['euro', 'cent']} -%}
|
||||
|
||||
{#- ---------- small helpers ---------- -#}
|
||||
{%- macro full(s, pat) -%}{%- set segs = tts_norm_scan(s, '^(?:' ~ pat ~ ')$') -%}{{ '1' if segs | length == 1 and segs[0].rule else '' }}{%- endmacro -%}
|
||||
{#- key of w in a lexicon, or ''; like misaki's grow_dictionary(), a lowercase entry also answers for Capitalized and vice versa -#}
|
||||
{%- macro dkey(name, w) -%}
|
||||
{%- if tts_dict(name, w) is not none -%}{{ w }}
|
||||
{%- elif w | length > 1 -%}
|
||||
{%- set l = w | lower -%}{%- set c = l | capitalize -%}
|
||||
{%- if w == c and w != l and tts_dict(name, l) is not none -%}{{ l }}
|
||||
{%- elif w == l and w != c and tts_dict(name, c) is not none -%}{{ c }}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
{%- macro has(w) -%}{{ '1' if dkey(GOLD, w) or dkey(SILVER, w) else '' }}{%- endmacro -%}
|
||||
{#- default pronunciation in gold, or '' -#}
|
||||
{%- macro gold(w) -%}
|
||||
{%- set k = dkey(GOLD, w) -%}
|
||||
{%- if k -%}{%- set e = tts_dict(GOLD, k) -%}{{ e.DEFAULT if e is mapping else e }}{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- ---------- words missing from the lexicon ---------- -#}
|
||||
{#- beam search over the letters with an n-gram model of units "letter:sounds"
|
||||
tts_dict(LTS, "<0 to 4 units before> <letter>") -> {sounds: probability}; "<s>" pads the start, the letter "$" is the end
|
||||
a unit takes its score from the longest context that has it, times 0.4 for each unit dropped
|
||||
fixed is none, or the sounds of each letter without stress: then only the stress is searched, with exactly one primary stress
|
||||
returns the sounds of each letter, joined by "|" -#}
|
||||
{%- macro lts_search(word, fixed) -%}
|
||||
{%- set bm = namespace(l=[{'p': 1.0, 'h': ['<s>', '<s>', '<s>', '<s>'], 'o': [], 's': false}]) -%}
|
||||
{%- for ch in (tts_norm_scan(word, '.') | map(attribute='text') | list) + ['$'] -%}
|
||||
{%- set i = loop.index0 -%}
|
||||
{%- set nx = namespace(l=[]) -%}
|
||||
{%- for hyp in bm.l -%}
|
||||
{%- set st = namespace(f=1.0, seen=[]) -%}
|
||||
{%- for n in range(4, -1, -1) -%}
|
||||
{%- set d = tts_dict(LTS, (hyp.h[4 - n:] + [ch]) | join(' ')) -%}
|
||||
{%- if d is not none -%}
|
||||
{%- for o, p in d | items -%}
|
||||
{%- if o not in st.seen -%}
|
||||
{%- set _ = st.seen.append(o) -%}
|
||||
{%- set prim = 'ˈ' in o -%}
|
||||
{%- set match = fixed is none or ch == '$' or (o | replace('ˈ', '') | replace('ˌ', '')) == fixed[i] -%}
|
||||
{%- if match and not (fixed is not none and prim and hyp.s) -%}
|
||||
{%- set _ = nx.l.append({'p': hyp.p * st.f * p, 'h': (hyp.h + [ch ~ ':' ~ o])[-4:], 'o': hyp.o + [o], 's': hyp.s or prim}) -%}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
{%- set st.f = st.f * 0.4 -%}
|
||||
{%- endfor -%}
|
||||
{%- endfor -%}
|
||||
{%- set srt = nx.l | sort(attribute='p', reverse=true) -%}
|
||||
{%- if fixed is none -%}{%- set bm.l = srt[:16] -%}
|
||||
{%- else -%}{%- set bm.l = (srt | selectattr('s') | list)[:8] + (srt | rejectattr('s') | list)[:8] -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- set done = (bm.l | selectattr('s') | list) or bm.l -%}
|
||||
{%- if done -%}{{ (done | sort(attribute='p', reverse=true))[0].o[:-1] | join('|') }}{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- pronunciation of a word that is not in the lexicon, '' if it cannot be read -#}
|
||||
{%- macro lts(word) -%}
|
||||
{%- set w = namespace(s=word | lower | replace('’', "'")) -%}
|
||||
{#- doubled consonants and "ck" say one sound -#}
|
||||
{%- for c in ['b', 'c', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'm', 'n', 'p', 'q', 'r', 's', 't', 'v', 'w', 'x', 'y', 'z'] -%}{%- set w.s = w.s | replace(c ~ c, c) -%}{%- endfor -%}
|
||||
{%- set w.s = w.s | replace('ck', 'k') -%}
|
||||
{%- if w.s | length >= 2 and full(w.s, "[a-z']+") -%}
|
||||
{#- first the sounds, then the stress marks on those sounds -#}
|
||||
{%- set first = lts_search(w.s, none) -%}
|
||||
{%- if first or w.s | length == 0 -%}
|
||||
{%- set sounds = first.split('|') | map('replace', 'ˈ', '') | map('replace', 'ˌ', '') | list -%}
|
||||
{%- if sounds | length == w.s | length -%}{{ lts_search(w.s, sounds) | replace('|', '') }}{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- a word missing from the lexicon: the predicted pronunciation; a word that cannot be read (e.g. "cpp") is spelled out -#}
|
||||
{%- macro fallback(word) -%}
|
||||
{%- set ps = lts(word) if tts_norm_scan(word, '[aeiouyAEIOUY]') | selectattr('rule') | list else '' -%}
|
||||
{%- if ps and tts_norm_scan(ps, '[' ~ V ~ ']') | selectattr('rule') | list -%}{{ ps }}
|
||||
{%- else -%}{{ get_nnp(word) or unk }}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro parent_tag(tag) -%}
|
||||
{%- if tag.startswith('VB') -%}VERB
|
||||
{%- elif tag.startswith('NN') -%}NOUN
|
||||
{%- elif tag.startswith('ADV') or tag.startswith('RB') -%}ADV
|
||||
{%- elif tag.startswith('ADJ') or tag.startswith('JJ') -%}ADJ
|
||||
{%- else -%}{{ tag }}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro apply_stress(ps, stress) -%}
|
||||
{%- if stress is none or not ps -%}{{ ps }}
|
||||
{%- elif stress < -1 -%}{{ ps | replace('ˈ', '') | replace('ˌ', '') }}
|
||||
{%- elif stress == -1 or ((stress == 0 or stress == -0.5) and 'ˈ' in ps) -%}{{ ps | replace('ˌ', '') | replace('ˈ', 'ˌ') }}
|
||||
{%- elif (stress == 0 or stress == 0.5 or stress == 1) and 'ˈ' not in ps and 'ˌ' not in ps -%}{{ tts_norm_sub(ps, '^([^' ~ V ~ ']*)([' ~ V ~ '])', '%1ˌ%2') }}
|
||||
{%- elif stress >= 1 and 'ˈ' not in ps and 'ˌ' in ps -%}{{ ps | replace('ˌ', 'ˈ') }}
|
||||
{%- elif stress > 1 and 'ˈ' not in ps and 'ˌ' not in ps -%}{{ tts_norm_sub(ps, '^([^' ~ V ~ ']*)([' ~ V ~ '])', '%1ˈ%2') }}
|
||||
{%- else -%}{{ ps }}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- spell out letters: "NASA" -> N A S A, "e.g." -> E G -#}
|
||||
{%- macro get_nnp(word) -%}
|
||||
{%- set ns = namespace(ps='', ok=true) -%}
|
||||
{%- for c in tts_norm_scan(word, '%a') -%}
|
||||
{%- if c.rule -%}
|
||||
{%- set p = gold(c.text | upper) -%}
|
||||
{%- if not p -%}{%- set ns.ok = false -%}{%- else -%}{%- set ns.ps = ns.ps ~ p -%}{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- if ns.ok and ns.ps -%}{{ tts_norm_sub(apply_stress(ns.ps, 0), 'ˌ([^ˌ]*)$', 'ˈ%1') }}{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- tag: Penn tag or '' ; fv = future vowel context: 'none' (end / punctuation follows), 'v', 'c', or '' (no context) -#}
|
||||
{%- macro lookup(word, tag, stress, fv) -%}
|
||||
{%- set ns = namespace(w=word, nnp=false, ps=none, t='') -%}
|
||||
{%- if word == word | upper and not dkey(GOLD, word) -%}{%- set ns.w = word | lower -%}{%- set ns.nnp = tag == 'NNP' -%}{%- endif -%}
|
||||
{%- set k = dkey(GOLD, ns.w) -%}
|
||||
{%- if k -%}
|
||||
{%- set e = tts_dict(GOLD, k) -%}
|
||||
{%- if e is mapping -%}
|
||||
{#- POS variants: misaki's ps.get(tag) with 'None' (no following vowel context) and parent tag fallback -#}
|
||||
{%- if fv == 'none' and 'None' in e -%}{%- set ns.t = 'None' -%}
|
||||
{%- elif tag and tag in e -%}{%- set ns.t = tag -%}
|
||||
{%- else -%}{%- set ns.t = parent_tag(tag) -%}
|
||||
{%- endif -%}
|
||||
{#- a null variant means "no pronunciation for this tag": spell the word out -#}
|
||||
{%- set ns.ps = (e[ns.t] or '') if ns.t and ns.t in e else e.DEFAULT -%}
|
||||
{%- else -%}{%- set ns.ps = e -%}
|
||||
{%- endif -%}
|
||||
{%- elif not ns.nnp -%}
|
||||
{%- set k = dkey(SILVER, ns.w) -%}
|
||||
{%- set ns.ps = tts_dict(SILVER, k) if k else none -%}
|
||||
{%- endif -%}
|
||||
{%- set r = get_nnp(ns.w) if (not ns.ps or (ns.nnp and 'ˈ' not in ns.ps)) else '' -%}
|
||||
{%- if r -%}{{ r }}{%- else -%}{{ apply_stress(ns.ps or '', stress) }}{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro is_known(word) -%}
|
||||
{%- if has(word) or word in SYMBOLS -%}1
|
||||
{%- elif not full(word, '[A-Za-z]+') -%}
|
||||
{%- elif word | length == 1 -%}1
|
||||
{%- elif word == word | upper and dkey(GOLD, word | lower) -%}1
|
||||
{%- elif word[1:] == word[1:] | upper -%}1
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- ---------- suffixes ---------- -#}
|
||||
{%- macro sfx_s(stem) -%}
|
||||
{%- if not stem -%}
|
||||
{%- elif stem[-1:] in 'ptkfθ' -%}{{ stem }}s
|
||||
{%- elif stem[-1:] in 'szʃʒʧʤ' -%}{{ stem }}{{ 'ɪ' if GB else 'ᵻ' }}z
|
||||
{%- else -%}{{ stem }}z
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro stem_s(word, tag, stress, fv) -%}
|
||||
{%- set n = word | length -%}
|
||||
{%- if n < 3 or not word.endswith('s') -%}
|
||||
{%- elif not word.endswith('ss') and is_known(word[:-1]) -%}{{ sfx_s(lookup(word[:-1], tag, stress, fv)) }}
|
||||
{%- elif (word.endswith("'s") or (n > 4 and word.endswith('es') and not word.endswith('ies'))) and is_known(word[:-2]) -%}{{ sfx_s(lookup(word[:-2], tag, stress, fv)) }}
|
||||
{%- elif n > 4 and word.endswith('ies') and is_known(word[:-3] ~ 'y') -%}{{ sfx_s(lookup(word[:-3] ~ 'y', tag, stress, fv)) }}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro sfx_ed(stem) -%}
|
||||
{%- if not stem -%}
|
||||
{%- elif stem[-1:] in 'pkfθʃsʧ' -%}{{ stem }}t
|
||||
{%- elif stem[-1:] == 'd' -%}{{ stem }}{{ 'ɪ' if GB else 'ᵻ' }}d
|
||||
{%- elif stem[-1:] != 't' -%}{{ stem }}d
|
||||
{%- elif GB or stem | length < 2 -%}{{ stem }}ɪd
|
||||
{%- elif stem[-2:-1] in TAUS -%}{{ stem[:-1] }}ɾᵻd
|
||||
{%- else -%}{{ stem }}ᵻd
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro stem_ed(word, tag, stress, fv) -%}
|
||||
{%- set n = word | length -%}
|
||||
{%- if n < 4 or not word.endswith('d') -%}
|
||||
{%- elif not word.endswith('dd') and is_known(word[:-1]) -%}{{ sfx_ed(lookup(word[:-1], tag, stress, fv)) }}
|
||||
{%- elif n > 4 and word.endswith('ed') and not word.endswith('eed') and is_known(word[:-2]) -%}{{ sfx_ed(lookup(word[:-2], tag, stress, fv)) }}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro sfx_ing(stem) -%}
|
||||
{%- if not stem -%}
|
||||
{%- elif GB -%}{%- if stem[-1:] not in 'əː' -%}{{ stem }}ɪŋ{%- endif -%}
|
||||
{%- elif stem | length > 1 and stem[-1:] == 't' and stem[-2:-1] in TAUS -%}{{ stem[:-1] }}ɾɪŋ
|
||||
{%- else -%}{{ stem }}ɪŋ
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro stem_ing(word, tag, stress, fv) -%}
|
||||
{%- set n = word | length -%}
|
||||
{%- if n < 5 or not word.endswith('ing') -%}
|
||||
{%- elif n > 5 and is_known(word[:-3]) -%}{{ sfx_ing(lookup(word[:-3], tag, stress, fv)) }}
|
||||
{%- elif is_known(word[:-3] ~ 'e') -%}{{ sfx_ing(lookup(word[:-3] ~ 'e', tag, stress, fv)) }}
|
||||
{%- elif n > 5 and tts_norm_scan(word, '(?:bb|cc|dd|gg|kk|ll|mm|nn|pp|rr|ss|tt|vv|xx|zz|ck)ing$') | selectattr('rule') | list and is_known(word[:-4]) -%}{{ sfx_ing(lookup(word[:-4], tag, stress, fv)) }}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- ---------- words ---------- -#}
|
||||
{#- misaki Lexicon.get_special_case(); fto = the next word is "to" (for "used to") -#}
|
||||
{%- macro special(word, tag, stress, fv, fto) -%}
|
||||
{%- if word in SYMBOLS -%}{{ lookup(SYMBOLS[word], '', none, fv) }}
|
||||
{%- elif word == 'a' or word == 'A' -%}{{ 'ɐ' if tag == 'DT' else 'ˈA' }}
|
||||
{%- elif word in ['am', 'Am', 'AM'] -%}
|
||||
{%- if tag.startswith('NN') -%}{{ get_nnp(word) }}
|
||||
{%- elif fv == 'none' or word != 'am' or (stress is not none and stress > 0) -%}{{ gold('am') }}
|
||||
{%- else -%}ɐm{%- endif -%}
|
||||
{%- elif word in ['an', 'An', 'AN'] -%}{{ get_nnp(word) if (word == 'AN' and tag.startswith('NN')) else 'ɐn' }}
|
||||
{%- elif word == 'I' and tag == 'PRP' -%}ˌI
|
||||
{%- elif word in ['by', 'By', 'BY'] and parent_tag(tag) == 'ADV' -%}bˈI
|
||||
{%- elif word in ['to', 'To'] or (word == 'TO' and tag in ['TO', 'IN']) -%}{{ {'v': 'tʊ', 'c': 'tə'}.get(fv, gold('to')) }}
|
||||
{%- elif word in ['in', 'In'] or (word == 'IN' and tag != 'NNP') -%}{{ 'ˈ' if (fv == 'none' or tag != 'IN') else '' }}ɪn
|
||||
{%- elif word in ['the', 'The'] or (word == 'THE' and tag == 'DT') -%}{{ 'ði' if fv == 'v' else 'ðə' }}
|
||||
{%- elif tag == 'IN' and full(word, '[Vv][Ss]%.?') -%}{{ lookup('versus', '', none, fv) }}
|
||||
{%- elif word in ['used', 'Used', 'USED'] -%}{{ tts_dict(GOLD, 'used').VBD if (tag in ['VBD', 'JJ'] and fto) else gold('used') }}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro get_word(word0, tag, stress, fv, fto) -%}
|
||||
{%- set sp = special(word0, tag, stress, fv, fto) -%}
|
||||
{%- if sp -%}{{ sp }}{%- else -%}
|
||||
{%- set ns = namespace(word=word0) -%}
|
||||
{%- set wl = word0 | lower -%}
|
||||
{%- if word0 | length > 1 and full(word0 | replace("'", ''), '%a+') and word0 != wl and (tag != 'NNP' or word0 | length > 7) and not has(word0)
|
||||
and (word0 == word0 | upper or word0[1:] == word0[1:] | lower)
|
||||
and (has(wl) or stem_s(wl, tag, stress, fv) or stem_ed(wl, tag, stress, fv) or stem_ing(wl, tag, stress, fv)) -%}
|
||||
{%- set ns.word = wl -%}
|
||||
{%- endif -%}
|
||||
{%- set w = ns.word -%}
|
||||
{%- if is_known(w) -%}{{ lookup(w, tag, stress, fv) }}
|
||||
{%- elif w.endswith("s'") and is_known(w[:-2] ~ "'s") -%}{{ lookup(w[:-2] ~ "'s", tag, stress, fv) }}
|
||||
{%- elif w.endswith("'") and is_known(w[:-1]) -%}{{ lookup(w[:-1], tag, stress, fv) }}
|
||||
{%- else -%}
|
||||
{%- set r = stem_s(w, tag, stress, fv) -%}
|
||||
{%- if not r -%}{%- set r = stem_ed(w, tag, stress, fv) -%}{%- endif -%}
|
||||
{%- if not r -%}{%- set r = stem_ing(w, tag, 0.5 if stress is none else stress, fv) -%}{%- endif -%}
|
||||
{{- r -}}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro lex_word(word0, tag, fv, fto=false) -%}
|
||||
{%- set w = word0 | replace('‘', "'") | replace('’', "'") -%}
|
||||
{%- set stress = none if w == w | lower else (2 if w == w | upper else 0.5) -%}
|
||||
{{- get_word(w, tag, stress, fv, fto) -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- ---------- numbers (num2words-compatible word lists, "and" dropped like misaki) ---------- -#}
|
||||
{%- set ONES = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'eleven', 'twelve',
|
||||
'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'] -%}
|
||||
{%- set TENS = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'] -%}
|
||||
{%- set SCALES = [[1000000000000, 'trillion'], [1000000000, 'billion'], [1000000, 'million'], [1000, 'thousand']] -%}
|
||||
{%- macro card(n) -%}
|
||||
{%- if n < 20 -%}{{ ONES[n] }}
|
||||
{%- elif n < 100 -%}{{ TENS[n // 10] }}{{ ' ' ~ ONES[n % 10] if n % 10 else '' }}
|
||||
{%- elif n < 1000 -%}{{ ONES[n // 100] }} hundred{{ ' ' ~ card(n % 100) if n % 100 else '' }}
|
||||
{%- else -%}
|
||||
{%- set ns = namespace(done=false) -%}
|
||||
{%- for sc in SCALES -%}
|
||||
{%- if not ns.done and n >= sc[0] -%}
|
||||
{{- card(n // sc[0]) }} {{ sc[1] }}{{ ' ' ~ card(n % sc[0]) if n % sc[0] else '' -}}
|
||||
{%- set ns.done = true -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro ordw(n) -%}
|
||||
{%- set ws = card(n).split(' ') -%}
|
||||
{%- set last = ws[-1] -%}
|
||||
{%- set IRR = {'one': 'first', 'two': 'second', 'three': 'third', 'five': 'fifth', 'eight': 'eighth', 'nine': 'ninth', 'twelve': 'twelfth'} -%}
|
||||
{%- if last in IRR -%}{%- set last = IRR[last] -%}{%- elif last.endswith('y') -%}{%- set last = last[:-1] ~ 'ieth' -%}{%- else -%}{%- set last = last ~ 'th' -%}{%- endif -%}
|
||||
{{- (ws[:-1] + [last]) | join(' ') -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro yearw(n) -%}
|
||||
{%- set hi = n // 100 -%}{%- set lo = n % 100 -%}
|
||||
{%- if hi == 0 or (hi % 10 == 0 and lo < 10) or hi >= 100 -%}{{ card(n) }}
|
||||
{%- else -%}{{ card(hi) }} {{ 'hundred' if lo == 0 else ('oh ' ~ card(lo) if lo < 10 else card(lo)) }}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro say(words) -%}
|
||||
{%- set ns = namespace(out=[]) -%}
|
||||
{%- for w in words.split(' ') -%}
|
||||
{%- if w -%}{%- set ns.out = ns.out + [lookup(w, '', -2 if w == 'point' else none, '')] -%}{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- ns.out | join(' ') -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro digits(s) -%}{%- set ns = namespace(out=[]) -%}{%- for c in tts_norm_scan(s, '%d') -%}{%- if c.rule -%}{%- set ns.out = ns.out + [ONES[c.text | int]] -%}{%- endif -%}{%- endfor -%}{{ ns.out | join(' ') }}{%- endmacro -%}
|
||||
|
||||
{#- num: digits with optional ,### groups and .decimals ; sfx: st/nd/rd/th/s/'s/ed/ing ; cur: currency symbol or '' -#}
|
||||
{%- macro get_number(neg, num, sfx, cur) -%}
|
||||
{%- set plain = num | replace(',', '') -%}
|
||||
{%- set ns = namespace(w='') -%}
|
||||
{%- if sfx in ['st', 'nd', 'rd', 'th'] and '.' not in plain -%}{%- set ns.w = ordw(plain | int) -%}
|
||||
{%- elif not neg and not cur and full(num, '%d%d%d%d') -%}{%- set ns.w = yearw(plain | int) -%}
|
||||
{%- elif cur and cur in CURRENCIES and full(plain, '%d*(?:%.%d%d?)?') -%}
|
||||
{%- set parts = plain.split('.') -%}
|
||||
{%- set units = CURRENCIES[cur] -%}
|
||||
{%- set a = parts[0] | int if parts[0] else 0 -%}
|
||||
{%- set b = parts[1] | int if parts | length > 1 and parts[1] else 0 -%}
|
||||
{%- set out = namespace(l=[]) -%}
|
||||
{%- for pair in [[a, units[0]], [b, units[1]]] -%}
|
||||
{%- if pair[0] or (loop.first and not b) -%}
|
||||
{%- set uw = lookup(pair[1], '', none, '') if pair[0] == 1 or pair[1] == 'pence' else stem_s(pair[1] ~ 's', '', none, '') -%}
|
||||
{%- set out.l = out.l + [say(card(pair[0])) ~ ' ' ~ uw] -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- set ns.w = '@' ~ (out.l | join(' ' ~ lookup('and', '', none, '') ~ ' ')) -%}
|
||||
{%- elif '.' in plain -%}
|
||||
{%- set ip, fp = plain.split('.') -%}
|
||||
{%- set ns.w = ((card(ip | int) ~ ' ') if ip else '') ~ 'point ' ~ digits(fp) -%}
|
||||
{%- else -%}{%- set ns.w = card(plain | int) -%}
|
||||
{%- endif -%}
|
||||
{%- set r = ns.w[1:] if ns.w.startswith('@') else say(ns.w | replace('-', ' ')) -%}
|
||||
{%- if neg -%}{%- set r = lookup('minus', '', none, '') ~ ' ' ~ r -%}{%- endif -%}
|
||||
{%- if sfx in ['s', "'s"] -%}{{ sfx_s(r) }}
|
||||
{%- elif sfx in ['ed', "'d"] -%}{{ sfx_ed(r) }}
|
||||
{%- elif sfx == 'ing' -%}{{ sfx_ing(r) }}
|
||||
{%- else -%}{{ r }}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- ---------- context ---------- -#}
|
||||
{%- macro next_fv(ps, fv) -%}
|
||||
{%- set ns = namespace(r=fv, done=false) -%}
|
||||
{%- for s in tts_norm_scan(ps, [['v', '[' ~ V ~ ']'], ['c', '[' ~ C ~ ']'], ['p', '[' ~ NQP ~ ']']]) -%}
|
||||
{%- if s.rule and not ns.done -%}{%- set ns.r = 'none' if s.rule == 'p' else s.rule -%}{%- set ns.done = true -%}{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- ns.r -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- ---------- POS tagger: nltk averaged perceptron, weights in the pos_* dictionaries ---------- -#}
|
||||
{%- macro pos_norm(w) -%}
|
||||
{%- if '-' in w and w[:1] != '-' -%}!HYPHEN
|
||||
{%- elif full(w, '%d%d%d%d') -%}!YEAR
|
||||
{%- elif full(w[:1], '%d') -%}!DIGITS
|
||||
{%- else -%}{{ w | lower }}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- set POS_CLASSES = tts_dict('pos_classes', '') -%}
|
||||
|
||||
{#- features -> best label; nltk: max(classes, key=(score, label)); classes are sorted, so >= keeps the larger label on ties -#}
|
||||
{%- macro pos_predict(feats) -%}
|
||||
{%- set ns = namespace(ws=[], best='', best_s=none) -%}
|
||||
{%- for f in feats -%}
|
||||
{%- set w = tts_dict('pos_weights', f) -%}
|
||||
{%- if w -%}{%- set ns.ws = ns.ws + [w] -%}{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- for c in POS_CLASSES -%}
|
||||
{%- set s = ns.ws | map(attribute='c' ~ loop.index0, default=0.0) | sum -%}
|
||||
{%- if ns.best_s is none or s >= ns.best_s -%}{%- set ns.best = c -%}{%- set ns.best_s = s -%}{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- ns.best -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- tokens: list of strings (PTB-style) -> space-separated tags -#}
|
||||
{%- macro pos_tag(tokens) -%}
|
||||
{%- set cn = namespace(l=['-START-', '-START2-']) -%}
|
||||
{%- for t in tokens -%}{%- set cn.l = cn.l + [pos_norm(t)] -%}{%- endfor -%}
|
||||
{%- set c = cn.l + ['-END-', '-END2-'] -%}
|
||||
{%- set ns = namespace(prev='-START-', prev2='-START2-', out=[]) -%}
|
||||
{%- for w in tokens -%}
|
||||
{%- set i = loop.index0 + 2 -%}
|
||||
{%- set tag = tts_dict('pos_tagdict', w) -%}
|
||||
{%- if not tag -%}
|
||||
{%- set tag = pos_predict([
|
||||
'bias',
|
||||
'i suffix ' ~ w[-3:],
|
||||
'i pref1 ' ~ w[:1],
|
||||
'i-1 tag ' ~ ns.prev,
|
||||
'i-2 tag ' ~ ns.prev2,
|
||||
'i tag+i-2 tag ' ~ ns.prev ~ ' ' ~ ns.prev2,
|
||||
'i word ' ~ c[i],
|
||||
'i-1 tag+i word ' ~ ns.prev ~ ' ' ~ c[i],
|
||||
'i-1 word ' ~ c[i - 1],
|
||||
'i-1 suffix ' ~ c[i - 1][-3:],
|
||||
'i-2 word ' ~ c[i - 2],
|
||||
'i+1 word ' ~ c[i + 1],
|
||||
'i+1 suffix ' ~ c[i + 1][-3:],
|
||||
'i+2 word ' ~ c[i + 2],
|
||||
]) -%}
|
||||
{%- endif -%}
|
||||
{%- set ns.out = ns.out + [tag] -%}
|
||||
{%- set ns.prev2 = ns.prev -%}{%- set ns.prev = tag -%}
|
||||
{%- endfor -%}
|
||||
{{- ns.out | join(' ') -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro weight(ps) -%}{{ (ps | length) + (tts_norm_scan(ps, '[AIOQWYʤʧ]') | selectattr('rule') | list | length) }}{%- endmacro -%}
|
||||
|
||||
{#- hyphenated compound: per-part lookup (right to left), joined without space, misaki's stress demotion -#}
|
||||
{%- macro compound(text, tag, fv) -%}
|
||||
{%- if has(text) -%}{{ lex_word(text, tag, fv) }}{%- else -%}
|
||||
{%- set ns = namespace(fv=fv, parts=[], bad=false) -%}
|
||||
{%- for s in tts_norm_scan(text, '%a+(?:[\'’]%a+)*') | reverse -%}
|
||||
{%- if s.rule -%}
|
||||
{%- set p = lex_word(s.text, tag, ns.fv) -%}
|
||||
{%- if not p -%}{%- set ns.bad = true -%}{%- endif -%}
|
||||
{%- set ns.parts = [{'t': s.text, 'p': p}] + ns.parts -%}
|
||||
{%- set ns.fv = next_fv(p, ns.fv) -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- set n = ns.parts | length -%}
|
||||
{%- set d = namespace(demote=[]) -%}
|
||||
{#- a part is missing from the lexicon: read the compound part by part -#}
|
||||
{%- if ns.bad -%}
|
||||
{%- for x in ns.parts -%}{{ x.p or fallback(x.t) }}{%- endfor -%}
|
||||
{%- else -%}
|
||||
{%- if not ns.bad -%}
|
||||
{%- set k = namespace(keys=[]) -%}
|
||||
{%- for x in ns.parts -%}
|
||||
{%- set pr = 1 if 'ˈ' in x.p else 0 -%}
|
||||
{#- misaki: sorted((has_primary, weight, index)) -#}
|
||||
{%- set k.keys = k.keys + [{'pr': pr, 'key': pr * 1000000 + (weight(x.p) | int) * 100 + loop.index0, 'i': loop.index0}] -%}
|
||||
{%- endfor -%}
|
||||
{%- set prim = k.keys | sum(attribute='pr') -%}
|
||||
{%- if n == 2 and ns.parts[0].t | length == 1 -%}{%- set d.demote = [1] -%}
|
||||
{%- elif n >= 2 and prim > (n + 1) // 2 -%}{%- set d.demote = (k.keys | sort(attribute='key'))[:n // 2] | map(attribute='i') | list -%}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- for x in ns.parts -%}{{ apply_stress(x.p, -0.5) if loop.index0 in d.demote else x.p }}{%- endfor -%}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- unknown word in mixed case, read part by part: "OneAPI" -> One + API, '' if the word has one part -#}
|
||||
{%- macro camel(text, tag, fv) -%}
|
||||
{%- set parts = tts_norm_scan(text, '(?:%u(?!%l))+|%u?%l+') | selectattr('rule') | map(attribute='text') | list -%}
|
||||
{%- if parts | length > 1 and parts | join('') == text -%}
|
||||
{%- for p in parts -%}
|
||||
{{- get_nnp(p) if p | length == 1 else (lex_word(p, tag, fv) or fallback(p)) -}}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- nxt: next token text ('' at end) -#}
|
||||
{%- macro word_tok(text, tag, fv, fto, nxt) -%}
|
||||
{%- set w = text | replace('‘', "'") | replace('’', "'") -%}
|
||||
{%- set ps = lex_word(w, tag, fv, fto) -%}
|
||||
{%- if ps -%}{{ ps }}
|
||||
{#- "etc" + "." -> dictionary entry "etc." (the dot is kept as punctuation) -#}
|
||||
{%- elif nxt[:1] == '.' and has(w ~ '.') -%}{{ lookup(w ~ '.', tag, none, fv) }}
|
||||
{#- trailing closing quote, e.g. after a plural possessive -#}
|
||||
{%- elif w.endswith("'") and lex_word(w[:-1], tag, fv) -%}{{ lex_word(w[:-1], tag, fv) }}”
|
||||
{#- unknown name + 's -#}
|
||||
{%- elif w.endswith("'s") -%}{{ (lex_word(w[:-2], tag, fv) or fallback(w[:-2])) ~ gold("'s") }}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- ---------- main ---------- -#}
|
||||
{#- only english has a text to phonemes step: the prompt of the other voices is already phonemes -#}
|
||||
{%- set lc = lang[:2] if lang else ('en' if not (voice is defined and voice) or voice[:1] in 'ab' else 'other') -%}
|
||||
{%- if lc != 'en' -%}
|
||||
{{- messages[-1].content | trim -}}
|
||||
{%- else -%}
|
||||
{%- set text = messages[-1].content | trim -%}
|
||||
{%- set t = namespace(s=tts_norm_sub(text, '%s+', ' ')) -%}
|
||||
{%- for f in FOLD -%}{%- set t.s = tts_norm_sub(t.s, f[0], f[1]) -%}{%- endfor -%}
|
||||
{%- set text = t.s -%}
|
||||
{%- set segs = tts_norm_scan(text, [
|
||||
['abbr', '(?<!%w)%a%a?(?:%.%a%a?)+%.?(?!%w)'],
|
||||
['money', '([$£€]) ?(%d+(?:,%d%d%d)*(?:%.%d+)?)(?!%d)'],
|
||||
['num', '((?<!%w)%-)?(%d+(?:,%d%d%d)*(?:%.%d+)?|%.%d+)(st|nd|rd|th|\'s|s|ed|ing)?(?!%w)'],
|
||||
['dig', '%d+(?:,%d%d%d)*(?:%.%d+)?'],
|
||||
['cmpd', '%a+(?:[\'’]%a+)*(?:[%-_]%a+(?:[\'’]%a+)*)+'],
|
||||
['wdot', '%a+%.(?= [%u%d]|$)'],
|
||||
['word', '%a+(?:[\'’‘]%a+)*[\'’‘]?'],
|
||||
['dash', '(?<= )%-(?= )|–|%-%-'],
|
||||
['qopen', '(?<!%S)"'],
|
||||
['quote', '"'],
|
||||
['punct', '[;:,.!?—…“”()]'],
|
||||
['sym', '[%%&+@]'],
|
||||
]) -%}
|
||||
{%- set nseg = segs | length -%}
|
||||
|
||||
{#- 1. PTB-style tokens for the tagger; first[i] = index of the first token of segment i (-1 if none) -#}
|
||||
{%- set PTB_PUNCT = {'“': '``', '”': "''", '—': '--'} -%}
|
||||
{%- set tk = namespace(toks=[], first=[]) -%}
|
||||
{%- for s in segs -%}
|
||||
{%- set n0 = tk.toks | length -%}
|
||||
{%- if s.rule is none -%}
|
||||
{%- elif s.rule == 'word' -%}
|
||||
{#- split clitics like PTB: do|n't, I|'m, dog|'s -#}
|
||||
{%- set w = s.text | replace('’', "'") | replace('‘', "'") -%}
|
||||
{%- set m = tts_norm_scan(w, '^(%a+)(\'(?:s|m|re|ll|ve|d))$') -%}
|
||||
{%- if (w | lower).endswith("n't") and w | length > 3 -%}{%- set tk.toks = tk.toks + [w[:-3], w[-3:]] -%}
|
||||
{%- elif m | length == 1 and m[0].rule -%}{%- set tk.toks = tk.toks + [m[0].groups[0], m[0].groups[1]] -%}
|
||||
{%- else -%}{%- set tk.toks = tk.toks + [w] -%}{%- endif -%}
|
||||
{%- elif s.rule == 'wdot' -%}
|
||||
{%- set tk.toks = tk.toks + ([s.text] if has(s.text) else [s.text[:-1], '.']) -%}
|
||||
{%- elif s.rule == 'money' -%}{%- set tk.toks = tk.toks + [s.groups[0], s.groups[1]] -%}
|
||||
{%- elif s.rule == 'qopen' -%}{%- set tk.toks = tk.toks + ['``'] -%}
|
||||
{%- elif s.rule == 'quote' -%}{%- set tk.toks = tk.toks + ["''"] -%}
|
||||
{%- elif s.rule == 'dash' -%}{%- set tk.toks = tk.toks + ['--'] -%}
|
||||
{%- elif s.rule == 'punct' -%}{%- set tk.toks = tk.toks + [PTB_PUNCT.get(s.text, s.text)] -%}
|
||||
{%- else -%}{%- set tk.toks = tk.toks + [s.text] -%}
|
||||
{%- endif -%}
|
||||
{%- set tk.first = tk.first + [n0 if (tk.toks | length) > n0 else -1] -%}
|
||||
{%- endfor -%}
|
||||
{%- set tags = pos_tag(tk.toks).split(' ') if tk.toks else [] -%}
|
||||
|
||||
{#- 2. G2P right to left (misaki's future-vowel context) -#}
|
||||
{%- set GLUE = ['abbr', 'money', 'num', 'dig', 'cmpd', 'wdot', 'word', 'sym'] -%}
|
||||
{%- set ns = namespace(fv='none', fto=false, out=[], next='', next_text='') -%}
|
||||
{%- for s in segs | reverse -%}
|
||||
{%- set i = nseg - 1 - loop.index0 -%}
|
||||
{%- set tag = tags[tk.first[i]] if tk.first[i] >= 0 else '' -%}
|
||||
{%- if s.rule == 'wdot' and has(s.text) -%}
|
||||
{%- set ps = lex_word(s.text, tag, ns.fv) -%}
|
||||
{%- elif s.rule == 'wdot' -%}
|
||||
{%- set w = lex_word(s.text[:-1], tag, 'none') or fallback(s.text[:-1]) -%}
|
||||
{%- set ps = w ~ '.' -%}
|
||||
{%- elif s.rule is none -%}
|
||||
{%- set ps = tts_norm_sub(s.text, '%S', '') -%}
|
||||
{%- elif s.rule == 'abbr' -%}
|
||||
{%- set ps = get_nnp(s.text) -%}
|
||||
{%- elif s.rule == 'money' -%}
|
||||
{%- set ps = get_number('', s.groups[1], '', s.groups[0]) -%}
|
||||
{%- elif s.rule == 'num' -%}
|
||||
{%- set ps = get_number(s.groups[0], s.groups[1], s.groups[2], '') -%}
|
||||
{%- elif s.rule == 'dig' -%}
|
||||
{#- digits glued to letters, e.g. "221B" -#}
|
||||
{%- set ps = get_number('', s.text, '', '') -%}
|
||||
{%- elif s.rule == 'cmpd' -%}
|
||||
{%- set ps = compound(s.text, tag, ns.fv) -%}
|
||||
{%- elif s.rule == 'word' -%}
|
||||
{%- set ps = word_tok(s.text, tag, ns.fv, ns.fto, ns.next_text) or camel(s.text, tag, ns.fv) or fallback(s.text) -%}
|
||||
{%- elif s.rule == 'sym' -%}
|
||||
{%- set ps = lex_word(s.text, tag, ns.fv) or unk -%}
|
||||
{%- elif s.rule == 'dash' -%}
|
||||
{%- set ps = '—' -%}
|
||||
{%- elif s.rule == 'qopen' -%}
|
||||
{%- set ps = '“' -%}
|
||||
{%- elif s.rule == 'quote' -%}
|
||||
{%- set ps = '”' -%}
|
||||
{%- else -%}
|
||||
{%- set ps = s.text -%}
|
||||
{%- endif -%}
|
||||
{%- if s.rule is not none -%}
|
||||
{%- set ns.fv = next_fv(ps, ns.fv) -%}
|
||||
{%- set ns.fto = s.text in ['to', 'To'] or (s.text == 'TO' and tag in ['TO', 'IN']) -%}
|
||||
{%- endif -%}
|
||||
{#- misaki puts a space between glued tokens of mixed kinds, e.g. "20%" -#}
|
||||
{%- if s.rule in GLUE and ns.next in GLUE and ps -%}{%- set ps = ps ~ ' ' -%}{%- endif -%}
|
||||
{%- set ns.next = s.rule if s.rule is not none else '' -%}
|
||||
{%- if s.rule is not none -%}{%- set ns.next_text = s.text -%}{%- endif -%}
|
||||
{%- set ns.out = [ps] + ns.out -%}
|
||||
{%- endfor -%}
|
||||
|
||||
{{- (ns.out | join('') | replace('ɾ', 'T') | replace('ʔ', 't')) | trim -}}
|
||||
{%- endif -%}
|
||||
@@ -176,6 +176,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
|
||||
{ LLM_ARCH_QWEN3TTS, "qwen3tts" },
|
||||
{ LLM_ARCH_POCKETTTS, "pockettts" },
|
||||
{ LLM_ARCH_ZONOS3, "zonos3" },
|
||||
{ LLM_ARCH_KOKORO, "kokoro" },
|
||||
{ LLM_ARCH_K2_HORIZON, "k2-horizon" },
|
||||
{ LLM_ARCH_BEAM, "beam" },
|
||||
{ LLM_ARCH_KOLIBRI1, "kolibri1" },
|
||||
@@ -629,6 +630,7 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
|
||||
{ LLM_TENSOR_POS_EMBD, "position_embd" },
|
||||
{ LLM_TENSOR_FFN_ACT, "blk.%d.ffn.act" },
|
||||
{ LLM_TENSOR_TOKEN_EMBD_NORM, "token_embd_norm" },
|
||||
{ LLM_TENSOR_TOKEN_EMBD_PROJ, "token_embd_proj" },
|
||||
{ LLM_TENSOR_TOKEN_TYPES, "token_types" },
|
||||
{ LLM_TENSOR_CLS, "cls" },
|
||||
{ LLM_TENSOR_CLS_OUT, "cls.output" },
|
||||
@@ -929,6 +931,7 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
|
||||
{LLM_TENSOR_TOKEN_TYPES, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},
|
||||
{LLM_TENSOR_HRM_Z_L_INIT, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ADD}},
|
||||
{LLM_TENSOR_TOKEN_EMBD_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, // do the norms on the first layer (not the input layer)
|
||||
{LLM_TENSOR_TOKEN_EMBD_PROJ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_OUTPUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_CLS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
{LLM_TENSOR_CLS_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
|
||||
|
||||
@@ -177,6 +177,7 @@ enum llm_arch {
|
||||
LLM_ARCH_QWEN3TTS,
|
||||
LLM_ARCH_POCKETTTS,
|
||||
LLM_ARCH_ZONOS3,
|
||||
LLM_ARCH_KOKORO,
|
||||
LLM_ARCH_MINIMAX_01,
|
||||
LLM_ARCH_HRM_TEXT,
|
||||
LLM_ARCH_K2_HORIZON,
|
||||
@@ -531,6 +532,7 @@ enum llm_kv {
|
||||
enum llm_tensor {
|
||||
LLM_TENSOR_TOKEN_EMBD,
|
||||
LLM_TENSOR_TOKEN_EMBD_NORM,
|
||||
LLM_TENSOR_TOKEN_EMBD_PROJ,
|
||||
LLM_TENSOR_TOKEN_TYPES,
|
||||
LLM_TENSOR_POS_EMBD,
|
||||
LLM_TENSOR_DENSE_2_OUT,
|
||||
|
||||
@@ -412,16 +412,28 @@ const llama_memory_hybrid_idx::kpool_layout & llama_memory_hybrid_idx::kpool_lay
|
||||
|
||||
sq.strm = unified ? 0 : mem_idx->get_stream(s);
|
||||
|
||||
if (mem_idx_stale[s] == POS_CLEAN && !sq.cells.empty() && !sp.empty() &&
|
||||
sq.pos_min == sp.begin()->first) {
|
||||
for (auto it = sp.upper_bound(sq.cells.back()); it != sp.end(); ++it) {
|
||||
// an append (POS_CLEAN) or an edit after pos_min keeps every cell and pool before the stale position
|
||||
// speculative decoding edits the end of the sequence on each rejected draft, a full rebuild there costs O(n_ctx) per step
|
||||
const llama_pos stale = mem_idx_stale[s];
|
||||
bool keep = !sq.cells.empty() && !sp.empty() && sq.pos_min == sp.begin()->first && stale > sq.pos_min;
|
||||
if (keep) {
|
||||
if (stale != POS_CLEAN) {
|
||||
const auto cut = std::lower_bound(sq.cells.begin(), sq.cells.end(), std::make_pair(stale, uint32_t(0)));
|
||||
sq.cells.erase(cut, sq.cells.end());
|
||||
while (!sq.pools.empty() && sq.pools.back() + kpool > sq.cells.size()) {
|
||||
sq.pools.pop_back();
|
||||
}
|
||||
sq.j_next = sq.pools.empty() ? 0 : sq.pools.back() + kpool;
|
||||
}
|
||||
auto it = stale == POS_CLEAN ? sp.upper_bound(sq.cells.back()) : sp.lower_bound(std::make_pair(stale, uint32_t(0)));
|
||||
for (; it != sp.end(); ++it) {
|
||||
sq.cells.push_back(*it);
|
||||
}
|
||||
keep = sq.cells.size() == sp.size();
|
||||
}
|
||||
|
||||
// the appended tail accounts for every cell only if nothing before it was dropped, but an edit can
|
||||
// regroup a sequence without changing its cell count, so a stale sequence must rebuild regardless
|
||||
if (sq.cells.size() != sp.size() || mem_idx_stale[s] != POS_CLEAN) {
|
||||
// a new sequence, an edit at or before pos_min, or cells that do not match: group the whole sequence again
|
||||
if (!keep) {
|
||||
sq.cells.assign(sp.begin(), sp.end());
|
||||
sq.pools.clear();
|
||||
sq.j_next = 0;
|
||||
|
||||
@@ -86,7 +86,8 @@ public:
|
||||
|
||||
// Which cells of a sequence make up which pool of kpool consecutive positions (or cells, in order mode).
|
||||
// It is kept here because it outlives the batch: pools are fixed by the positions relative to the
|
||||
// sequence's first one, so a ubatch only ever appends to it. Sequence edits drop it, see mem_idx_stale.
|
||||
// sequence's first one, so a ubatch only ever appends to it. A sequence edit drops it from the stale
|
||||
// position on, or all of it for an edit at or before the first position, see mem_idx_stale.
|
||||
struct kpool_layout;
|
||||
|
||||
const kpool_layout & kpool_layout_update();
|
||||
|
||||
@@ -571,6 +571,7 @@ void llama_model_saver::add_tensors_from_model() {
|
||||
add_tensor(model->hc_head_base);
|
||||
add_tensor(model->hc_head_scale);
|
||||
add_tensor(model->per_layer_tok_embd);
|
||||
add_tensor(model->per_layer_tok_embd_s);
|
||||
add_tensor(model->per_layer_model_proj);
|
||||
add_tensor(model->per_layer_proj_norm);
|
||||
add_tensor(model->value_embd);
|
||||
|
||||
@@ -128,6 +128,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
|
||||
return new llama_model_pockettts(params);
|
||||
case LLM_ARCH_ZONOS3:
|
||||
return new llama_model_zonos3(params);
|
||||
case LLM_ARCH_KOKORO:
|
||||
return new llama_model_kokoro(params);
|
||||
case LLM_ARCH_PHI2:
|
||||
return new llama_model_phi2(params);
|
||||
case LLM_ARCH_PHI3:
|
||||
@@ -2185,6 +2187,10 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
|
||||
}
|
||||
}
|
||||
|
||||
if (per_layer_tok_embd_s) {
|
||||
ggml_backend_tensor_get(per_layer_tok_embd_s, &per_layer_tok_embd_scale, 0, sizeof(per_layer_tok_embd_scale));
|
||||
}
|
||||
|
||||
load_tensors_hadamard();
|
||||
|
||||
return true;
|
||||
@@ -2983,6 +2989,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
|
||||
case LLM_ARCH_LLADA_MOE:
|
||||
case LLM_ARCH_RND1:
|
||||
case LLM_ARCH_CLEF:
|
||||
case LLM_ARCH_KOKORO:
|
||||
{
|
||||
res = nullptr;
|
||||
} break;
|
||||
@@ -3715,6 +3722,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
|
||||
case LLM_ARCH_GLM5_NEXT:
|
||||
case LLM_ARCH_STARCODER:
|
||||
case LLM_ARCH_INKLING2:
|
||||
case LLM_ARCH_KOKORO:
|
||||
return LLAMA_ROPE_TYPE_NONE;
|
||||
|
||||
// use what we call a normal RoPE, operating on pairs of consecutive head values
|
||||
|
||||
@@ -881,6 +881,8 @@ struct llama_model {
|
||||
struct ggml_tensor * altup_proj = nullptr;
|
||||
struct ggml_tensor * altup_unembd_proj = nullptr;
|
||||
struct ggml_tensor * per_layer_tok_embd = nullptr;
|
||||
struct ggml_tensor * per_layer_tok_embd_s = nullptr; // optional scalar scale of an FP8 table
|
||||
float per_layer_tok_embd_scale = 1.0f; // value of per_layer_tok_embd_s, read after load
|
||||
|
||||
// limite value embeddings
|
||||
struct ggml_tensor * value_embd = nullptr;
|
||||
|
||||
+40
-40
@@ -20,6 +20,7 @@
|
||||
#include <map>
|
||||
#include <queue>
|
||||
#include <set>
|
||||
#include <string_view>
|
||||
#include <unordered_map>
|
||||
|
||||
//
|
||||
@@ -1480,9 +1481,16 @@ struct llm_tokenizer_plamo2 : llm_tokenizer {
|
||||
to_suffix_id_.clear();
|
||||
table_.clear();
|
||||
|
||||
// Build token list and byte mapping
|
||||
std::unordered_map<std::string, float> suffix_to_score;
|
||||
std::unordered_map<std::string, llama_token> token_to_id;
|
||||
// Suffixes and pieces are views into the token texts of the vocab, so no strings are built
|
||||
std::unordered_map<std::string_view, float> suffix_to_score;
|
||||
std::unordered_map<std::string_view, llama_token> token_to_id;
|
||||
suffix_to_score.reserve(vocab.n_tokens() * 4);
|
||||
token_to_id.reserve(vocab.n_tokens());
|
||||
|
||||
// Byte offset of the next code point
|
||||
auto next_cpt = [](std::string_view s, size_t offset) -> size_t {
|
||||
return offset + std::min(s.size() - offset, unicode_len_utf8(s[offset]));
|
||||
};
|
||||
|
||||
for (size_t token_id = 0; token_id < vocab.n_tokens(); ++token_id) {
|
||||
const auto & entry = vocab.get_token_data(token_id);
|
||||
@@ -1502,18 +1510,10 @@ struct llm_tokenizer_plamo2 : llm_tokenizer {
|
||||
}
|
||||
|
||||
// Add token and all its suffixes to suffix_to_score
|
||||
suffix_to_score[entry.text] = entry.score;
|
||||
|
||||
// Extract suffixes character by character (UTF-8 aware)
|
||||
std::vector<uint32_t> cpts = unicode_cpts_from_utf8(entry.text);
|
||||
for (size_t i = 1; i < cpts.size(); ++i) {
|
||||
std::string suffix;
|
||||
for (size_t j = i; j < cpts.size(); ++j) {
|
||||
suffix += unicode_cpt_to_utf8(cpts[j]);
|
||||
}
|
||||
if (suffix_to_score.find(suffix) == suffix_to_score.end()) {
|
||||
suffix_to_score[suffix] = std::numeric_limits<float>::quiet_NaN();
|
||||
}
|
||||
const std::string_view text = entry.text;
|
||||
suffix_to_score[text] = entry.score;
|
||||
for (size_t i = text.empty() ? 0 : next_cpt(text, 0); i < text.size(); i = next_cpt(text, i)) {
|
||||
suffix_to_score.try_emplace(text.substr(i), std::numeric_limits<float>::quiet_NaN());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1525,44 +1525,47 @@ struct llm_tokenizer_plamo2 : llm_tokenizer {
|
||||
}
|
||||
|
||||
// Build suffix list in lexicographical order of reversed strings
|
||||
std::vector<std::string> suffixes;
|
||||
std::vector<std::string_view> suffixes;
|
||||
suffixes.reserve(suffix_to_score.size() + 1);
|
||||
for (const auto & pair : suffix_to_score) {
|
||||
suffixes.push_back(pair.first);
|
||||
}
|
||||
suffixes.push_back(""); // Empty suffix
|
||||
|
||||
std::sort(suffixes.begin(), suffixes.end(), [](const std::string & a, const std::string & b) {
|
||||
std::string rev_a(a.rbegin(), a.rend());
|
||||
std::string rev_b(b.rbegin(), b.rend());
|
||||
return rev_a < rev_b;
|
||||
std::sort(suffixes.begin(), suffixes.end(), [](std::string_view a, std::string_view b) {
|
||||
return std::lexicographical_compare(a.rbegin(), a.rend(), b.rbegin(), b.rend(), [](char x, char y) {
|
||||
return static_cast<unsigned char>(x) < static_cast<unsigned char>(y);
|
||||
});
|
||||
});
|
||||
|
||||
// Byte length of each prefix of the suffix, by code point
|
||||
std::vector<size_t> prefix_ends;
|
||||
auto get_prefix_ends = [&](std::string_view suffix) {
|
||||
prefix_ends.clear();
|
||||
for (size_t i = 0; i < suffix.size();) {
|
||||
i = next_cpt(suffix, i);
|
||||
prefix_ends.push_back(i);
|
||||
}
|
||||
};
|
||||
|
||||
// Build suffix_to_id and to_suffix_id_
|
||||
std::unordered_map<std::string, int32_t> suffix_to_id;
|
||||
std::unordered_map<std::string_view, int32_t> suffix_to_id;
|
||||
suffix_to_id.reserve(suffixes.size());
|
||||
int32_t num_pieces = 0;
|
||||
|
||||
for (const auto & suffix : suffixes) {
|
||||
suffix_to_id[suffix] = num_pieces;
|
||||
if (!suffix.empty()) {
|
||||
std::vector<uint32_t> cpts = unicode_cpts_from_utf8(suffix);
|
||||
get_prefix_ends(suffix);
|
||||
const uint32_t first_cpt = unicode_cpts_from_utf8(std::string(suffix.substr(0, prefix_ends[0])))[0];
|
||||
|
||||
std::string remaining;
|
||||
for (size_t i = 1; i < cpts.size(); ++i) {
|
||||
remaining += unicode_cpt_to_utf8(cpts[i]);
|
||||
}
|
||||
|
||||
int64_t piece_code = (static_cast<int64_t>(cpts[0]) << 32) | suffix_to_id[remaining];
|
||||
int64_t piece_code = (static_cast<int64_t>(first_cpt) << 32) | suffix_to_id[suffix.substr(prefix_ends[0])];
|
||||
to_suffix_id_[piece_code] = num_pieces;
|
||||
|
||||
// Count number of pieces for this suffix
|
||||
int32_t pieces_for_suffix = 1; // sentinel row
|
||||
for (int32_t piece_length = static_cast<int32_t>(cpts.size()); piece_length > 0; --piece_length) {
|
||||
std::string piece;
|
||||
for (int32_t i = 0; i < piece_length; ++i) {
|
||||
piece += unicode_cpt_to_utf8(cpts[i]);
|
||||
}
|
||||
if (suffix_to_score.find(piece) != suffix_to_score.end()) {
|
||||
for (size_t end : prefix_ends) {
|
||||
if (suffix_to_score.find(suffix.substr(0, end)) != suffix_to_score.end()) {
|
||||
pieces_for_suffix++;
|
||||
}
|
||||
}
|
||||
@@ -1578,12 +1581,9 @@ struct llm_tokenizer_plamo2 : llm_tokenizer {
|
||||
|
||||
for (const auto & suffix : suffixes) {
|
||||
// Add all prefixes of the suffix to the table (in decreasing order of length)
|
||||
std::vector<uint32_t> cpts = unicode_cpts_from_utf8(suffix);
|
||||
for (int32_t piece_length = static_cast<int32_t>(cpts.size()); piece_length > 0; --piece_length) {
|
||||
std::string piece;
|
||||
for (int32_t i = 0; i < piece_length; ++i) {
|
||||
piece += unicode_cpt_to_utf8(cpts[i]);
|
||||
}
|
||||
get_prefix_ends(suffix);
|
||||
for (int32_t piece_length = static_cast<int32_t>(prefix_ends.size()); piece_length > 0; --piece_length) {
|
||||
const std::string_view piece = suffix.substr(0, prefix_ends[piece_length - 1]);
|
||||
|
||||
auto score_it = suffix_to_score.find(piece);
|
||||
if (score_it == suffix_to_score.end()) {
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
#include "models.h"
|
||||
|
||||
// backbone of the Kokoro TTS pipeline: the PL-BERT phoneme encoder (an ALBERT model)
|
||||
// - token and position embeddings have a small width, a projection brings them to n_embd
|
||||
// - one layer is stored and applied n_layer times
|
||||
// it has no lm_head, the rest of the model is inside the mmproj
|
||||
|
||||
void llama_model_kokoro::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
|
||||
|
||||
type = LLM_TYPE_UNKNOWN;
|
||||
}
|
||||
|
||||
void llama_model_kokoro::load_arch_tensors(llama_model_loader & ml) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
// the width of the token and position embeddings comes from the weights
|
||||
const ggml_tensor * pos_meta = ml.get_tensor_meta(tn(LLM_TENSOR_POS_EMBD, "weight").str().c_str());
|
||||
const int64_t n_embd_tok = pos_meta ? pos_meta->ne[0] : n_embd;
|
||||
|
||||
// the converter pads the rows with zeros from n_embd_tok to n_embd, token and embedding inputs must have the same width
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);
|
||||
// the token type embedding is folded into the positions by the converter
|
||||
pos_embd = create_tensor(tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd_tok, n_ctx_train}, 0);
|
||||
|
||||
tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd_tok}, 0);
|
||||
tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias", 0), {n_embd_tok}, 0);
|
||||
|
||||
tok_embd_proj = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_PROJ, "weight", 0), {n_embd_tok, n_embd}, 0);
|
||||
tok_embd_proj_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_PROJ, "bias", 0), {n_embd}, 0);
|
||||
|
||||
// no output head, the logits are unused; reuse the embedding table so a sampler can still run
|
||||
output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
|
||||
|
||||
// the layers share their weights, only the first one is stored
|
||||
{
|
||||
auto & layer = layers[0];
|
||||
const int i = 0;
|
||||
|
||||
create_tensor_qkv(layer, i, n_embd, n_embd, n_embd_gqa, n_embd_gqa, 0);
|
||||
|
||||
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
|
||||
layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, 0);
|
||||
|
||||
layer.attn_out_norm = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "weight", i), {n_embd}, 0);
|
||||
layer.attn_out_norm_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_NORM, "bias", i), {n_embd}, 0);
|
||||
|
||||
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
|
||||
layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {n_ff}, 0);
|
||||
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
|
||||
layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, 0);
|
||||
|
||||
layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, 0);
|
||||
layer.layer_out_norm_b = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "bias", i), {n_embd}, 0);
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<llm_graph_context> llama_model_kokoro::build_arch_graph(const llm_graph_params & params) const {
|
||||
return std::make_unique<graph>(params);
|
||||
}
|
||||
|
||||
llama_model_kokoro::graph::graph(const llm_graph_params & params) : llm_graph_context(params) {
|
||||
const auto & model = static_cast<const llama_model_kokoro &>(this->model);
|
||||
|
||||
const int64_t n_embd_head = hparams.n_embd_head_v();
|
||||
const int64_t n_embd_tok = model.pos_embd->ne[0];
|
||||
|
||||
GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
|
||||
|
||||
ggml_tensor * cur;
|
||||
ggml_tensor * inpL;
|
||||
|
||||
ggml_tensor * inp_pos = build_inp_pos();
|
||||
ggml_tensor * embd_pos = ggml_get_rows(ctx0, model.pos_embd, inp_pos);
|
||||
ggml_build_forward_order(gf, embd_pos);
|
||||
|
||||
// token + position (+ type) embeddings, in the small width
|
||||
inpL = build_inp_embd(model.tok_embd);
|
||||
inpL = ggml_cont(ctx0, ggml_view_2d(ctx0, inpL, n_embd_tok, n_tokens, inpL->nb[1], 0));
|
||||
inpL = ggml_add(ctx0, inpL, embd_pos);
|
||||
cb(inpL, "inp_embd", -1);
|
||||
|
||||
inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, 0);
|
||||
cb(inpL, "inp_norm", 0);
|
||||
|
||||
inpL = ggml_add(ctx0, ggml_mul_mat(ctx0, model.tok_embd_proj, inpL), model.tok_embd_proj_b);
|
||||
cb(inpL, "inp_proj", 0);
|
||||
|
||||
auto * inp_attn = build_attn_inp_no_cache();
|
||||
|
||||
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
||||
|
||||
const auto & layer = model.layers[0];
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
cur = inpL;
|
||||
|
||||
{
|
||||
auto [Qcur, Kcur, Vcur] = build_qkv(layer, cur,
|
||||
n_embd_head, n_head, n_head_kv, il);
|
||||
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
|
||||
cur = build_attn(inp_attn,
|
||||
layer.wo, layer.wo_b, layer.wo_s,
|
||||
Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
|
||||
cb(cur, "kqv_out", il);
|
||||
}
|
||||
|
||||
if (il == n_layer - 1 && inp_out_ids) {
|
||||
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
||||
inpL = ggml_get_rows(ctx0, inpL, inp_out_ids);
|
||||
}
|
||||
|
||||
cur = ggml_add(ctx0, cur, inpL);
|
||||
cur = build_norm(cur, layer.attn_out_norm, layer.attn_out_norm_b, LLM_NORM, il);
|
||||
|
||||
ggml_tensor * ffn_inp = cur;
|
||||
cb(ffn_inp, "ffn_inp", il);
|
||||
|
||||
cur = build_ffn(cur,
|
||||
layer.ffn_up, layer.ffn_up_b, NULL,
|
||||
NULL, NULL, NULL,
|
||||
layer.ffn_down, layer.ffn_down_b, NULL, NULL,
|
||||
LLM_FFN_GELU, LLM_FFN_SEQ, il);
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
cur = ggml_add(ctx0, cur, ffn_inp);
|
||||
cur = build_norm(cur, layer.layer_out_norm, layer.layer_out_norm_b, LLM_NORM, il);
|
||||
|
||||
inpL = cur;
|
||||
}
|
||||
|
||||
cur = inpL;
|
||||
|
||||
cb(cur, "result_embd", -1);
|
||||
res->t_embd = cur;
|
||||
|
||||
cur = build_lora_mm(model.output, cur);
|
||||
|
||||
cb(cur, "result_output", -1);
|
||||
res->t_logits = cur;
|
||||
}
|
||||
@@ -769,6 +769,22 @@ struct llama_model_pockettts : public llama_model_base {
|
||||
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
|
||||
};
|
||||
|
||||
struct llama_model_kokoro : public llama_model_base {
|
||||
llama_model_kokoro(const struct llama_model_params & params) : llama_model_base(params) {}
|
||||
void load_arch_hparams(llama_model_loader & ml) override;
|
||||
void load_arch_tensors(llama_model_loader & ml) override;
|
||||
|
||||
// token + position embeddings are narrower than n_embd, then projected up
|
||||
ggml_tensor * tok_embd_proj = nullptr;
|
||||
ggml_tensor * tok_embd_proj_b = nullptr;
|
||||
|
||||
struct graph : public llm_graph_context {
|
||||
graph(const llm_graph_params & params);
|
||||
};
|
||||
|
||||
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
|
||||
};
|
||||
|
||||
struct llama_model_zonos3 : public llama_model_base {
|
||||
llama_model_zonos3(const struct llama_model_params & params) : llama_model_base(params) {}
|
||||
void load_arch_hparams(llama_model_loader & ml) override;
|
||||
|
||||
@@ -215,6 +215,11 @@ void llama_model_qwen4exp::load_arch_tensors(llama_model_loader & ml) {
|
||||
|
||||
per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"),
|
||||
{ hparams.ple_head_dim, ple_rows }, TENSOR_READ_LAZY);
|
||||
// an FP8 table keeps its checkpoint bytes, so its per-tensor scale is applied after the gather
|
||||
per_layer_tok_embd_s = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "scale"), { 1 }, TENSOR_NOT_REQUIRED);
|
||||
if (per_layer_tok_embd_s && per_layer_tok_embd_s->type != GGML_TYPE_F32) {
|
||||
throw std::runtime_error(format("%s must be F32", per_layer_tok_embd_s->name));
|
||||
}
|
||||
}
|
||||
|
||||
auto load_block = [&](int il, int flags) {
|
||||
@@ -1408,6 +1413,9 @@ ggml_tensor * llama_model_qwen4exp::graph::build_inp_ple(
|
||||
|
||||
// gather then flatten the heads: get_rows lays the head dimension out slowest, as the reference does
|
||||
ggml_tensor * emb = ggml_get_rows(ctx0, model.per_layer_tok_embd, rows);
|
||||
if (model.per_layer_tok_embd_s) {
|
||||
emb = ggml_scale(ctx0, emb, model.per_layer_tok_embd_scale);
|
||||
}
|
||||
emb = ggml_reshape_2d(ctx0, emb, hparams.ple_head_dim * n_heads, n_tokens);
|
||||
cb(emb, "ple_embd", -1);
|
||||
|
||||
|
||||
@@ -399,7 +399,7 @@ endif()
|
||||
# libmtmd
|
||||
llama_build_and_test(test-mtmd-c-api.c LABEL behavior LINK mtmd)
|
||||
|
||||
llama_build_and_test(test-mtmd-impl.cpp LABEL behavior LINK mtmd)
|
||||
llama_build_and_test(test-mtmd-impl.cpp LABEL behavior LINK mtmd llama-common)
|
||||
|
||||
# GGUF model data fetcher library for tests that need real model metadata
|
||||
# Only compile when cpp-httplib has SSL support (CPPHTTPLIB_OPENSSL_SUPPORT)
|
||||
|
||||
@@ -239,6 +239,17 @@ static void test(void) {
|
||||
argv = {"binary_name", "-lm", "hello"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
// --ignore-eos is not supported with a grammar
|
||||
{
|
||||
common_params grammar_params;
|
||||
argv = {"binary_name", "-m", "model_file.gguf", "--grammar", "root ::= \"ok\""};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), grammar_params, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
common_params grammar_eos_params;
|
||||
argv = {"binary_name", "-m", "model_file.gguf", "--grammar", "root ::= \"ok\"", "--ignore-eos"};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), grammar_eos_params, LLAMA_EXAMPLE_COMMON));
|
||||
}
|
||||
|
||||
printf("test-arg-parser: test valid usage\n\n");
|
||||
|
||||
argv = {"binary_name", "-m", "model_file.gguf"};
|
||||
|
||||
@@ -10920,6 +10920,11 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
}
|
||||
}
|
||||
}
|
||||
// one window over the whole row (a row reduction)
|
||||
for (int64_t iw : {10, 128, 256, 1000, 2048, 4099}) {
|
||||
test_cases.emplace_back(new test_pool1d(pool_type, type_input, { iw, 3, 2, 1 }, (int) iw, (int) iw, 0));
|
||||
test_cases.emplace_back(new test_pool1d(pool_type, type_input, { iw, 1024, 1, 1 }, (int) iw, (int) iw, 0));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12025,6 +12030,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 6170, 13, k, {1, 1}, {1, 1}));
|
||||
test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 1000, 21, k, {1, 1}, {1, 1}));
|
||||
}
|
||||
// MUL_MAT_ID: K = 8 units (the warps split K, broadcast src1), K = 2 units (the warps split the rows, more than 8 tokens per expert, unaligned experts)
|
||||
test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 16, 4, true, 72, 13, 2048));
|
||||
test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 2, false, 202, 24, 512));
|
||||
}
|
||||
// permuted and broadcast src0/src1 for the 2 byte aligned types
|
||||
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 64, 13, 2304, {2, 3}, {1, 1}, {0, 2, 1, 3}));
|
||||
@@ -12039,6 +12047,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_mul_mat_pos(GGML_TYPE_Q5_K, GGML_TYPE_F32, 1000, 13, 4608, {1, 1}, {1, 1}));
|
||||
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_K, GGML_TYPE_F32, 6170, 13, 4608, {1, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 0, 0, 0, true));
|
||||
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 1000, 21, 4608, {1, 1}, {1, 1}, {0, 1, 2, 3}, 0, 1, false, 0, 0, 0, true));
|
||||
// CUDA few-row MMQ for MUL_MAT_ID: strided experts, more than 32 experts
|
||||
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 16, 4, true, 72, 13, 2048, 80));
|
||||
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q4_K, GGML_TYPE_F32, 128, 8, false, 72, 16, 2048));
|
||||
|
||||
#if 0
|
||||
{
|
||||
@@ -13406,6 +13417,8 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_mul_mat_fusion_w4a8(GGML_TYPE_NVFP4, GGML_GLU_OP_SWIGLU, 150, 2560, 256, true, 16, 8, false, false, true, true));
|
||||
// K = 2048: few-row MMQ, where enabled, takes the gate and up matmuls instead of the fused MMQ path
|
||||
test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 9, 128, 2048, false, 1, 1, false, false, true, false, { 1, 1 }));
|
||||
// on DGX Spark also the expert gate and up matmuls at 8 tokens, instead of the fused MMVQ path
|
||||
test_cases.emplace_back(new test_mul_mat_fusion(GGML_TYPE_Q4_K, GGML_GLU_OP_SWIGLU, 8, 128, 2048, true, 16, 4, true));
|
||||
|
||||
for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT, GATING_FUNC_SQRT_SOFTPLUS}) {
|
||||
for (bool with_norm : {false, true}) {
|
||||
@@ -13652,6 +13665,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
|
||||
}
|
||||
}
|
||||
}
|
||||
// POOL_1D whole-row reduction with short rows and many rows
|
||||
for (ggml_op_pool pool_type : {GGML_OP_POOL_AVG, GGML_OP_POOL_MAX}) {
|
||||
test_cases.emplace_back(new test_pool1d(pool_type, GGML_TYPE_F32, {16, 65536, 1, 1}, 16, 16, 0));
|
||||
}
|
||||
|
||||
// LEAKY_RELU at FFN activation width, for direct comparison with RELU
|
||||
for (int64_t n_tokens : {512, 2048}) {
|
||||
|
||||
@@ -1019,6 +1019,58 @@ static bool check_causal_attn_toggle(
|
||||
return ok;
|
||||
}
|
||||
|
||||
// speculative decoding: each step decodes a token and 0 to n_draft_max drafts, then rejects the drafts
|
||||
// the logits of the token must match a decode without drafts
|
||||
static double check_draft_rollback(llama_model * model, const std::vector<llama_token> & tokens, const std::vector<float> & logits_ref) {
|
||||
const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));
|
||||
const uint32_t n_tokens = tokens.size();
|
||||
const uint32_t n_draft_max = 2;
|
||||
|
||||
llama_context_params ctx_params = llama_context_default_params();
|
||||
ctx_params.n_ctx = 0;
|
||||
ctx_params.n_threads = 4;
|
||||
ctx_params.n_threads_batch = 4;
|
||||
ctx_params.n_ubatch = 64;
|
||||
ctx_params.n_rs_seq = n_draft_max;
|
||||
|
||||
llama_context_ptr lctx(llama_init_from_model(model, ctx_params));
|
||||
if (!lctx) {
|
||||
throw std::runtime_error("failed to create llama context");
|
||||
}
|
||||
|
||||
common_batch batch(lctx.get());
|
||||
for (uint32_t pos = 0; pos < n_tokens/2; pos++) {
|
||||
batch.add(tokens[pos], pos, 0, false);
|
||||
}
|
||||
if (llama_process(lctx.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
|
||||
throw std::runtime_error("failed to decode batch");
|
||||
}
|
||||
|
||||
std::vector<float> logits;
|
||||
std::vector<float> ref;
|
||||
for (uint32_t pos = n_tokens/2; pos < n_tokens; pos++) {
|
||||
batch.clear();
|
||||
batch.add(tokens[pos], pos, 0, true);
|
||||
const uint32_t n_draft = pos % (n_draft_max + 1);
|
||||
for (uint32_t i = 1; i <= n_draft; i++) {
|
||||
batch.add((tokens[pos] + i) % n_vocab, pos + i, 0, true);
|
||||
}
|
||||
if (llama_process(lctx.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
|
||||
throw std::runtime_error("failed to decode batch");
|
||||
}
|
||||
|
||||
const float * logits_ith = llama_get_logits_ith(lctx.get(), 0);
|
||||
logits.insert(logits.end(), logits_ith, logits_ith + n_vocab);
|
||||
ref.insert(ref.end(), logits_ref.begin() + (size_t) pos*n_vocab, logits_ref.begin() + (size_t) (pos + 1)*n_vocab);
|
||||
|
||||
if (!llama_memory_seq_rm(llama_get_memory(lctx.get()), 0, pos + 1, -1)) {
|
||||
throw std::runtime_error("failed to reject the drafts");
|
||||
}
|
||||
}
|
||||
|
||||
return nmse(ref, logits);
|
||||
}
|
||||
|
||||
static bool moe_mandatory(const llm_arch arch) {
|
||||
switch (arch) {
|
||||
case LLM_ARCH_FLINT:
|
||||
@@ -1132,6 +1184,9 @@ static bool arch_supported(const llm_arch arch) {
|
||||
arch == LLM_ARCH_NEO_BERT || arch == LLM_ARCH_JINA_BERT_V2 || arch == LLM_ARCH_JINA_BERT_V3 || arch == LLM_ARCH_EUROBERT) {
|
||||
return false; // TODO vocab
|
||||
}
|
||||
if (arch == LLM_ARCH_KOKORO) {
|
||||
return false; // TODO encoder without memory, a sequence must fit in one ubatch
|
||||
}
|
||||
if (arch == LLM_ARCH_PLM) {
|
||||
return false; // TODO tensor shapes
|
||||
}
|
||||
@@ -1446,6 +1501,18 @@ static int test_backends(const std::string & arch_filter, const size_t seed, con
|
||||
}
|
||||
test_ok = false;
|
||||
}
|
||||
|
||||
// the k-pool layout is updated in place when drafts are rejected
|
||||
// TODO: the meta backend aborts with n_rs_seq > 0 (view of permuted tensor not implemented)
|
||||
if ((arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_GLM5_NEXT) && dc.split_mode != LLAMA_SPLIT_MODE_TENSOR) {
|
||||
const double nmse_rollback = check_draft_rollback(model_and_ctx_dev.first.get(), tokens, logits_dev);
|
||||
if (!(nmse_rollback <= 1e-4)) {
|
||||
if (test_ok) {
|
||||
status_nmse = "\033[1;31mFAIL\033[0m (rollback)";
|
||||
}
|
||||
test_ok = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (llama_arch_supports_save(arch) && dc.split_mode != LLAMA_SPLIT_MODE_TENSOR) {
|
||||
|
||||
@@ -7,12 +7,22 @@
|
||||
#include "mtmd-helper.h"
|
||||
#include "mtmd-image.h"
|
||||
#include "mtmd-internal.h"
|
||||
#include "models/models.h"
|
||||
|
||||
#include "chat-tts-norm.h"
|
||||
#include "jinja/lexer.h"
|
||||
#include "jinja/parser.h"
|
||||
#include "jinja/runtime.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <iterator>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
@@ -881,6 +891,177 @@ MAKE_TEST(test_helper_resampler) {
|
||||
t.assert_true("invalid rate", mtmd_helper_resampler_init(0, rate_out) == nullptr);
|
||||
}
|
||||
|
||||
//
|
||||
// kokoro
|
||||
//
|
||||
|
||||
// a dictionary for tts_dict(): entries in any order, sorted here
|
||||
static common_tts_dict kokoro_dict(std::vector<std::pair<std::string, std::string>> entries) {
|
||||
std::sort(entries.begin(), entries.end());
|
||||
common_tts_dict dict;
|
||||
for (auto & e : entries) {
|
||||
dict.keys.push_back(e.first);
|
||||
dict.values.push_back(e.second);
|
||||
}
|
||||
return dict;
|
||||
}
|
||||
|
||||
// the "tts" template with small dictionaries: a few lexicon words, fixed tags, and a model for unknown words over the letters z a b i n g
|
||||
MAKE_TEST(test_kokoro_tts_template) {
|
||||
std::string path = __FILE__;
|
||||
const size_t pos = path.find_last_of("/\\");
|
||||
path = (pos == std::string::npos ? std::string(".") : path.substr(0, pos)) + "/../models/templates/hexgrad-Kokoro-82M-tts.jinja";
|
||||
std::ifstream fs(path, std::ios_base::binary);
|
||||
if (!t.assert_true("template file is readable: " + path, fs.is_open())) {
|
||||
return;
|
||||
}
|
||||
const std::string tmpl((std::istreambuf_iterator<char>(fs)), std::istreambuf_iterator<char>());
|
||||
|
||||
const auto dicts = std::make_shared<const common_tts_dicts>(common_tts_dicts{
|
||||
{"gold", kokoro_dict({
|
||||
{"hello", "\"h\xc9\x99l\xcb\x88O\""},
|
||||
{"world", "\"w\xcb\x88\xc9\x9c\xc9\xb9ld\""},
|
||||
{"I", "\"\xcb\x88I\""},
|
||||
{"read", "{\"ADJ\":\"\xc9\xb9\xcb\x88\xc9\x9b" "d\",\"DEFAULT\":\"\xc9\xb9\xcb\x88id\",\"VBD\":\"\xc9\xb9\xcb\x88\xc9\x9b" "d\",\"VBN\":\"\xc9\xb9\xcb\x88\xc9\x9b" "d\",\"VBP\":\"\xc9\xb9\xcb\x88\xc9\x9b" "d\"}"},
|
||||
{"it", "\"\xc9\xaat\""},
|
||||
{"three", "\"\xce\xb8\xc9\xb9\xcb\x88i\""},
|
||||
{"cat", "\"k\xcb\x88\xc3\xa6t\""},
|
||||
{"walk", "\"w\xcb\x88\xc9\x94k\""},
|
||||
{"the", "\"\xc3\xb0i\""},
|
||||
{"Z", "\"z\xcb\x88i\""},
|
||||
{"'s", "\"s\""},
|
||||
{"a", "\"A\""},
|
||||
{"and", "\"\xc3\xa6nd\""},
|
||||
{"is", "\"\xc9\xaaz\""},
|
||||
})},
|
||||
{"silver", kokoro_dict({
|
||||
})},
|
||||
{"gold_gb", kokoro_dict({
|
||||
{"hello", "\"h\xc9\x99l\xcb\x88Q\""},
|
||||
{"world", "\"w\xcb\x88\xc9\x9c\xcb\x90ld\""},
|
||||
{"walk", "\"w\xcb\x88\xc9\x94\xcb\x90k\""},
|
||||
})},
|
||||
{"silver_gb", kokoro_dict({
|
||||
})},
|
||||
{"pos_classes", kokoro_dict({
|
||||
{"", "[\"NN\",\"NNP\",\"PRP\",\"VBD\"]"},
|
||||
})},
|
||||
{"pos_tagdict", kokoro_dict({
|
||||
{"I", "\"PRP\""},
|
||||
{"read", "\"VBD\""},
|
||||
{"it", "\"PRP\""},
|
||||
{".", "\".\""},
|
||||
{",", "\",\""},
|
||||
})},
|
||||
{"pos_weights", kokoro_dict({
|
||||
{"bias", "{\"c0\":1.0}"},
|
||||
})},
|
||||
{"lts", kokoro_dict({
|
||||
{"z", "{\"z\":0.1}"},
|
||||
{"b", "{\"b\":0.1}"},
|
||||
{"i", "{\"\xc9\xaa\":0.1}"},
|
||||
{"n", "{\"\xc5\x8b\":0.1}"},
|
||||
{"g", "{\"\":0.1}"},
|
||||
{"a", "{\"\xcb\x88\xc3\xa6\":0.1,\"\xc3\xa6\":0.06}"},
|
||||
{"z:z a", "{\"\xcb\x88\xc3\xa6\":0.2,\"\xc3\xa6\":0.01}"},
|
||||
{"$", "{\"\":0.2}"},
|
||||
})},
|
||||
{"lts_gb", kokoro_dict({
|
||||
{"z", "{\"z\":0.1}"},
|
||||
{"b", "{\"b\":0.1}"},
|
||||
{"i", "{\"\xc9\xaa\":0.1}"},
|
||||
{"n", "{\"\xc5\x8b\":0.1}"},
|
||||
{"g", "{\"\":0.1}"},
|
||||
{"a", "{\"\xcb\x88\xc9\x91\xcb\x90\":0.1,\"\xc9\x91\xcb\x90\":0.06}"},
|
||||
{"z:z a", "{\"\xcb\x88\xc9\x91\xcb\x90\":0.2,\"\xc9\x91\xcb\x90\":0.01}"},
|
||||
{"$", "{\"\":0.2}"},
|
||||
})},
|
||||
});
|
||||
|
||||
// voice, text, expected phonemes
|
||||
const std::vector<std::array<std::string, 3>> cases = {
|
||||
// words of the lexicon
|
||||
{ "af_heart", "Hello world.", "h\xc9\x99l\xcb\x88O w\xcb\x88\xc9\x9c\xc9\xb9ld." },
|
||||
// the pronunciation depends on the part of speech
|
||||
{ "af_heart", "I read it.", "\xcb\x8cI \xc9\xb9\xcb\x88\xc9\x9b" "d \xc9\xaat." },
|
||||
// a number, and the -s suffix rule
|
||||
{ "af_heart", "3 cats", "\xce\xb8\xc9\xb9\xcb\x88i k\xcb\x88\xc3\xa6ts" },
|
||||
// the -ed and -ing suffix rules
|
||||
{ "af_heart", "walked, walking", "w\xcb\x88\xc9\x94kt, w\xcb\x88\xc9\x94k\xc9\xaa\xc5\x8b" },
|
||||
// not in the lexicon: predicted from the spelling
|
||||
{ "af_heart", "zab", "z\xcb\x88\xc3\xa6" "b" },
|
||||
// the free search gives two primary stresses, the stress pass keeps one
|
||||
{ "af_heart", "zaba", "z\xcb\x88\xc3\xa6" "b\xc3\xa6" },
|
||||
// a doubled consonant says one sound
|
||||
{ "af_heart", "zabbing", "z\xcb\x88\xc3\xa6" "b\xc9\xaa\xc5\x8b" },
|
||||
// an unknown word with 's
|
||||
{ "af_heart", "The zab's cat", "\xc3\xb0\xc9\x99 z\xcb\x88\xc3\xa6" "bs k\xcb\x88\xc3\xa6t" },
|
||||
// no vowel: spelled out
|
||||
{ "af_heart", "zz", "z\xcb\x8ciz\xcb\x88i" },
|
||||
// an accented letter is read as its base letter
|
||||
{ "af_heart", "H\xc3\xa9ll\xc3\xb3 world.", "h\xc9\x99l\xcb\x88O w\xcb\x88\xc9\x9c\xc9\xb9ld." },
|
||||
// digits glued to letters are read as a number
|
||||
{ "af_heart", "3Z cats", "\xce\xb8\xc9\xb9\xcb\x88i z\xcb\x88i k\xcb\x88\xc3\xa6ts" },
|
||||
// an unknown word in mixed case is read part by part
|
||||
{ "af_heart", "catWalk", "k\xcb\x88\xc3\xa6tw\xcb\x88\xc9\x94k" },
|
||||
// a british voice uses the british lexicon
|
||||
{ "bf_emma", "Hello world.", "h\xc9\x99l\xcb\x88Q w\xcb\x88\xc9\x9c\xcb\x90ld." },
|
||||
// and the british model for unknown words
|
||||
{ "bf_emma", "zaba", "z\xcb\x88\xc9\x91\xcb\x90" "b\xc9\x91\xcb\x90" },
|
||||
// a voice of another language takes phonemes, they pass through
|
||||
{ "ef_dora", "\xcb\x88ola, m\xcb\x88undo.", "\xcb\x88ola, m\xcb\x88undo." },
|
||||
};
|
||||
|
||||
jinja::lexer lexer;
|
||||
const jinja::program ast = jinja::parse_from_tokens(lexer.tokenize(tmpl));
|
||||
for (const auto & c : cases) {
|
||||
std::string out;
|
||||
try {
|
||||
jinja::context ctx(tmpl);
|
||||
common_chat_tts_norm_add_builtins(ctx, dicts);
|
||||
auto msg = jinja::mk_val<jinja::value_object>();
|
||||
msg->insert("role", jinja::mk_val<jinja::value_string>("user"));
|
||||
msg->insert("content", jinja::mk_val<jinja::value_string>(c[1]));
|
||||
auto msgs = jinja::mk_val<jinja::value_array>();
|
||||
msgs->push_back(msg);
|
||||
ctx.set_val("messages", msgs);
|
||||
ctx.set_val("language", jinja::mk_val<jinja::value_string>(""));
|
||||
ctx.set_val("voice", jinja::mk_val<jinja::value_string>(c[0]));
|
||||
jinja::runtime runtime(ctx);
|
||||
out = jinja::runtime::gather_string_parts(runtime.execute(ast))->as_string().str();
|
||||
} catch (const std::exception & e) {
|
||||
out = std::string("exception: ") + e.what();
|
||||
}
|
||||
if (!t.assert_true(c[0] + ": " + c[1], out == c[2])) {
|
||||
t.log("Expected: " + c[2]);
|
||||
t.log("Actual : " + out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// every frame is the constant 0.25 (only the DC bin, of magnitude n_fft * 0.25)
|
||||
// the overlap-add of hann windows at a quarter hop sums to 2, their squares to 1.5: the output is 0.25 * 2 / 1.5
|
||||
MAKE_TEST(test_kokoro_istft) {
|
||||
const int n_fft = 20, hop = 5, n_frames = 9;
|
||||
std::vector<float> spec((size_t) n_frames * (n_fft + 2), 0.0f);
|
||||
for (int fr = 0; fr < n_frames; fr++) {
|
||||
float * row = spec.data() + (size_t) fr * (n_fft + 2);
|
||||
// the first half of a row is log magnitudes, the second half phases
|
||||
for (int k = 0; k <= n_fft / 2; k++) {
|
||||
row[k] = k == 0 ? logf(n_fft * 0.25f) : -100.0f;
|
||||
}
|
||||
}
|
||||
std::vector<float> audio;
|
||||
mtmd_graph_kokoro::istft(spec, audio);
|
||||
t.assert_equal("samples", (size_t) (n_frames - 1) * hop, audio.size());
|
||||
// the first and last hop are covered by fewer frames
|
||||
float max_err = 0.0f;
|
||||
for (size_t i = hop; i + hop < audio.size(); i++) {
|
||||
max_err = std::max(max_err, std::fabs(audio[i] - 1.0f / 3.0f));
|
||||
}
|
||||
t.assert_true("constant signal, max error " + std::to_string(max_err), max_err < 1e-4f);
|
||||
}
|
||||
|
||||
//
|
||||
// main
|
||||
//
|
||||
|
||||
@@ -25,6 +25,7 @@ from conversion.llama import LlamaModel
|
||||
from conversion.nemotron import NemotronHModel
|
||||
from conversion.phi import PhiMoeModel
|
||||
from conversion.qwen import Qwen2MoeModel, Qwen3NextModel
|
||||
from conversion.qwen4exp import Qwen4ExpTextModel
|
||||
|
||||
|
||||
class ForcedF16Model(Qwen2MoeModel):
|
||||
@@ -366,6 +367,27 @@ class TestFP8Conversion(unittest.TestCase):
|
||||
self.assertEqual(dtype, gguf.GGMLQuantizationType.Q8_0)
|
||||
np.testing.assert_allclose(gguf.dequantize(data, dtype), np.full((4, 32), 0.25), atol=1e-4)
|
||||
|
||||
def test_modelopt_block_scale_inv(self):
|
||||
name = "model.layers.0.self_attn.q_proj.weight"
|
||||
weight = (torch.arange(32, dtype=torch.float32).reshape(4, 8) - 16).to(torch.float8_e4m3fn)
|
||||
scale = torch.tensor([[0.25, 0.5], [1.0, 2.0]])
|
||||
expected = weight.float() * scale.repeat_interleave(2, 0).repeat_interleave(4, 1)
|
||||
for eager in (False, True):
|
||||
with self.subTest(eager=eager):
|
||||
result = self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": scale}, eager=eager, check_lazy=not eager)
|
||||
self.assertEqual(set(result), {"blk.0.attn_q.weight"})
|
||||
dtype, data = result["blk.0.attn_q.weight"]
|
||||
self.assertEqual(dtype, gguf.GGMLQuantizationType.F16)
|
||||
np.testing.assert_array_equal(data, expected.numpy())
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "not tiled"):
|
||||
self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": torch.ones((3, 2))})
|
||||
|
||||
# a scalar scale keeps the FP8 bytes, as with weight_scale
|
||||
result = self.convert(Qwen2MoeModel, {name: weight, name + "_scale_inv": torch.tensor([0.25])}, check_lazy=True)
|
||||
self.assertEqual(result["blk.0.attn_q.weight"][0], gguf.GGMLQuantizationType.F8_E4M3)
|
||||
np.testing.assert_array_equal(result["blk.0.attn_q.scale"][1], [0.25])
|
||||
|
||||
|
||||
class TestInkling2PackedConversion(unittest.TestCase):
|
||||
"""Conversion of the packed Inkling2 expert weights (see _dequant_expert_chunk in conversion/inkling2.py).
|
||||
@@ -675,5 +697,84 @@ class TestInkling2TextNames(unittest.TestCase):
|
||||
np.testing.assert_array_equal(renamed[name], actual[name], name)
|
||||
|
||||
|
||||
class TestQwen4ExpPleScale(unittest.TestCase):
|
||||
"""FP8 checkpoints store the n-gram table as shards with one shared scale under the parent module name."""
|
||||
|
||||
PREFIX = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding."
|
||||
SHARDS = (torch.tensor([[1., -2.], [3., 4.]]), torch.tensor([[5., 6.], [-7., 8.]]))
|
||||
TABLE = "per_layer_token_embd.weight"
|
||||
SCALE = "per_layer_token_embd.scale"
|
||||
|
||||
def convert(self, ftype=gguf.LlamaFileType.GUESSED, scaled=True, quant_method="modelopt", **kwargs):
|
||||
# shard 1 first: the row order must follow the shard index
|
||||
tensors = {self.PREFIX + f"shard_{i}.weight": self.SHARDS[i].to(torch.float8_e4m3fn) for i in (1, 0)}
|
||||
if scaled:
|
||||
tensors[self.PREFIX + "weight_scale"] = torch.tensor([0.25], dtype=torch.bfloat16)
|
||||
hparams = {
|
||||
"model_type": "qwen4exp",
|
||||
"architectures": ["Qwen4ExpForCausalLM"],
|
||||
"num_hidden_layers": 2,
|
||||
"mtp_num_hidden_layers": 1,
|
||||
"split_ngram_parts": 2,
|
||||
"quantization_config": {"quant_method": quant_method},
|
||||
}
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory)
|
||||
(path / "config.json").write_text(json.dumps(hparams))
|
||||
save_file(tensors, str(path / "model.safetensors"))
|
||||
model = Qwen4ExpTextModel(path, ftype, path / "out.gguf", hparams=hparams, **kwargs)
|
||||
model.prepare_tensors()
|
||||
result = {}
|
||||
for name, info in model.gguf_writer.tensors[0].items():
|
||||
if isinstance(info.tensor, gguf.LazyChunkedTensor):
|
||||
# written like the GGUF writer does, a chunked tensor has no other read path
|
||||
with (path / "tensor.bin").open("wb") as f:
|
||||
info.tensor.tofile(f)
|
||||
data = np.frombuffer((path / "tensor.bin").read_bytes(), dtype=info.tensor.dtype).reshape(info.tensor.shape)
|
||||
else:
|
||||
data = np.asarray(gguf.LazyNumpyTensor.to_eager(info.tensor))
|
||||
result[name] = (info.dtype, data)
|
||||
return result
|
||||
|
||||
def test_fp8_table_keeps_bytes_and_scale(self):
|
||||
raw = torch.cat(self.SHARDS).to(torch.float8_e4m3fn).view(torch.uint8).numpy()
|
||||
for kwargs, names in (
|
||||
({}, {self.TABLE, self.SCALE}),
|
||||
({"quant_method": "fp8"}, {self.TABLE, self.SCALE}),
|
||||
({"ple_only": True}, {self.TABLE}),
|
||||
({"no_ple": True}, {self.SCALE}),
|
||||
):
|
||||
with self.subTest(**kwargs):
|
||||
result = self.convert(**kwargs)
|
||||
self.assertEqual(set(result), names)
|
||||
if self.TABLE in result:
|
||||
dtype, data = result[self.TABLE]
|
||||
self.assertEqual(dtype, gguf.GGMLQuantizationType.F8_E4M3)
|
||||
np.testing.assert_array_equal(data, raw)
|
||||
if self.SCALE in result:
|
||||
dtype, data = result[self.SCALE]
|
||||
self.assertEqual(dtype, gguf.GGMLQuantizationType.F32)
|
||||
np.testing.assert_array_equal(data, [0.25])
|
||||
|
||||
result = self.convert(scaled=False)
|
||||
self.assertEqual(set(result), {self.TABLE})
|
||||
np.testing.assert_array_equal(result[self.TABLE][1], raw)
|
||||
|
||||
def test_scale_is_folded_into_other_types(self):
|
||||
values = torch.cat(self.SHARDS).numpy()
|
||||
for kwargs, qtype in (
|
||||
({"ftype": gguf.LlamaFileType.ALL_F32}, gguf.GGMLQuantizationType.F32),
|
||||
({"ftype": gguf.LlamaFileType.MOSTLY_F16}, gguf.GGMLQuantizationType.F16),
|
||||
({"fp8_as_q8": True}, gguf.GGMLQuantizationType.F16),
|
||||
):
|
||||
for scaled in (False, True):
|
||||
with self.subTest(scaled=scaled, **kwargs):
|
||||
result = self.convert(scaled=scaled, **kwargs)
|
||||
self.assertEqual(set(result), {self.TABLE})
|
||||
dtype, data = result[self.TABLE]
|
||||
self.assertEqual(dtype, qtype)
|
||||
np.testing.assert_array_equal(data, values * (0.25 if scaled else 1.))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -95,6 +95,7 @@ add_library(mtmd
|
||||
models/pockettts-spkenc.cpp
|
||||
models/pockettts-gen.cpp
|
||||
models/soprano.cpp
|
||||
models/kokoro.cpp
|
||||
models/step3vl.cpp
|
||||
models/siglip.cpp
|
||||
models/whisper-enc.cpp
|
||||
|
||||
@@ -96,6 +96,28 @@ Some stages are a full LLM-sized transformer that attends to the keys/values of
|
||||
- Non-causal rows: `LLAMA_ATTENTION_TYPE_NON_CAUSAL`, all rows in one ubatch (`n_ubatch` >= rows), they see every cached position of the sequence. Remove them with `llama_memory_seq_rm()` after each pass
|
||||
- Size the context per call (`n_ctx`, `n_ubatch`) instead of keeping one for the whole session
|
||||
|
||||
### Text preprocessing (the `tts` chat template)
|
||||
|
||||
Anything a model needs done to the input text before tokenization (prompt format, text normalization, text to phonemes) goes in `tokenizer.chat_template.tts`, as a jinja program. It receives the text as `messages[0].content`, plus `language` and `voice`.
|
||||
|
||||
Examples:
|
||||
- Prompt format: `conversion/lfm3.py`
|
||||
- Text normalization: `models/templates/Zyphra-ZONOS3-Mini-tts.jinja`
|
||||
- Text to phonemes (G2P): `models/templates/hexgrad-Kokoro-82M-tts.jinja`
|
||||
|
||||
Functions available to the template (see `docs/tts-norm-patterns.md`):
|
||||
- `tts_norm_scan()` / `tts_norm_sub()`: pattern matching for text normalization
|
||||
- `tts_dict()`: lookup in a dictionary stored in the GGUF, such as a pronunciation lexicon
|
||||
|
||||
Add test cases to `tests/test-jinja.cpp`, which also checks them against the python reference with `-py`.
|
||||
|
||||
Rules for phoneme-based models:
|
||||
- C++ only holds the pieces that every such model needs (the functions above). The rules of one model (lexicon handling, stress, suffixes, number spelling, ...) live in its jinja template
|
||||
- The reason is binary size: phoneme logic compiled into C++ is shipped to every user, and most of them never run a TTS model. A template only costs the users of that model
|
||||
- Do not add a model-specific or language-specific G2P step in C++, and do not add a dependency for it (e.g. espeak-ng)
|
||||
|
||||
G2P in jinja has limits, mostly for languages other than English (no lexicon, or rules that need a real morphological analyzer). For now the escape hatch is on the application side: the user runs the text to phonemes step in their own code and sends the phonemes with `raw_prompt` (see the `/tts` endpoint of `llama-server`), which skips the template. Document it in `docs/multimodal/tts.md` when a model relies on it.
|
||||
|
||||
### Checklist for porting new audio generation models to mtmd
|
||||
|
||||
1. Make sure to consult merged PRs about adding new TTS models, especially reviewer comments
|
||||
@@ -116,7 +138,7 @@ Some stages are a full LLM-sized transformer that attends to the keys/values of
|
||||
- In most cases, you can hard-code model configs in the model graph class, or in `mtmd_hparams`
|
||||
- If some values need to be exposed to the `mtmd_helper` layer, hard-code them in `mtmd_helper` and distinguish by pipeline and `mtmd_gen_audio_info::model_variant` if necessary
|
||||
- Do NOT add new GGUF metadata or new fields to `mtmd_gen_audio_info` unless you can prove that you absolutely need them
|
||||
- If the model requires special preprocessing of the input text prompt (prompt format, text normalization), put it in `tokenizer.chat_template.tts` as a jinja program. It receives the text as `messages[0].content`, plus `language` and `voice`. See `conversion/lfm3.py` (prompt format) and `models/templates/Zyphra-ZONOS3-Mini-tts.jinja` (text normalization) as examples. For text normalization, use `tts_norm_scan()` / `tts_norm_sub()`, and `tts_dict()` for dictionaries stored in the GGUF such as a pronunciation lexicon (see `docs/tts-norm-patterns.md`), and add test cases to `tests/test-jinja.cpp`, which also checks them against the python reference with `-py`
|
||||
- If the model requires special preprocessing of the input text prompt, see [Text preprocessing](#text-preprocessing-the-tts-chat-template)
|
||||
4. Make sure most of the changes happen inside `mtmd-helper-gen.cpp`. A good PR looks like this:
|
||||
- 10-20% changes is to add new backbone (text) model and conversion
|
||||
- 60% changes inside `mtmd-helper-gen.cpp`
|
||||
|
||||
@@ -0,0 +1,515 @@
|
||||
#include "models.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <random>
|
||||
#include <stdexcept>
|
||||
|
||||
// Kokoro: StyleTTS2 prosody predictor + iSTFTNet decoder. Tensors keep the names of the reference modules (a.gen.<module>.*)
|
||||
// One utterance runs the 3 graphs of stage_type. The LSTMs, the alignment, the harmonic source and the ISTFT run on the host in between
|
||||
// The graphs keep activations as [T, C], the layout of the conv ops
|
||||
|
||||
static const int KOKORO_N_HIDDEN = 512;
|
||||
static const int KOKORO_N_HARM = 9; // fundamental + 8 overtones
|
||||
static const int KOKORO_UPS = 300; // samples per F0 step: upsample rates 10 * 6, times the ISTFT hop
|
||||
static const int KOKORO_N_FFT = 20;
|
||||
static const int KOKORO_HOP = 5;
|
||||
static const float KOKORO_SR = 24000.0f;
|
||||
|
||||
ggml_tensor * mtmd_graph_kokoro::tensor(const std::string & name) const {
|
||||
const auto it = model.cbx_tensors.find(name);
|
||||
if (it == model.cbx_tensors.end() || !it->second) {
|
||||
throw std::runtime_error("kokoro: missing tensor " + name);
|
||||
}
|
||||
return it->second;
|
||||
}
|
||||
|
||||
bool mtmd_graph_kokoro::has_tensor(const std::string & name) const {
|
||||
return model.cbx_tensors.find(name) != model.cbx_tensors.end();
|
||||
}
|
||||
|
||||
// x [T, IC] -> [T_out, OC], with the bias if the module has one
|
||||
ggml_tensor * mtmd_graph_kokoro::conv(const std::string & name, ggml_tensor * x, int stride, int pad, int dil) const {
|
||||
ggml_tensor * y = ggml_conv_1d(ctx0, tensor(name + ".weight"), x, stride, pad, dil);
|
||||
if (has_tensor(name + ".bias")) {
|
||||
ggml_tensor * b = tensor(name + ".bias");
|
||||
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
||||
}
|
||||
return y;
|
||||
}
|
||||
|
||||
// AdaIN1d: instance norm over time, then scale and shift from the style
|
||||
ggml_tensor * mtmd_graph_kokoro::adain(const std::string & name, ggml_tensor * x, ggml_tensor * style) const {
|
||||
const int64_t n_ch = x->ne[1];
|
||||
ggml_tensor * h = ggml_add(ctx0, ggml_mul_mat(ctx0, tensor(name + ".fc.weight"), style), tensor(name + ".fc.bias"));
|
||||
ggml_tensor * gamma = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, h, n_ch, 0), 1, n_ch);
|
||||
ggml_tensor * beta = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, h, n_ch, n_ch * ggml_element_size(h)), 1, n_ch);
|
||||
ggml_tensor * xn = ggml_norm(ctx0, x, 1e-5f);
|
||||
return ggml_add(ctx0, ggml_add(ctx0, xn, ggml_mul(ctx0, xn, gamma)), beta);
|
||||
}
|
||||
|
||||
// x + sin^2(alpha x) / alpha, alpha per channel
|
||||
ggml_tensor * mtmd_graph_kokoro::snake(const std::string & name, ggml_tensor * x) const {
|
||||
ggml_tensor * alpha = tensor(name);
|
||||
alpha = ggml_reshape_2d(ctx0, alpha, 1, alpha->ne[0]);
|
||||
ggml_tensor * sx = ggml_sin(ctx0, ggml_mul(ctx0, x, alpha));
|
||||
sx = ggml_div(ctx0, ggml_mul(ctx0, sx, sx), alpha);
|
||||
return ggml_add(ctx0, x, sx);
|
||||
}
|
||||
|
||||
// a [T, C] and b [T, C] -> [2T, C] with a on the even steps
|
||||
ggml_tensor * mtmd_graph_kokoro::interleave(ggml_tensor * a, ggml_tensor * b) const {
|
||||
const int64_t n_t = a->ne[0], n_ch = a->ne[1];
|
||||
ggml_tensor * ab = ggml_concat(ctx0, ggml_reshape_3d(ctx0, ggml_cont(ctx0, a), 1, n_t, n_ch),
|
||||
ggml_reshape_3d(ctx0, ggml_cont(ctx0, b), 1, n_t, n_ch), 0);
|
||||
return ggml_reshape_2d(ctx0, ggml_cont(ctx0, ab), 2 * n_t, n_ch);
|
||||
}
|
||||
|
||||
// AdainResBlk1d, upsample doubles the length
|
||||
ggml_tensor * mtmd_graph_kokoro::adain_resblk(const std::string & name, ggml_tensor * x, ggml_tensor * style, bool upsample) const {
|
||||
const int64_t n_t = x->ne[0], n_ch = x->ne[1];
|
||||
|
||||
ggml_tensor * r = adain(name + ".norm1", x, style);
|
||||
r = ggml_leaky_relu(ctx0, r, 0.2f, false);
|
||||
if (upsample) {
|
||||
// depthwise ConvTranspose1d(kernel 3, stride 2, padding 1, output_padding 1), weight stored as [C, 3]:
|
||||
// y[2i] = w1 x[i], y[2i+1] = w2 x[i] + w0 x[i+1]
|
||||
ggml_tensor * w = tensor(name + ".pool.weight");
|
||||
ggml_tensor * b = tensor(name + ".pool.bias");
|
||||
auto tap = [&](int k) {
|
||||
return ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, w, n_ch, (size_t) k * n_ch * ggml_element_size(w)), 1, n_ch);
|
||||
};
|
||||
ggml_tensor * next = ggml_cont(ctx0, ggml_view_2d(ctx0, r, n_t - 1, n_ch, r->nb[1], ggml_element_size(r)));
|
||||
next = ggml_pad(ctx0, next, 1, 0, 0, 0);
|
||||
ggml_tensor * even = ggml_mul(ctx0, r, tap(1));
|
||||
ggml_tensor * odd = ggml_add(ctx0, ggml_mul(ctx0, r, tap(2)), ggml_mul(ctx0, next, tap(0)));
|
||||
r = ggml_add(ctx0, interleave(even, odd), ggml_reshape_2d(ctx0, b, 1, n_ch));
|
||||
}
|
||||
r = conv(name + ".conv1", r, 1, 1, 1);
|
||||
r = adain(name + ".norm2", r, style);
|
||||
r = ggml_leaky_relu(ctx0, r, 0.2f, false);
|
||||
r = conv(name + ".conv2", r, 1, 1, 1);
|
||||
|
||||
ggml_tensor * sc = upsample ? interleave(x, x) : x;
|
||||
if (has_tensor(name + ".conv1x1.weight")) {
|
||||
sc = conv(name + ".conv1x1", sc, 1, 0, 1);
|
||||
}
|
||||
return ggml_scale(ctx0, ggml_add(ctx0, r, sc), (float) M_SQRT1_2);
|
||||
}
|
||||
|
||||
// AdaINResBlock1 of the generator: three units with dilations 1/3/5 on convs1
|
||||
ggml_tensor * mtmd_graph_kokoro::gen_resblock(const std::string & name, ggml_tensor * x, ggml_tensor * style) const {
|
||||
static const int dil[3] = {1, 3, 5};
|
||||
for (int j = 0; j < 3; j++) {
|
||||
const std::string js = std::to_string(j);
|
||||
const int k = (int) tensor(name + ".convs1." + js + ".weight")->ne[0];
|
||||
ggml_tensor * xt = adain(name + ".adain1." + js, x, style);
|
||||
xt = snake(name + ".alpha1." + js, xt);
|
||||
xt = conv(name + ".convs1." + js, xt, 1, (k * dil[j] - dil[j]) / 2, dil[j]);
|
||||
xt = adain(name + ".adain2." + js, xt, style);
|
||||
xt = snake(name + ".alpha2." + js, xt);
|
||||
xt = conv(name + ".convs2." + js, xt, 1, (k - 1) / 2, 1);
|
||||
x = ggml_add(ctx0, xt, x);
|
||||
}
|
||||
return x;
|
||||
}
|
||||
|
||||
ggml_tensor * mtmd_graph_kokoro::input(const char * name, ggml_type type, int64_t ne0, int64_t ne1) const {
|
||||
ggml_tensor * t = ggml_new_tensor_2d(ctx0, type, ne0, ne1);
|
||||
ggml_set_name(t, name);
|
||||
ggml_set_input(t);
|
||||
return t;
|
||||
}
|
||||
|
||||
ggml_cgraph * mtmd_graph_kokoro::output(const char * name, ggml_tensor * t) const {
|
||||
ggml_set_name(t, name);
|
||||
ggml_set_output(t);
|
||||
ggml_build_forward_expand(gf, t);
|
||||
return gf;
|
||||
}
|
||||
|
||||
// tokens -> text encoder CNN output [C, n_tok]
|
||||
ggml_cgraph * mtmd_graph_kokoro::build_text() {
|
||||
const std::string p = "a.gen.text.";
|
||||
ggml_tensor * tokens = input("inp_tokens", GGML_TYPE_I32, n_tok, 1);
|
||||
tokens = ggml_reshape_1d(ctx0, tokens, n_tok);
|
||||
|
||||
ggml_tensor * x = ggml_get_rows(ctx0, tensor(p + "embedding.weight"), tokens); // [C, T]
|
||||
for (int i = 0; has_tensor(p + "cnn." + std::to_string(i) + ".0.weight"); i++) {
|
||||
const std::string q = p + "cnn." + std::to_string(i);
|
||||
const int k = (int) tensor(q + ".0.weight")->ne[0];
|
||||
x = conv(q + ".0", ggml_cont(ctx0, ggml_transpose(ctx0, x)), 1, (k - 1) / 2, 1);
|
||||
// LayerNorm over the channels
|
||||
x = ggml_norm(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, x)), 1e-5f);
|
||||
x = ggml_add(ctx0, ggml_mul(ctx0, x, tensor(q + ".1.gamma")), tensor(q + ".1.beta"));
|
||||
x = ggml_leaky_relu(ctx0, x, 0.2f, false);
|
||||
}
|
||||
return output("out_text", x);
|
||||
}
|
||||
|
||||
// shared LSTM output [C, n_frames] -> F0 and energy curves, 2 steps per frame
|
||||
ggml_cgraph * mtmd_graph_kokoro::build_f0n() {
|
||||
const std::string p = "a.gen.pred.";
|
||||
ggml_tensor * inp = input("inp_x", GGML_TYPE_F32, KOKORO_N_HIDDEN, n_frames);
|
||||
ggml_tensor * style = input("inp_style", GGML_TYPE_F32, n_style, 1);
|
||||
style = ggml_reshape_1d(ctx0, style, n_style);
|
||||
ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
|
||||
|
||||
for (const char * curve : {"F0", "N"}) {
|
||||
ggml_tensor * cur = x;
|
||||
for (int i = 0; i < 3; i++) {
|
||||
const std::string q = p + curve + "." + std::to_string(i);
|
||||
cur = adain_resblk(q, cur, style, has_tensor(q + ".pool.weight"));
|
||||
}
|
||||
cur = conv(p + curve + "_proj", cur, 1, 0, 1);
|
||||
ggml_set_name(cur, curve[0] == 'F' ? "out_f0" : "out_n");
|
||||
ggml_set_output(cur);
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
}
|
||||
return gf;
|
||||
}
|
||||
|
||||
// aligned text features, F0, energy and the source STFT -> conv_post output [n_fft + 2, n_har]
|
||||
ggml_cgraph * mtmd_graph_kokoro::build_dec() {
|
||||
const std::string p = "a.gen.dec.";
|
||||
ggml_tensor * asr = input("inp_asr", GGML_TYPE_F32, KOKORO_N_HIDDEN, n_frames);
|
||||
ggml_tensor * f0 = input("inp_f0", GGML_TYPE_F32, 2 * n_frames, 1);
|
||||
ggml_tensor * en = input("inp_n", GGML_TYPE_F32, 2 * n_frames, 1);
|
||||
ggml_tensor * har = input("inp_har", GGML_TYPE_F32, KOKORO_N_FFT + 2, n_har);
|
||||
ggml_tensor * style = input("inp_style", GGML_TYPE_F32, n_style, 1);
|
||||
style = ggml_reshape_1d(ctx0, style, n_style);
|
||||
asr = ggml_cont(ctx0, ggml_transpose(ctx0, asr));
|
||||
har = ggml_cont(ctx0, ggml_transpose(ctx0, har));
|
||||
|
||||
f0 = conv(p + "F0_conv", f0, 2, 1, 1);
|
||||
en = conv(p + "N_conv", en, 2, 1, 1);
|
||||
|
||||
ggml_tensor * x = ggml_concat(ctx0, ggml_concat(ctx0, asr, f0, 1), en, 1);
|
||||
x = adain_resblk(p + "encode", x, style, false);
|
||||
|
||||
ggml_tensor * asr_res = conv(p + "asr_res.0", asr, 1, 0, 1);
|
||||
for (int i = 0; has_tensor(p + "decode." + std::to_string(i) + ".conv1.weight"); i++) {
|
||||
const std::string q = p + "decode." + std::to_string(i);
|
||||
x = ggml_concat(ctx0, ggml_concat(ctx0, ggml_concat(ctx0, x, asr_res, 1), f0, 1), en, 1);
|
||||
x = adain_resblk(q, x, style, has_tensor(q + ".pool.weight"));
|
||||
}
|
||||
|
||||
const std::string g = p + "generator.";
|
||||
int n_ups = 0;
|
||||
while (has_tensor(g + "ups." + std::to_string(n_ups) + ".weight")) {
|
||||
n_ups++;
|
||||
}
|
||||
for (int i = 0; i < n_ups; i++) {
|
||||
const std::string is = std::to_string(i);
|
||||
ggml_tensor * up_w = tensor(g + "ups." + is + ".weight");
|
||||
ggml_tensor * up_b = tensor(g + "ups." + is + ".bias");
|
||||
// the transposed conv has kernel 2 * stride and torch padding stride / 2
|
||||
const int stride = (int) up_w->ne[0] / 2;
|
||||
const int trim = stride / 2;
|
||||
|
||||
x = ggml_leaky_relu(ctx0, x, 0.1f, false);
|
||||
|
||||
// the source STFT is brought to the rate of this stage
|
||||
ggml_tensor * src_w = tensor(g + "noise_convs." + is + ".weight");
|
||||
const int src_k = (int) src_w->ne[0];
|
||||
const int src_s = src_k > 1 ? src_k / 2 : 1;
|
||||
ggml_tensor * src = conv(g + "noise_convs." + is, har, src_s, src_k > 1 ? (src_s + 1) / 2 : 0, 1);
|
||||
src = gen_resblock(g + "noise_res." + is, src, style);
|
||||
|
||||
x = ggml_conv_transpose_1d(ctx0, up_w, x, stride, 0, 1);
|
||||
x = ggml_cont(ctx0, ggml_view_2d(ctx0, x, x->ne[0] - 2 * trim, x->ne[1], x->nb[1], (size_t) trim * ggml_element_size(x)));
|
||||
x = ggml_add(ctx0, x, ggml_reshape_2d(ctx0, up_b, 1, up_b->ne[0]));
|
||||
if (i == n_ups - 1) {
|
||||
x = ggml_pad_reflect_1d(ctx0, x, 1, 0);
|
||||
}
|
||||
x = ggml_add(ctx0, x, src);
|
||||
|
||||
ggml_tensor * acc = nullptr;
|
||||
for (int j = 0; j < 3; j++) {
|
||||
ggml_tensor * r = gen_resblock(g + "resblocks." + std::to_string(i * 3 + j), x, style);
|
||||
acc = acc ? ggml_add(ctx0, acc, r) : r;
|
||||
}
|
||||
x = ggml_scale(ctx0, acc, 1.0f / 3.0f);
|
||||
}
|
||||
x = ggml_leaky_relu(ctx0, x, 0.01f, false);
|
||||
x = conv(g + "conv_post", x, 1, 3, 1);
|
||||
return output("out_spec", ggml_cont(ctx0, ggml_transpose(ctx0, x)));
|
||||
}
|
||||
|
||||
ggml_cgraph * mtmd_graph_kokoro::build() {
|
||||
switch (stage) {
|
||||
case STAGE_TEXT: return build_text();
|
||||
case STAGE_F0N: return build_f0n();
|
||||
case STAGE_DEC: return build_dec();
|
||||
}
|
||||
GGML_ABORT("invalid kokoro stage");
|
||||
}
|
||||
|
||||
//
|
||||
// host side
|
||||
//
|
||||
|
||||
static std::vector<float> kokoro_read(const mtmd_model & model, const std::string & name, size_t n_expect) {
|
||||
std::vector<float> v(n_expect);
|
||||
if (model.cbx_read_tensor(name.c_str(), v.data(), v.size()) != n_expect) {
|
||||
throw std::runtime_error("kokoro: unexpected tensor " + name);
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
static inline float kokoro_sigmoid(float x) {
|
||||
return 1.0f / (1.0f + expf(-x));
|
||||
}
|
||||
|
||||
// y = W x, W row-major [n_out, n_in]
|
||||
static void kokoro_matvec(const float * w, const float * x, int n_out, int n_in, float * y) {
|
||||
for (int o = 0; o < n_out; o++) {
|
||||
const float * row = w + (size_t) o * n_in;
|
||||
float acc[8] = {0.0f};
|
||||
int i = 0;
|
||||
for (; i + 8 <= n_in; i += 8) {
|
||||
for (int k = 0; k < 8; k++) {
|
||||
acc[k] += row[i + k] * x[i + k];
|
||||
}
|
||||
}
|
||||
float sum = 0.0f;
|
||||
for (; i < n_in; i++) {
|
||||
sum += row[i] * x[i];
|
||||
}
|
||||
for (int k = 0; k < 8; k++) {
|
||||
sum += acc[k];
|
||||
}
|
||||
y[o] = sum;
|
||||
}
|
||||
}
|
||||
|
||||
// bidirectional single-layer LSTM (torch layout, gates i f g o): x [n_t, n_in] -> y [n_t, 2 * n_h]
|
||||
static void kokoro_lstm(const mtmd_model & model, const std::string & name, const std::vector<float> & x, int n_t, int n_in,
|
||||
std::vector<float> & y) {
|
||||
const int n_h = KOKORO_N_HIDDEN / 2;
|
||||
y.assign((size_t) n_t * 2 * n_h, 0.0f);
|
||||
std::vector<float> gi(4 * n_h), gh(4 * n_h), h(n_h), c(n_h);
|
||||
for (int dir = 0; dir < 2; dir++) {
|
||||
const std::string sfx = dir == 0 ? "" : "_reverse";
|
||||
const auto w_ih = kokoro_read(model, name + ".weight_ih_l0" + sfx, (size_t) 4 * n_h * n_in);
|
||||
const auto w_hh = kokoro_read(model, name + ".weight_hh_l0" + sfx, (size_t) 4 * n_h * n_h);
|
||||
const auto bias = kokoro_read(model, name + ".bias_l0" + sfx, (size_t) 4 * n_h);
|
||||
std::fill(h.begin(), h.end(), 0.0f);
|
||||
std::fill(c.begin(), c.end(), 0.0f);
|
||||
for (int s = 0; s < n_t; s++) {
|
||||
const int t = dir == 0 ? s : n_t - 1 - s;
|
||||
kokoro_matvec(w_ih.data(), x.data() + (size_t) t * n_in, 4 * n_h, n_in, gi.data());
|
||||
kokoro_matvec(w_hh.data(), h.data(), 4 * n_h, n_h, gh.data());
|
||||
float * out = y.data() + (size_t) t * 2 * n_h + (size_t) dir * n_h;
|
||||
for (int j = 0; j < n_h; j++) {
|
||||
const float ig = kokoro_sigmoid(gi[j] + gh[j] + bias[j]);
|
||||
const float fg = kokoro_sigmoid(gi[n_h + j] + gh[n_h + j] + bias[n_h + j]);
|
||||
const float gg = tanhf(gi[2 * n_h + j] + gh[2 * n_h + j] + bias[2 * n_h + j]);
|
||||
const float og = kokoro_sigmoid(gi[3 * n_h + j] + gh[3 * n_h + j] + bias[3 * n_h + j]);
|
||||
c[j] = fg * c[j] + ig * gg;
|
||||
h[j] = og * tanhf(c[j]);
|
||||
out[j] = h[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rows of x repeated by their duration: [n_tok, n_ch] -> [n_frames, n_ch]
|
||||
static std::vector<float> kokoro_align(const std::vector<float> & x, const std::vector<int32_t> & dur, int n_ch) {
|
||||
std::vector<float> y;
|
||||
for (size_t t = 0; t < dur.size(); t++) {
|
||||
for (int32_t r = 0; r < dur[t]; r++) {
|
||||
y.insert(y.end(), x.begin() + t * n_ch, x.begin() + (t + 1) * n_ch);
|
||||
}
|
||||
}
|
||||
return y;
|
||||
}
|
||||
|
||||
int mtmd_graph_kokoro::prosody(const mtmd_model & model, const float * bert, int n_embd, const float * text_cnn, int n_tok,
|
||||
const float * style, float speed, std::vector<float> & shared, std::vector<float> & asr) {
|
||||
const int n_h = KOKORO_N_HIDDEN;
|
||||
const int n_cat = n_h + n_style;
|
||||
|
||||
// bert_encoder, then each row is followed by the style
|
||||
const auto enc_w = kokoro_read(model, "a.gen.bert_enc.weight", (size_t) n_h * n_embd);
|
||||
const auto enc_b = kokoro_read(model, "a.gen.bert_enc.bias", n_h);
|
||||
std::vector<float> x((size_t) n_tok * n_cat);
|
||||
for (int t = 0; t < n_tok; t++) {
|
||||
float * row = x.data() + (size_t) t * n_cat;
|
||||
kokoro_matvec(enc_w.data(), bert + (size_t) t * n_embd, n_h, n_embd, row);
|
||||
for (int j = 0; j < n_h; j++) {
|
||||
row[j] += enc_b[j];
|
||||
}
|
||||
memcpy(row + n_h, style, n_style * sizeof(float));
|
||||
}
|
||||
|
||||
// DurationEncoder: LSTM, AdaLayerNorm, style again
|
||||
std::vector<float> y;
|
||||
std::vector<float> gamma_beta(2 * n_h);
|
||||
for (int il = 0; model.cbx_tensors.count(mtmd_format("a.gen.pred.text_encoder.lstms.%d.weight_ih_l0", 2 * il)); il++) {
|
||||
kokoro_lstm(model, mtmd_format("a.gen.pred.text_encoder.lstms.%d", 2 * il), x, n_tok, n_cat, y);
|
||||
const std::string norm = mtmd_format("a.gen.pred.text_encoder.lstms.%d.fc", 2 * il + 1);
|
||||
const auto fc_w = kokoro_read(model, norm + ".weight", (size_t) 2 * n_h * n_style);
|
||||
const auto fc_b = kokoro_read(model, norm + ".bias", (size_t) 2 * n_h);
|
||||
kokoro_matvec(fc_w.data(), style, 2 * n_h, n_style, gamma_beta.data());
|
||||
for (int t = 0; t < n_tok; t++) {
|
||||
const float * src = y.data() + (size_t) t * n_h;
|
||||
float * dst = x.data() + (size_t) t * n_cat;
|
||||
double mean = 0.0, var = 0.0;
|
||||
for (int j = 0; j < n_h; j++) {
|
||||
mean += src[j];
|
||||
}
|
||||
mean /= n_h;
|
||||
for (int j = 0; j < n_h; j++) {
|
||||
var += (src[j] - mean) * (src[j] - mean);
|
||||
}
|
||||
const double scale = 1.0 / sqrt(var / n_h + 1e-5);
|
||||
for (int j = 0; j < n_h; j++) {
|
||||
const float v = (float) ((src[j] - mean) * scale);
|
||||
dst[j] = (1.0f + gamma_beta[j] + fc_b[j]) * v + gamma_beta[n_h + j] + fc_b[n_h + j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// durations: frames per token
|
||||
kokoro_lstm(model, "a.gen.pred.lstm", x, n_tok, n_cat, y);
|
||||
const ggml_tensor * proj = model.cbx_tensors.at("a.gen.pred.duration_proj.linear_layer.weight");
|
||||
const int n_bins = (int) proj->ne[1];
|
||||
const auto proj_w = kokoro_read(model, "a.gen.pred.duration_proj.linear_layer.weight", (size_t) n_bins * n_h);
|
||||
const auto proj_b = kokoro_read(model, "a.gen.pred.duration_proj.linear_layer.bias", n_bins);
|
||||
std::vector<int32_t> dur(n_tok);
|
||||
std::vector<float> logits(n_bins);
|
||||
int n_frames = 0;
|
||||
for (int t = 0; t < n_tok; t++) {
|
||||
kokoro_matvec(proj_w.data(), y.data() + (size_t) t * n_h, n_bins, n_h, logits.data());
|
||||
float sum = 0.0f;
|
||||
for (int j = 0; j < n_bins; j++) {
|
||||
sum += kokoro_sigmoid(logits[j] + proj_b[j]);
|
||||
}
|
||||
// round half to even, like torch.round
|
||||
dur[t] = std::max(1, (int) nearbyintf(sum / speed));
|
||||
n_frames += dur[t];
|
||||
}
|
||||
|
||||
kokoro_lstm(model, "a.gen.pred.shared", kokoro_align(x, dur, n_cat), n_frames, n_cat, shared);
|
||||
|
||||
// text encoder: its LSTM runs on the CNN output
|
||||
kokoro_lstm(model, "a.gen.text.lstm", std::vector<float>(text_cnn, text_cnn + (size_t) n_tok * n_h), n_tok, n_h, y);
|
||||
asr = kokoro_align(y, dur, n_h);
|
||||
return n_frames;
|
||||
}
|
||||
|
||||
// hn-nsf source: f0 [2 * n_frames] -> harmonic source, then its STFT as [n_har, 22] rows of magnitudes and phases
|
||||
int mtmd_graph_kokoro::har_source(const mtmd_model & model, const std::vector<float> & f0, std::mt19937 & rng, std::vector<float> & har) {
|
||||
const int n_f0 = (int) f0.size();
|
||||
const int64_t n_wav = (int64_t) n_f0 * KOKORO_UPS;
|
||||
const auto lin_w = kokoro_read(model, "a.gen.dec.generator.m_source.l_linear.weight", KOKORO_N_HARM);
|
||||
const auto lin_b = kokoro_read(model, "a.gen.dec.generator.m_source.l_linear.bias", 1);
|
||||
|
||||
// SineGen: the phase adds up at the F0 rate, then it is interpolated to the sample rate
|
||||
// the reference also adds a random start phase, but its own downsampling drops it
|
||||
std::vector<float> phase((size_t) n_f0 * KOKORO_N_HARM);
|
||||
for (int h = 0; h < KOKORO_N_HARM; h++) {
|
||||
float cum = 0.0f;
|
||||
for (int i = 0; i < n_f0; i++) {
|
||||
float rad = f0[i] * (float) (h + 1) / KOKORO_SR;
|
||||
rad -= floorf(rad);
|
||||
cum += rad;
|
||||
phase[(size_t) i * KOKORO_N_HARM + h] = cum * 2.0f * (float) M_PI * KOKORO_UPS;
|
||||
}
|
||||
}
|
||||
std::normal_distribution<float> gauss(0.0f, 1.0f);
|
||||
std::vector<float> src((size_t) n_wav);
|
||||
for (int64_t t = 0; t < n_wav; t++) {
|
||||
const float pos = std::max(0.0f, ((float) t + 0.5f) / KOKORO_UPS - 0.5f);
|
||||
const int i0 = std::min((int) pos, n_f0 - 1);
|
||||
const int i1 = std::min(i0 + 1, n_f0 - 1);
|
||||
const float frac = pos - (float) i0;
|
||||
const float uv = f0[t / KOKORO_UPS] > 10.0f ? 1.0f : 0.0f; // voiced threshold
|
||||
const float namp = uv * 0.003f + (1.0f - uv) * 0.1f / 3.0f;
|
||||
float merged = lin_b[0];
|
||||
for (int h = 0; h < KOKORO_N_HARM; h++) {
|
||||
const float ph = (1.0f - frac) * phase[(size_t) i0 * KOKORO_N_HARM + h] + frac * phase[(size_t) i1 * KOKORO_N_HARM + h];
|
||||
merged += lin_w[h] * (0.1f * sinf(ph) * uv + namp * gauss(rng));
|
||||
}
|
||||
src[(size_t) t] = tanhf(merged);
|
||||
}
|
||||
|
||||
// STFT: periodic hann window, centered with reflect padding
|
||||
const int n_bins = KOKORO_N_FFT / 2 + 1;
|
||||
const int n_har = (int) (n_wav / KOKORO_HOP) + 1;
|
||||
float win[KOKORO_N_FFT], tab_cos[KOKORO_N_FFT], tab_sin[KOKORO_N_FFT];
|
||||
for (int i = 0; i < KOKORO_N_FFT; i++) {
|
||||
win[i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
|
||||
tab_cos[i] = cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
|
||||
tab_sin[i] = sinf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
|
||||
}
|
||||
har.resize((size_t) n_har * 2 * n_bins);
|
||||
for (int fr = 0; fr < n_har; fr++) {
|
||||
float frame[KOKORO_N_FFT];
|
||||
for (int i = 0; i < KOKORO_N_FFT; i++) {
|
||||
int64_t idx = (int64_t) fr * KOKORO_HOP - KOKORO_N_FFT / 2 + i;
|
||||
if (idx < 0) {
|
||||
idx = -idx;
|
||||
}
|
||||
if (idx >= n_wav) {
|
||||
idx = 2 * (n_wav - 1) - idx;
|
||||
}
|
||||
frame[i] = src[(size_t) idx] * win[i];
|
||||
}
|
||||
float * out = har.data() + (size_t) fr * 2 * n_bins;
|
||||
for (int k = 0; k < n_bins; k++) {
|
||||
float re = 0.0f, im = 0.0f;
|
||||
for (int i = 0; i < KOKORO_N_FFT; i++) {
|
||||
const int a = (k * i) % KOKORO_N_FFT;
|
||||
re += frame[i] * tab_cos[a];
|
||||
im -= frame[i] * tab_sin[a];
|
||||
}
|
||||
out[k] = sqrtf(re * re + im * im);
|
||||
out[n_bins + k] = atan2f(im, re);
|
||||
}
|
||||
}
|
||||
return n_har;
|
||||
}
|
||||
|
||||
// conv_post rows [n_frames, 22] -> waveform: magnitude exp(x), phase sin(x), hann overlap-add
|
||||
void mtmd_graph_kokoro::istft(const std::vector<float> & spec, std::vector<float> & audio) {
|
||||
const int n_bins = KOKORO_N_FFT / 2 + 1;
|
||||
const int n_frames = (int) (spec.size() / (2 * n_bins));
|
||||
const int64_t n_out = (int64_t) (n_frames - 1) * KOKORO_HOP;
|
||||
float win[KOKORO_N_FFT], tab_cos[KOKORO_N_FFT], tab_sin[KOKORO_N_FFT];
|
||||
for (int i = 0; i < KOKORO_N_FFT; i++) {
|
||||
win[i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
|
||||
tab_cos[i] = cosf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
|
||||
tab_sin[i] = sinf(2.0f * (float) M_PI * i / KOKORO_N_FFT);
|
||||
}
|
||||
std::vector<float> acc((size_t) n_out + KOKORO_N_FFT, 0.0f);
|
||||
std::vector<float> wsum((size_t) n_out + KOKORO_N_FFT, 0.0f);
|
||||
for (int fr = 0; fr < n_frames; fr++) {
|
||||
const float * row = spec.data() + (size_t) fr * 2 * n_bins;
|
||||
float re[KOKORO_N_FFT / 2 + 1], im[KOKORO_N_FFT / 2 + 1];
|
||||
for (int k = 0; k < n_bins; k++) {
|
||||
const float mag = expf(row[k]);
|
||||
const float ph = sinf(row[n_bins + k]);
|
||||
re[k] = mag * cosf(ph);
|
||||
im[k] = mag * sinf(ph);
|
||||
}
|
||||
for (int i = 0; i < KOKORO_N_FFT; i++) {
|
||||
float v = 0.0f;
|
||||
for (int k = 0; k < n_bins; k++) {
|
||||
const int a = (k * i) % KOKORO_N_FFT;
|
||||
const float w = (k == 0 || k == KOKORO_N_FFT / 2) ? 1.0f : 2.0f;
|
||||
v += w * (re[k] * tab_cos[a] - im[k] * tab_sin[a]);
|
||||
}
|
||||
const size_t o = (size_t) fr * KOKORO_HOP + i;
|
||||
acc[o] += v / KOKORO_N_FFT * win[i];
|
||||
wsum[o] += win[i] * win[i];
|
||||
}
|
||||
}
|
||||
audio.resize((size_t) std::max<int64_t>(n_out, 0));
|
||||
for (size_t i = 0; i < audio.size(); i++) {
|
||||
const size_t j = i + KOKORO_N_FFT / 2; // drop the centering pad
|
||||
audio[i] = wsum[j] > 1e-11f ? acc[j] / wsum[j] : 0.0f;
|
||||
}
|
||||
}
|
||||
@@ -258,6 +258,47 @@ struct mtmd_graph_chatterbox : mtmd_graph_chatterbox_base {
|
||||
ggml_cgraph * build_vocoder(int n_mel, int n_stft);
|
||||
};
|
||||
|
||||
struct mtmd_graph_kokoro : mtmd_graph {
|
||||
enum stage_type {
|
||||
STAGE_TEXT, // tokens -> text encoder CNN
|
||||
STAGE_F0N, // shared LSTM output -> F0 and energy curves
|
||||
STAGE_DEC, // aligned features + source STFT -> STFT coefficients
|
||||
};
|
||||
static constexpr int n_style = 128; // a voice row is the decoder style, then the predictor style
|
||||
|
||||
stage_type stage;
|
||||
int n_tok = 0;
|
||||
int n_frames = 0;
|
||||
int n_har = 0;
|
||||
mtmd_graph_kokoro(mtmd_subcontext * ctx, const mtmd_image_f32 & img, stage_type stage, int n_tok, int n_frames, int n_har)
|
||||
: mtmd_graph(ctx, img), stage(stage), n_tok(n_tok), n_frames(n_frames), n_har(n_har) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
// host side, between the graphs
|
||||
// bert [n_tok, n_embd], text_cnn [n_tok, 512], style is the predictor style
|
||||
// fills shared and asr as [n_frames, 512], returns n_frames
|
||||
static int prosody(const mtmd_model & model, const float * bert, int n_embd, const float * text_cnn, int n_tok,
|
||||
const float * style, float speed, std::vector<float> & shared, std::vector<float> & asr);
|
||||
// fills har as [n_har, 22], returns n_har
|
||||
static int har_source(const mtmd_model & model, const std::vector<float> & f0, std::mt19937 & rng, std::vector<float> & har);
|
||||
static void istft(const std::vector<float> & spec, std::vector<float> & audio);
|
||||
|
||||
private:
|
||||
ggml_tensor * tensor(const std::string & name) const;
|
||||
bool has_tensor(const std::string & name) const;
|
||||
ggml_tensor * conv(const std::string & name, ggml_tensor * x, int stride, int pad, int dil) const;
|
||||
ggml_tensor * adain(const std::string & name, ggml_tensor * x, ggml_tensor * style) const;
|
||||
ggml_tensor * snake(const std::string & name, ggml_tensor * x) const;
|
||||
ggml_tensor * interleave(ggml_tensor * a, ggml_tensor * b) const;
|
||||
ggml_tensor * adain_resblk(const std::string & name, ggml_tensor * x, ggml_tensor * style, bool upsample) const;
|
||||
ggml_tensor * gen_resblock(const std::string & name, ggml_tensor * x, ggml_tensor * style) const;
|
||||
ggml_tensor * input(const char * name, ggml_type type, int64_t ne0, int64_t ne1) const;
|
||||
ggml_cgraph * output(const char * name, ggml_tensor * t) const;
|
||||
ggml_cgraph * build_text();
|
||||
ggml_cgraph * build_f0n();
|
||||
ggml_cgraph * build_dec();
|
||||
};
|
||||
|
||||
struct mtmd_graph_lfm3a : mtmd_graph {
|
||||
mtmd_graph_lfm3a(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
@@ -390,6 +431,15 @@ struct mtmd_graph_qwen3tts_gen : mtmd_graph {
|
||||
: mtmd_graph(ctx, img), gen_process(gen_process), top_k(top_k), top_p(top_p) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
// MTMD_SUBCTX_GEN_PROCESS_TOKENIZE: reference clip -> codes and their talker input rows
|
||||
ggml_cgraph * build_ref_encoder();
|
||||
ggml_tensor * nearest_code(ggml_tensor * x, ggml_tensor * codebook) const;
|
||||
|
||||
// reference encoder transformer, see encoder_config in speech_tokenizer/config.json
|
||||
static constexpr int ref_enc_n_head = 8;
|
||||
static constexpr float ref_enc_eps = 1e-5f;
|
||||
static constexpr float ref_enc_rope_theta = 10000.0f;
|
||||
|
||||
// which sub-graph build() constructs, fixed at graph-build time
|
||||
mtmd_subctx_gen_process gen_process;
|
||||
|
||||
@@ -483,6 +533,11 @@ struct mtmd_graph_pockettts_seanet : mtmd_graph {
|
||||
mtmd_graph_pockettts_seanet(const mtmd_graph & parent) : mtmd_graph(parent) {}
|
||||
ggml_cgraph * build() override { GGML_ABORT("call encode()/decode() instead"); }
|
||||
|
||||
// qwen3tts reference codes: the output is quantized, so the im2col is F32 (the kernels must be F32 too)
|
||||
bool f32_im2col = false;
|
||||
// qwen3tts reference codes: pad_replicate also repeats the last frame on the right, like MimiConv1d
|
||||
bool pad_replicate_end = false;
|
||||
|
||||
// per-call streaming state, keyed by slot name (see list_pockettts_state_slots)
|
||||
std::map<std::string, ggml_tensor *> state_in;
|
||||
mutable std::vector<std::pair<std::string, ggml_tensor *>> state_out;
|
||||
@@ -501,11 +556,22 @@ struct mtmd_graph_pockettts_seanet : mtmd_graph {
|
||||
};
|
||||
|
||||
// mimi encoder + speaker_proj: reference waveform -> voice conditioning rows
|
||||
// the qwen3tts reference encoder reuses encode() with its own transformer options
|
||||
struct mtmd_graph_pockettts_spkenc : mtmd_graph {
|
||||
mtmd_graph_pockettts_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
|
||||
mtmd_graph_pockettts_spkenc(const mtmd_graph & parent) : mtmd_graph(parent) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
int tfm_n_head = n_head;
|
||||
float tfm_eps = eps;
|
||||
float tfm_rope_theta = hparams.rope_theta;
|
||||
int tfm_rope_type = GGML_ROPE_TYPE_NORMAL;
|
||||
ffn_op_type tfm_ffn_type = FFN_GELU;
|
||||
|
||||
ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const;
|
||||
|
||||
// waveform [n_samples, 1] -> [n_frames, C], at the frame rate after model.downsample_w
|
||||
ggml_tensor * encode(const mtmd_graph_pockettts_seanet & seanet, ggml_tensor * x, const std::vector<mtmd_layer> & layers);
|
||||
};
|
||||
|
||||
//
|
||||
|
||||
@@ -33,12 +33,26 @@ ggml_tensor * mtmd_graph_pockettts_seanet::conv1d(ggml_tensor * x, ggml_tensor *
|
||||
ggml_tensor * first = ggml_view_2d(ctx0, x, 1, x->ne[1], x->nb[1], 0);
|
||||
ggml_tensor * left = ggml_repeat_4d(ctx0, first, p_total, x->ne[1], 1, 1);
|
||||
x = ggml_concat(ctx0, left, x, 0);
|
||||
x = ggml_pad_ext(ctx0, x, 0, p_extra, 0, 0, 0, 0, 0, 0);
|
||||
if (pad_replicate_end && p_extra > 0) {
|
||||
ggml_tensor * last = ggml_view_2d(ctx0, x, 1, x->ne[1], x->nb[1], (size_t) (x->ne[0] - 1) * x->nb[0]);
|
||||
ggml_tensor * right = ggml_repeat_4d(ctx0, last, p_extra, x->ne[1], 1, 1);
|
||||
x = ggml_concat(ctx0, x, right, 0);
|
||||
} else {
|
||||
x = ggml_pad_ext(ctx0, x, 0, p_extra, 0, 0, 0, 0, 0, 0);
|
||||
}
|
||||
} else {
|
||||
x = ggml_pad_ext(ctx0, x, p_total, p_extra, 0, 0, 0, 0, 0, 0);
|
||||
}
|
||||
|
||||
ggml_tensor * y = ggml_conv_1d(ctx0, w, x, stride, 0, dilation);
|
||||
ggml_tensor * y;
|
||||
if (f32_im2col) {
|
||||
// ggml_conv_1d() with an F32 im2col, the kernel must be F32
|
||||
ggml_tensor * col = ggml_im2col(ctx0, w, x, stride, 0, 0, 0, dilation, 0, false, GGML_TYPE_F32);
|
||||
y = ggml_mul_mat(ctx0, ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[2] * col->ne[1]),
|
||||
ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2]));
|
||||
} else {
|
||||
y = ggml_conv_1d(ctx0, w, x, stride, 0, dilation);
|
||||
}
|
||||
y = ggml_reshape_2d(ctx0, y, y->ne[0], y->ne[1]);
|
||||
if (b) {
|
||||
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
||||
|
||||
@@ -5,44 +5,41 @@
|
||||
|
||||
// pre-norm block with layer scale on both residual paths, see mimi_transformer.py
|
||||
ggml_tensor * mtmd_graph_pockettts_spkenc::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, ggml_tensor * inp_pos, ggml_tensor * kq_mask, int il) const {
|
||||
const int64_t n_pos = cur->ne[1];
|
||||
const int tfm_d_head = (int) cur->ne[0] / tfm_n_head;
|
||||
|
||||
ggml_tensor * inp = cur;
|
||||
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, tfm_eps, il);
|
||||
|
||||
ggml_tensor * Qcur = build_mm(layer.q_w, cur);
|
||||
ggml_tensor * Kcur = build_mm(layer.k_w, cur);
|
||||
ggml_tensor * Vcur = build_mm(layer.v_w, cur);
|
||||
|
||||
const int64_t n_pos = cur->ne[1];
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);
|
||||
Qcur = ggml_reshape_3d(ctx0, Qcur, tfm_d_head, tfm_n_head, n_pos);
|
||||
Kcur = ggml_reshape_3d(ctx0, Kcur, tfm_d_head, tfm_n_head, n_pos);
|
||||
Vcur = ggml_reshape_3d(ctx0, Vcur, tfm_d_head, tfm_n_head, n_pos);
|
||||
|
||||
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
|
||||
hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, d_head, GGML_ROPE_TYPE_NORMAL, 0,
|
||||
hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, tfm_d_head, tfm_rope_type, 0,
|
||||
tfm_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, tfm_d_head, tfm_rope_type, 0,
|
||||
tfm_rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
|
||||
cur = build_attn(layer.o_w, nullptr, Qcur, Kcur, Vcur, kq_mask, kq_scale, il);
|
||||
cur = build_attn(layer.o_w, nullptr, Qcur, Kcur, Vcur, kq_mask, 1.0f / sqrtf((float) tfm_d_head), il);
|
||||
cur = ggml_mul(ctx0, cur, layer.ls_1_w);
|
||||
cur = ggml_add(ctx0, cur, inp);
|
||||
|
||||
inp = cur;
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);
|
||||
cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, FFN_GELU, il);
|
||||
cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, tfm_eps, il);
|
||||
cur = build_ffn(cur, layer.ff_up_w, nullptr, nullptr, nullptr, layer.ff_down_w, nullptr, tfm_ffn_type, il);
|
||||
cur = ggml_mul(ctx0, cur, layer.ls_2_w);
|
||||
cur = ggml_add(ctx0, cur, inp);
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_cgraph * mtmd_graph_pockettts_spkenc::build() {
|
||||
// the preprocessor hands over the waveform as a single-row "mel", already [n_samples, 1]
|
||||
ggml_tensor * inp_raw = build_inp_raw(1);
|
||||
ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]);
|
||||
|
||||
mtmd_graph_pockettts_seanet seanet(*this);
|
||||
cur = seanet.encode(cur);
|
||||
ggml_tensor * mtmd_graph_pockettts_spkenc::encode(const mtmd_graph_pockettts_seanet & seanet, ggml_tensor * x, const std::vector<mtmd_layer> & layers) {
|
||||
ggml_tensor * cur = seanet.encode(x);
|
||||
cb(cur, "mimi_enc", -1);
|
||||
|
||||
// [T, 512] -> transformer works on [512, T]
|
||||
@@ -55,16 +52,27 @@ ggml_cgraph * mtmd_graph_pockettts_spkenc::build() {
|
||||
// the mimi transformer is causal with a sliding window, see _build_attention_mask()
|
||||
ggml_tensor * kq_mask = build_inp_attn_mask(cur->ne[1], cur->ne[1]);
|
||||
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
cur = tfm_layer_forward(cur, model.layers[il], inp_pos, kq_mask, il);
|
||||
for (size_t il = 0; il < layers.size(); il++) {
|
||||
cur = tfm_layer_forward(cur, layers[il], inp_pos, kq_mask, (int) il);
|
||||
}
|
||||
cb(cur, "mimi_enc_tfm", -1);
|
||||
|
||||
// downsample to the model frame rate, [512, T] -> [T, 512] -> [T / 16, 32]
|
||||
// downsample to the model frame rate, [512, T] -> [T, 512] -> [T / mimi_downsample, C]
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = seanet.conv1d(cur, model.downsample_w, nullptr, hparams.mimi_downsample, 1, true);
|
||||
cb(cur, "mimi_downsample", -1);
|
||||
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_cgraph * mtmd_graph_pockettts_spkenc::build() {
|
||||
// the preprocessor hands over the waveform as a single-row "mel", already [n_samples, 1]
|
||||
ggml_tensor * inp_raw = build_inp_raw(1);
|
||||
ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]);
|
||||
|
||||
mtmd_graph_pockettts_seanet seanet(*this);
|
||||
cur = encode(seanet, cur, model.layers);
|
||||
|
||||
// voice latent -> backbone embd
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
|
||||
cur = build_mm(model.spk_proj_w, cur);
|
||||
|
||||
@@ -664,9 +664,77 @@ std::vector<c2w_state_slot> list_c2w_state_slots(const mtmd_hparams & hparams, c
|
||||
return slots;
|
||||
}
|
||||
|
||||
// x: [dim, T], codebook: [dim, n_entries] -> [T] index of the nearest entry
|
||||
ggml_tensor * mtmd_graph_qwen3tts_gen::nearest_code(ggml_tensor * x, ggml_tensor * codebook) const {
|
||||
// argmin |x - e|^2 = argmax 2 x.e - |e|^2
|
||||
ggml_tensor * dots = ggml_mul_mat(ctx0, codebook, x); // [n_entries, T]
|
||||
ggml_tensor * norms = ggml_sum_rows(ctx0, ggml_sqr(ctx0, codebook)); // [1, n_entries]
|
||||
norms = ggml_reshape_2d(ctx0, norms, norms->ne[1], 1); // [n_entries, 1]
|
||||
return ggml_argmax(ctx0, ggml_sub(ctx0, ggml_scale(ctx0, dots, 2.0f), norms));
|
||||
}
|
||||
|
||||
// the mimi encoder of the speech tokenizer, then its RVQ: one semantic codebook and n_acoustic acoustic ones
|
||||
ggml_cgraph * mtmd_graph_qwen3tts_gen::build_ref_encoder() {
|
||||
// the waveform comes in as a single-row "mel", [n_samples, 1]
|
||||
ggml_tensor * inp_raw = build_inp_raw(1);
|
||||
ggml_tensor * cur = ggml_reshape_2d(ctx0, inp_raw, inp_raw->ne[0], inp_raw->ne[1]);
|
||||
|
||||
mtmd_graph_pockettts_seanet seanet(*this);
|
||||
seanet.f32_im2col = true;
|
||||
seanet.pad_replicate_end = true;
|
||||
|
||||
// the checkpoint is in the transformers layout, so rope rotates halves (neox)
|
||||
mtmd_graph_pockettts_spkenc mimi(*this);
|
||||
mimi.tfm_n_head = ref_enc_n_head;
|
||||
mimi.tfm_eps = ref_enc_eps;
|
||||
mimi.tfm_rope_theta = ref_enc_rope_theta;
|
||||
mimi.tfm_rope_type = GGML_ROPE_TYPE_NEOX;
|
||||
mimi.tfm_ffn_type = FFN_GELU_ERF;
|
||||
|
||||
cur = mimi.encode(seanet, cur, model.ref_enc_layers);
|
||||
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); // [512, T]
|
||||
|
||||
std::vector<ggml_tensor *> codes;
|
||||
codes.push_back(nearest_code(build_mm(model.ref_quant_first_in_w, cur), model.ref_quant_first_cb));
|
||||
|
||||
const int64_t n_acoustic = model.ref_quant_rest_cb->ne[2];
|
||||
ggml_tensor * residual = build_mm(model.ref_quant_rest_in_w, cur);
|
||||
for (int64_t i = 0; i < n_acoustic; i++) {
|
||||
ggml_tensor * codebook = ggml_view_2d(ctx0, model.ref_quant_rest_cb, model.ref_quant_rest_cb->ne[0], model.ref_quant_rest_cb->ne[1],
|
||||
model.ref_quant_rest_cb->nb[1], (size_t) i * model.ref_quant_rest_cb->nb[2]);
|
||||
ggml_tensor * code = nearest_code(residual, codebook);
|
||||
codes.push_back(code);
|
||||
if (i + 1 < n_acoustic) {
|
||||
residual = ggml_sub(ctx0, residual, ggml_get_rows(ctx0, codebook, code));
|
||||
}
|
||||
}
|
||||
|
||||
// talker input rows of the reference frames: codebook 0 from the talker table, the rest from the code predictor's
|
||||
ggml_tensor * rows = ggml_get_rows(ctx0, model.gen_code_out_embd_w, codes[0]);
|
||||
for (int64_t g = 1; g <= n_acoustic; g++) {
|
||||
ggml_tensor * embd = ggml_view_2d(ctx0, model.gen_code_embd_w, model.gen_code_embd_w->ne[0], model.gen_code_embd_w->ne[1],
|
||||
model.gen_code_embd_w->nb[1], (size_t) (g - 1) * model.gen_code_embd_w->nb[2]);
|
||||
rows = ggml_add(ctx0, rows, ggml_get_rows(ctx0, embd, codes[(size_t) g]));
|
||||
}
|
||||
|
||||
for (size_t g = 0; g < codes.size(); g++) {
|
||||
ggml_set_name(codes[g], ("out_ref_code_" + std::to_string(g)).c_str());
|
||||
ggml_set_output(codes[g]);
|
||||
ggml_build_forward_expand(gf, codes[g]);
|
||||
}
|
||||
ggml_set_name(rows, "out_ref_rows");
|
||||
ggml_set_output(rows);
|
||||
ggml_build_forward_expand(gf, rows);
|
||||
return gf;
|
||||
}
|
||||
|
||||
// both sub-graphs are always built, so the topology stays constant
|
||||
// ggml_build_forward_select() then picks the one that actually runs
|
||||
ggml_cgraph * mtmd_graph_qwen3tts_gen::build() {
|
||||
if (gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
|
||||
return build_ref_encoder();
|
||||
}
|
||||
|
||||
GGML_ASSERT(n_batch == 1); // this module only ever processes one frame at a time
|
||||
|
||||
int idx;
|
||||
|
||||
@@ -1097,6 +1097,13 @@ std::unique_ptr<mtmd_graph> mtmd_get_graph_builder(mtmd_subcontext * ctx, const
|
||||
const int n_frames = params && params->feats ? (int) (params->feats->size() / ctx->model.hparams.projection_dim) : 2;
|
||||
builder = std::make_unique<mtmd_graph_soprano>(ctx, img, n_frames);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_KOKORO:
|
||||
{
|
||||
// without a stage (warmup), a short decoder graph: 2 frames, 600 samples each, one STFT row per 5 samples
|
||||
const auto * io = params ? params->kokoro : nullptr;
|
||||
builder = io ? std::make_unique<mtmd_graph_kokoro>(ctx, img, (mtmd_graph_kokoro::stage_type) io->stage, io->n_tok, io->n_frames, io->n_har)
|
||||
: std::make_unique<mtmd_graph_kokoro>(ctx, img, mtmd_graph_kokoro::STAGE_DEC, 0, 2, 241);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_YOUTUVL:
|
||||
{
|
||||
builder = std::make_unique<mtmd_graph_youtuvl>(ctx, img);
|
||||
|
||||
@@ -128,6 +128,8 @@ public:
|
||||
virtual bool supports_continue() const { return false; }
|
||||
// mtmd_helper_gen_audio_inp::speed and loudness other than the defaults
|
||||
virtual bool supports_delivery() const { return false; }
|
||||
// speed alone, for models without loudness control
|
||||
virtual bool supports_speed() const { return supports_delivery(); }
|
||||
// mtmd_helper_gen_audio_inp::cfg_scale other than the default
|
||||
virtual bool supports_cfg() const { return false; }
|
||||
// mtmd_helper_gen_audio_inp::flow_steps other than the default
|
||||
@@ -201,6 +203,7 @@ public:
|
||||
seq_id = 0;
|
||||
pos = 0;
|
||||
codes_buf.clear();
|
||||
next_window_frames = window_frames;
|
||||
gen_state.reset();
|
||||
audio_pcm.clear();
|
||||
overlay.clear();
|
||||
@@ -232,12 +235,21 @@ public:
|
||||
|
||||
std::vector<float> speaker_embd;
|
||||
if (inp->speaker_ref) {
|
||||
mtmd_bitmap_set_transcript(inp->speaker_ref, inp->speaker_ref_text);
|
||||
if (!encode_speaker(inp->speaker_ref, speaker_embd)) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
const int n_e = n_embd;
|
||||
|
||||
// cloning with the reference transcript continues the reference speech, see generate_icl_prompt() upstream
|
||||
const bool has_ref_text = inp->speaker_ref_text && inp->speaker_ref_text[0];
|
||||
const size_t n_ref_frames = speaker_embd.empty() ? 0 : speaker_embd.size() / (size_t) n_e - 1;
|
||||
const bool icl = has_ref_text && n_ref_frames > 0;
|
||||
if (has_ref_text && !icl) {
|
||||
LOG_WRN("mtmd_helper_gen_audio: no reference codes (mmproj without reference encoder, or clip too long), cloning from the speaker embedding only\n");
|
||||
}
|
||||
auto row = [&](llama_token t) {
|
||||
return std::vector<float>(tok_embd.begin() + (size_t) t * n_e,
|
||||
tok_embd.begin() + (size_t) (t + 1) * n_e);
|
||||
@@ -256,14 +268,31 @@ public:
|
||||
// upstream chat wrap, then slices: [0:3] role, [3:-5] utterance body
|
||||
const std::string full = "<|im_start|>assistant\n" + std::string(inp->prompt, inp->prompt_len) +
|
||||
"<|im_end|>\n<|im_start|>assistant\n";
|
||||
std::vector<llama_token> ids(full.size() + 16);
|
||||
int n_ids = llama_tokenize(vocab, full.c_str(), (int32_t) full.size(), ids.data(), (int32_t) ids.size(),
|
||||
false, true);
|
||||
auto tokenize = [&](const std::string & text) {
|
||||
std::vector<llama_token> tokens(text.size() + 16);
|
||||
const int n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(), (int32_t) tokens.size(),
|
||||
false, true);
|
||||
tokens.resize(n < 0 ? 0 : (size_t) n);
|
||||
return tokens;
|
||||
};
|
||||
|
||||
std::vector<llama_token> ids = tokenize(full);
|
||||
const int n_ids = (int) ids.size();
|
||||
if (n_ids < 8) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
|
||||
return 1;
|
||||
}
|
||||
ids.resize((size_t) n_ids);
|
||||
|
||||
// the reference transcript goes before the text, with the same chat wrap minus the next turn: slices [3:-2]
|
||||
std::vector<llama_token> ref_ids;
|
||||
if (icl) {
|
||||
ref_ids = tokenize("<|im_start|>assistant\n" + std::string(inp->speaker_ref_text) + "<|im_end|>\n");
|
||||
if (ref_ids.size() < 6) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: tokenization of the reference text failed\n");
|
||||
return 1;
|
||||
}
|
||||
ref_ids = std::vector<llama_token>(ref_ids.begin() + 3, ref_ids.end() - 2);
|
||||
}
|
||||
|
||||
std::vector<std::vector<float>> prompt;
|
||||
for (int i = 0; i < 3; i++) prompt.push_back(row(ids[(size_t) i]));
|
||||
@@ -273,9 +302,17 @@ public:
|
||||
prompt.push_back(sum_row(tts_pad, c_think_e));
|
||||
if (!speaker_embd.empty()) prompt.push_back(sum_vec(tts_pad, speaker_embd));
|
||||
prompt.push_back(sum_row(tts_bos, codec_pad));
|
||||
for (llama_token t : ref_ids) prompt.push_back(sum_row(t, codec_pad));
|
||||
for (int i = 3; i < n_ids - 5; i++) prompt.push_back(sum_row(ids[(size_t) i], codec_pad));
|
||||
prompt.push_back(sum_row(tts_eos, codec_pad));
|
||||
prompt.push_back(sum_row(tts_pad, codec_bos));
|
||||
// the reference frames, as if the talker had just spoken them
|
||||
if (icl) {
|
||||
for (size_t f = 0; f < n_ref_frames; f++) {
|
||||
const float * frame = speaker_embd.data() + (f + 1) * (size_t) n_e;
|
||||
prompt.push_back(sum_vec(tts_pad, std::vector<float>(frame, frame + n_e)));
|
||||
}
|
||||
}
|
||||
|
||||
n_prompt = (int) prompt.size();
|
||||
|
||||
@@ -303,6 +340,13 @@ public:
|
||||
if (!init_gen_state(inp->seed)) {
|
||||
return 1;
|
||||
}
|
||||
// the decoder continues from the reference codes of the encode above
|
||||
if (icl) {
|
||||
if (mtmd_gen_state_set_ref(mctx, gen_state.get()) != 0) {
|
||||
return 1;
|
||||
}
|
||||
next_window_frames = window_frames - std::min(n_ref_frames, window_frames - 1);
|
||||
}
|
||||
|
||||
// the prompt above holds the whole text stream up to tts_eos, so every generated
|
||||
// frame adds tts_pad on top of the codes embedding
|
||||
@@ -366,7 +410,7 @@ public:
|
||||
}
|
||||
|
||||
codes_buf.insert(codes_buf.end(), out.codes, out.codes + out.n_codes);
|
||||
if (out.n_codes > 0 && codes_buf.size() / out.n_codes >= window_frames) {
|
||||
if (out.n_codes > 0 && codes_buf.size() / out.n_codes >= next_window_frames) {
|
||||
if (!flush_gen_wav()) {
|
||||
return 1;
|
||||
}
|
||||
@@ -502,6 +546,7 @@ private:
|
||||
}
|
||||
audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
|
||||
codes_buf.clear();
|
||||
next_window_frames = window_frames;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -534,6 +579,7 @@ private:
|
||||
int32_t top_k = 50;
|
||||
float top_p = 1.0f;
|
||||
std::vector<int32_t> codes_buf;
|
||||
size_t next_window_frames = window_frames; // the first window leaves room for the reference codes
|
||||
std::vector<float> audio_pcm;
|
||||
std::vector<float> overlay;
|
||||
std::vector<float> h_state_buf;
|
||||
@@ -1077,6 +1123,244 @@ private:
|
||||
bool wav_header_sent = false;
|
||||
};
|
||||
|
||||
// Kokoro is not autoregressive: the backbone encodes the phonemes once, the mmproj turns its states into the whole utterance
|
||||
// the prompt is a phoneme string (the output of the "tts" chat template), one token per character
|
||||
class kokoro_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
|
||||
public:
|
||||
using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
|
||||
|
||||
// the predicted durations are divided by the speed
|
||||
bool supports_speed() const override { return true; }
|
||||
|
||||
void reset() override {
|
||||
segments.clear();
|
||||
audio.clear();
|
||||
wav.clear();
|
||||
n_encoded = 0;
|
||||
decoded = false;
|
||||
sent = false;
|
||||
}
|
||||
|
||||
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
|
||||
reset();
|
||||
if (!inp->prompt || inp->prompt_len == 0 || inp->seq_id < 0 || (uint32_t) inp->seq_id >= llama_n_seq_max(lctx)) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: kokoro requires a prompt\n");
|
||||
return 1;
|
||||
}
|
||||
if (inp->speaker_ref && !mtmd_bitmap_is_voice(inp->speaker_ref)) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: kokoro has no voice cloning, use one of its voices\n");
|
||||
return 1;
|
||||
}
|
||||
seq_id = inp->seq_id;
|
||||
out_type = inp->out_type;
|
||||
stream = inp->stream;
|
||||
speed = inp->speed;
|
||||
|
||||
// one token per phoneme character, unknown characters are dropped like in the reference
|
||||
if (token_of.empty()) {
|
||||
for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); id++) {
|
||||
if (llama_vocab_get_attr(vocab, id) & LLAMA_TOKEN_ATTR_NORMAL) {
|
||||
token_of[llama_vocab_get_text(vocab, id)] = id;
|
||||
}
|
||||
}
|
||||
}
|
||||
std::vector<llama_token> phonemes;
|
||||
const std::string prompt(inp->prompt, inp->prompt_len);
|
||||
for (size_t i = 0; i < prompt.size();) {
|
||||
const unsigned char c = (unsigned char) prompt[i];
|
||||
const size_t n = c < 0x80 ? 1 : c < 0xE0 ? 2 : c < 0xF0 ? 3 : 4;
|
||||
const auto it = token_of.find(prompt.substr(i, n));
|
||||
if (it != token_of.end()) {
|
||||
phonemes.push_back(it->second);
|
||||
}
|
||||
i += n;
|
||||
}
|
||||
if (phonemes.empty()) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: the prompt has no phoneme the model knows\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
// a voice is a table of style rows, the row is picked by the number of phonemes
|
||||
const char * voice = inp->speaker_ref ? mtmd_bitmap_get_id(inp->speaker_ref) : mtmd_gen_audio_voice_name(mctx, 0);
|
||||
if (!voice) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: the model has no voice\n");
|
||||
return 1;
|
||||
}
|
||||
const std::string name = std::string("voice.") + voice;
|
||||
std::vector<float> table(mtmd_gen_audio_read_tensor(mctx, name.c_str(), nullptr, 0));
|
||||
if (table.size() < n_voice_row || table.size() % n_voice_row != 0 ||
|
||||
mtmd_gen_audio_read_tensor(mctx, name.c_str(), table.data(), table.size()) != table.size()) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: unknown voice \"%s\"\n", voice);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// the backbone takes a whole segment in one batch, padded on both ends
|
||||
const size_t n_max = std::min({(size_t) llama_n_ctx_seq(lctx), (size_t) llama_n_ubatch(lctx), (size_t) llama_model_n_ctx_train(model)});
|
||||
if (n_max < 3) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: the context is too small\n");
|
||||
return 1;
|
||||
}
|
||||
const llama_token pad = llama_vocab_bos(vocab);
|
||||
for (size_t pos = 0; pos < phonemes.size();) {
|
||||
const size_t len = split_len(phonemes, pos, n_max - 2);
|
||||
segment seg;
|
||||
seg.tokens.push_back(pad);
|
||||
seg.tokens.insert(seg.tokens.end(), phonemes.begin() + pos, phonemes.begin() + pos + len);
|
||||
seg.tokens.push_back(pad);
|
||||
const size_t row = std::min(len - 1, table.size() / n_voice_row - 1);
|
||||
seg.feats.assign(table.begin() + row * n_voice_row, table.begin() + (row + 1) * n_voice_row);
|
||||
segments.push_back(std::move(seg));
|
||||
pos += len;
|
||||
// a segment does not start with a space
|
||||
while (pos < phonemes.size() && is_any(phonemes[pos], " ")) {
|
||||
pos++;
|
||||
}
|
||||
}
|
||||
|
||||
if (!init_gen_state(inp->seed)) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// the encoder is not causal, it takes one whole segment per call
|
||||
int32_t step_prompt(int32_t) override {
|
||||
if (segments.empty()) {
|
||||
return -1;
|
||||
}
|
||||
if (n_encoded == segments.size()) {
|
||||
return 0;
|
||||
}
|
||||
auto & seg = segments[n_encoded];
|
||||
const int n = (int) seg.tokens.size();
|
||||
llama_batch_ext_ptr batch(llama_batch_ext_init(lctx));
|
||||
if (!batch_set_tokens(batch.get(), seg.tokens.data(), n, 0, seq_id)) {
|
||||
return -1;
|
||||
}
|
||||
for (int i = 0; i < n; ++i) {
|
||||
llama_batch_ext_set_output_embd(batch.get(), i, true);
|
||||
}
|
||||
if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) != 0) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: kokoro backbone failed\n");
|
||||
return -1;
|
||||
}
|
||||
for (int i = 0; i < n; ++i) {
|
||||
const float * h = llama_get_embeddings_ith(lctx, i);
|
||||
if (!h) {
|
||||
return -1;
|
||||
}
|
||||
seg.feats.insert(seg.feats.end(), h, h + n_embd);
|
||||
}
|
||||
n_encoded++;
|
||||
return (int32_t) (segments.size() - n_encoded);
|
||||
}
|
||||
|
||||
// nothing is generated step by step
|
||||
int32_t step_gen(llama_token, const float *, const float ** h_state_out, bool * out_stop) override {
|
||||
*h_state_out = nullptr;
|
||||
*out_stop = true;
|
||||
return !segments.empty() && n_encoded == segments.size() ? 0 : 1;
|
||||
}
|
||||
|
||||
int32_t get_output(int32_t * sample_rate, const char ** data, size_t * data_len, int64_t * n_samples) override {
|
||||
*sample_rate = info.sample_rate;
|
||||
*data_len = 0;
|
||||
if (n_samples) { *n_samples = 0; }
|
||||
if (stream) {
|
||||
// the audio comes out in one chunk after flush()
|
||||
if (!decoded || sent) {
|
||||
return 0;
|
||||
}
|
||||
sent = true;
|
||||
} else if (!decode_audio()) {
|
||||
return 1;
|
||||
}
|
||||
if (n_samples) { *n_samples = audio.size(); }
|
||||
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
|
||||
*data = (const char *) audio.data();
|
||||
*data_len = audio.size() * sizeof(float);
|
||||
} else {
|
||||
wav.clear();
|
||||
if (!write_wav16(wav, audio, info.sample_rate)) {
|
||||
return 1;
|
||||
}
|
||||
*data = wav.data();
|
||||
*data_len = wav.size();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t flush() override {
|
||||
return decode_audio() ? 0 : 1;
|
||||
}
|
||||
|
||||
private:
|
||||
struct segment {
|
||||
std::vector<llama_token> tokens;
|
||||
std::vector<float> feats; // the voice row, then the backbone state of each token
|
||||
};
|
||||
|
||||
bool is_any(llama_token id, const char * chars) const {
|
||||
const char * text = llama_vocab_get_text(vocab, id);
|
||||
return text[0] != '\0' && strstr(chars, text) != nullptr;
|
||||
}
|
||||
|
||||
// length of the segment that starts at pos: a long text is cut at the last sentence end that fits, else at a weaker break
|
||||
size_t split_len(const std::vector<llama_token> & phonemes, size_t pos, size_t n_max) const {
|
||||
if (phonemes.size() - pos <= n_max) {
|
||||
return phonemes.size() - pos;
|
||||
}
|
||||
for (const char * breaks : {"!.?\xe2\x80\xa6", ":;", ",\xe2\x80\x94", " "}) {
|
||||
for (size_t len = n_max; len > 1; len--) {
|
||||
if (is_any(phonemes[pos + len - 1], breaks)) {
|
||||
return len;
|
||||
}
|
||||
}
|
||||
}
|
||||
return n_max;
|
||||
}
|
||||
|
||||
bool decode_audio() {
|
||||
if (decoded) {
|
||||
return true;
|
||||
}
|
||||
if (segments.empty() || n_encoded != segments.size()) {
|
||||
return false;
|
||||
}
|
||||
for (const auto & seg : segments) {
|
||||
std::vector<int32_t> codes(seg.tokens.begin(), seg.tokens.end());
|
||||
mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
|
||||
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
|
||||
inp.codes = codes.data();
|
||||
inp.n_codes = codes.size();
|
||||
inp.feats = seg.feats.data();
|
||||
inp.n_feats = seg.feats.size();
|
||||
inp.speed = speed;
|
||||
mtmd_gen_out out{};
|
||||
if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
|
||||
return false;
|
||||
}
|
||||
audio.insert(audio.end(), out.audio, out.audio + out.n_samples);
|
||||
}
|
||||
decoded = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
static constexpr size_t n_voice_row = 256; // decoder style, then predictor style
|
||||
|
||||
llama_seq_id seq_id = 0;
|
||||
std::unordered_map<std::string, llama_token> token_of;
|
||||
std::vector<segment> segments;
|
||||
size_t n_encoded = 0;
|
||||
float speed = 1.0f;
|
||||
std::vector<float> audio;
|
||||
std::vector<char> wav;
|
||||
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
|
||||
bool stream = false;
|
||||
bool decoded = false;
|
||||
bool sent = false;
|
||||
};
|
||||
|
||||
// Soprano uses the hidden state that predicts each non-EOS token, including the prompt's last state.
|
||||
class soprano_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
|
||||
public:
|
||||
@@ -4278,6 +4562,8 @@ static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lc
|
||||
return std::unique_ptr<mtmd_gen_audio_pipeline>(new pockettts_gen_audio_pipeline(lctx, mctx));
|
||||
case MTMD_GEN_AUDIO_TYPE_SOPRANO:
|
||||
return std::unique_ptr<mtmd_gen_audio_pipeline>(new soprano_gen_audio_pipeline(lctx, mctx));
|
||||
case MTMD_GEN_AUDIO_TYPE_KOKORO:
|
||||
return std::unique_ptr<mtmd_gen_audio_pipeline>(new kokoro_gen_audio_pipeline(lctx, mctx));
|
||||
case MTMD_GEN_AUDIO_TYPE_LFM3A:
|
||||
return std::unique_ptr<mtmd_gen_audio_pipeline>(new lfm3a_gen_audio_pipeline(lctx, mctx));
|
||||
case MTMD_GEN_AUDIO_TYPE_CHATTERBOX:
|
||||
@@ -4354,8 +4640,12 @@ int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_
|
||||
LOG_ERR("mtmd_helper_gen_audio: speed must be > 0 and loudness < 0 LUFS (0 = model default)\n");
|
||||
return 2;
|
||||
}
|
||||
if ((inp->speed != 1.0f || inp->loudness != 0.0f) && !ctx->pipeline->supports_delivery()) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: the model does not support speed or loudness\n");
|
||||
if (inp->speed != 1.0f && !ctx->pipeline->supports_speed()) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: the model does not support speed\n");
|
||||
return 2;
|
||||
}
|
||||
if (inp->loudness != 0.0f && !ctx->pipeline->supports_delivery()) {
|
||||
LOG_ERR("mtmd_helper_gen_audio: the model does not support loudness\n");
|
||||
return 2;
|
||||
}
|
||||
if (!std::isfinite(inp->cfg_scale) || inp->cfg_scale < 0.0f) {
|
||||
|
||||
@@ -310,7 +310,7 @@ struct mtmd_helper_gen_audio_inp {
|
||||
enum mtmd_helper_gen_audio_out_mode out_mode; // n_past > 0 only (default: auto)
|
||||
|
||||
mtmd_bitmap * speaker_ref; // optional, can be NULL: audio to clone, or a voice bitmap
|
||||
const char * speaker_ref_text; // optional, can be NULL: transcript of speaker_ref, for models that put it in the prompt (e.g. Fish S2)
|
||||
const char * speaker_ref_text; // optional, can be NULL: transcript of speaker_ref, for models that use it to clone the voice (e.g. Fish S2, Qwen3-TTS)
|
||||
const char * lang; // optional, can be NULL
|
||||
|
||||
int32_t top_k;
|
||||
|
||||
@@ -252,6 +252,12 @@
|
||||
#define TN_A_ASP_ATTN "a.asp_attn.%s"
|
||||
#define TN_A_ASP_TDNN "a.asp_tdnn.%s"
|
||||
|
||||
// qwen3tts reference encoder, the transformer blocks use the generic names with the "a.gen.enc.tfm" prefix
|
||||
#define TN_A_GEN_ENC_QUANT_FIRST_IN "a.gen.enc.quant.first.in_proj.%s"
|
||||
#define TN_A_GEN_ENC_QUANT_FIRST_CB "a.gen.enc.quant.first.codebook.%s"
|
||||
#define TN_A_GEN_ENC_QUANT_REST_IN "a.gen.enc.quant.rest.in_proj.%s"
|
||||
#define TN_A_GEN_ENC_QUANT_REST_CB "a.gen.enc.quant.rest.codebook.%s"
|
||||
|
||||
// qwen3tts code_predictor
|
||||
#define TN_A_GEN_CODE_PROJ_IN "a.gen.code.proj_in.%s"
|
||||
#define TN_A_GEN_CODE_EMBD "a.gen.code.embd.%s"
|
||||
@@ -609,6 +615,7 @@ enum projector_type {
|
||||
PROJECTOR_TYPE_POCKETTTS_SPKENC,
|
||||
PROJECTOR_TYPE_POCKETTTS_GEN,
|
||||
PROJECTOR_TYPE_SOPRANO,
|
||||
PROJECTOR_TYPE_KOKORO,
|
||||
PROJECTOR_TYPE_LFM3A_GEN,
|
||||
PROJECTOR_TYPE_DRAMA3_GEN,
|
||||
PROJECTOR_TYPE_ZONOS3_GEN,
|
||||
@@ -705,6 +712,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
|
||||
{ PROJECTOR_TYPE_POCKETTTS_SPKENC, "pockettts_spkenc"},
|
||||
{ PROJECTOR_TYPE_POCKETTTS_GEN, "pockettts_gen"},
|
||||
{ PROJECTOR_TYPE_SOPRANO, "soprano"},
|
||||
{ PROJECTOR_TYPE_KOKORO, "kokoro"},
|
||||
{ PROJECTOR_TYPE_LFM3A_GEN, "lfm3a_gen"},
|
||||
{ PROJECTOR_TYPE_DRAMA3_GEN, "drama3_gen"},
|
||||
{ PROJECTOR_TYPE_ZONOS3_GEN, "zonos3_gen"},
|
||||
|
||||
@@ -1075,6 +1075,14 @@ struct mtmd_model_loader {
|
||||
// TODO: hardcoded for now, read from code_predictor_config instead
|
||||
hparams.rope_theta = 1000000.0f;
|
||||
|
||||
// reference encoder: mimi with the stride-2 downsample of the 12.5 Hz tokenizer
|
||||
// seanet ratios are [8,6,5,4] in the config, the encoder reverses them
|
||||
hparams.audio_sample_rate = 24000;
|
||||
hparams.seanet_ratios = { 4, 5, 6, 8 };
|
||||
hparams.seanet_n_stage = (int32_t) hparams.seanet_ratios.size();
|
||||
hparams.mimi_downsample = 2;
|
||||
hparams.mimi_tfm_context = 250;
|
||||
|
||||
// code2wav params
|
||||
hparams.wav_tfm_n_layer = 8;
|
||||
hparams.wav_tfm_n_embd = 512;
|
||||
@@ -1164,6 +1172,10 @@ struct mtmd_model_loader {
|
||||
hparams.audio_sample_rate = 32000;
|
||||
hparams.audio_hop_len = 512;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_KOKORO:
|
||||
{
|
||||
hparams.audio_sample_rate = 24000;
|
||||
} break;
|
||||
case PROJECTOR_TYPE_PADDLEOCR:
|
||||
{
|
||||
hparams.n_merge = 2;
|
||||
@@ -1570,6 +1582,7 @@ struct mtmd_model_loader {
|
||||
model.proj_type != PROJECTOR_TYPE_ZONOS3_SPKENC &&
|
||||
model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN &&
|
||||
model.proj_type != PROJECTOR_TYPE_SOPRANO &&
|
||||
model.proj_type != PROJECTOR_TYPE_KOKORO &&
|
||||
model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO &&
|
||||
model.proj_type != PROJECTOR_TYPE_VIBEVOICE_ASR &&
|
||||
model.proj_type != PROJECTOR_TYPE_CHATTERBOX &&
|
||||
@@ -2826,6 +2839,47 @@ struct mtmd_model_loader {
|
||||
|
||||
// code2wav: RVQ codes -> raw PCM, lives in the same ctx as code_predictor
|
||||
load_code2wav(true);
|
||||
|
||||
// reference encoder, absent from files converted before it was added
|
||||
model.ref_quant_rest_cb = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_REST_CB, "weight"), false);
|
||||
if (model.ref_quant_rest_cb) {
|
||||
load_seanet(model.seanet, false);
|
||||
model.downsample_w = get_tensor(mtmd_format(TN_A_DOWNSAMPLE_CONV, "weight"));
|
||||
for (int il = 0; ; il++) {
|
||||
const char * p = "a.gen.enc.tfm";
|
||||
ggml_tensor * ln_1_w = get_tensor(mtmd_format(TN_LN_1, p, il, "weight"), false);
|
||||
if (!ln_1_w) {
|
||||
break;
|
||||
}
|
||||
mtmd_layer layer;
|
||||
layer.ln_1_w = ln_1_w;
|
||||
layer.ln_1_b = get_tensor(mtmd_format(TN_LN_1, p, il, "bias"));
|
||||
layer.q_w = get_tensor(mtmd_format(TN_ATTN_Q, p, il, "weight"));
|
||||
layer.k_w = get_tensor(mtmd_format(TN_ATTN_K, p, il, "weight"));
|
||||
layer.v_w = get_tensor(mtmd_format(TN_ATTN_V, p, il, "weight"));
|
||||
layer.o_w = get_tensor(mtmd_format(TN_ATTN_OUTPUT, p, il, "weight"));
|
||||
layer.ls_1_w = get_tensor(mtmd_format(TN_LS_1, p, il, "weight"));
|
||||
layer.ln_2_w = get_tensor(mtmd_format(TN_LN_2, p, il, "weight"));
|
||||
layer.ln_2_b = get_tensor(mtmd_format(TN_LN_2, p, il, "bias"));
|
||||
layer.ff_up_w = get_tensor(mtmd_format(TN_FFN_UP, p, il, "weight"));
|
||||
layer.ff_down_w = get_tensor(mtmd_format(TN_FFN_DOWN, p, il, "weight"));
|
||||
layer.ls_2_w = get_tensor(mtmd_format(TN_LS_2, p, il, "weight"));
|
||||
model.ref_enc_layers.push_back(layer);
|
||||
}
|
||||
model.ref_quant_first_in_w = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_FIRST_IN, "weight"));
|
||||
model.ref_quant_first_cb = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_FIRST_CB, "weight"));
|
||||
model.ref_quant_rest_in_w = get_tensor(mtmd_format(TN_A_GEN_ENC_QUANT_REST_IN, "weight"));
|
||||
|
||||
// the codes index the code predictor tables, see build_ref_encoder()
|
||||
const int64_t n_acoustic = model.ref_quant_rest_cb->ne[2];
|
||||
if (model.ref_enc_layers.empty() ||
|
||||
n_acoustic != model.gen_code_head_w->ne[2] ||
|
||||
n_acoustic > model.gen_code_embd_w->ne[2] ||
|
||||
model.ref_quant_first_cb->ne[1] > model.gen_code_out_embd_w->ne[1] ||
|
||||
model.ref_quant_rest_cb->ne[1] > model.gen_code_embd_w->ne[1]) {
|
||||
throw std::runtime_error(mtmd_format("%s: qwen3tts: unexpected reference encoder shape\n", __func__));
|
||||
}
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_ZONOS3_GEN:
|
||||
{
|
||||
@@ -3410,6 +3464,21 @@ struct mtmd_model_loader {
|
||||
}
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_KOKORO:
|
||||
{
|
||||
// the graphs and the host code look the tensors up by name, see models/kokoro.cpp
|
||||
// a voice is a table of style rows, the audio pipeline reads the row it needs
|
||||
for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) {
|
||||
const std::string name = t->name;
|
||||
if (name.rfind("a.gen.", 0) == 0) {
|
||||
model.cbx_tensors[name] = get_tensor(name);
|
||||
} else if (name.rfind("voice.", 0) == 0) {
|
||||
model.cbx_tensors[name] = get_tensor(name);
|
||||
model.gen_voices[name.substr(6)] = {};
|
||||
}
|
||||
}
|
||||
LOG_INF("%s: %zu predefined voices\n", __func__, model.gen_voices.size());
|
||||
} break;
|
||||
case PROJECTOR_TYPE_SOPRANO:
|
||||
{
|
||||
auto & v = model.vocos;
|
||||
@@ -4906,6 +4975,7 @@ int mtmd_model::n_mmproj_embd() const {
|
||||
// talker conditioning rows, the speaker projection carries their width
|
||||
return cbx_tensors.at("a.cenc.spkr_enc.weight")->ne[1];
|
||||
case PROJECTOR_TYPE_SOPRANO:
|
||||
case PROJECTOR_TYPE_KOKORO:
|
||||
return hparams.projection_dim;
|
||||
case PROJECTOR_TYPE_PARAKEET:
|
||||
return mm_1_w->ne[1];
|
||||
|
||||
@@ -213,6 +213,8 @@ struct mtmd_hparams {
|
||||
|
||||
// pocket-tts
|
||||
static constexpr int32_t pockettts_max_spk_seconds = 30;
|
||||
// qwen3tts: the reference encoder mask is dense, so cost is quadratic in the clip
|
||||
static constexpr int32_t qwen3tts_max_ref_seconds = 30;
|
||||
// fish_s2: the codec encoder runs at 44.1 kHz, memory grows with the clip
|
||||
static constexpr int32_t fish_s2_max_spk_seconds = 30;
|
||||
int32_t seanet_n_stage = 0;
|
||||
@@ -1300,6 +1302,17 @@ struct mtmd_model {
|
||||
ggml_tensor * gen_code_head_norm_w = nullptr; // per-codebook norm before the tied head, merged 2D
|
||||
ggml_tensor * gen_code_mask_embd = nullptr;
|
||||
|
||||
// qwen3tts reference encoder: reference clip -> RVQ codes, uses seanet and downsample_w too
|
||||
std::vector<mtmd_layer> ref_enc_layers;
|
||||
ggml_tensor * ref_quant_first_in_w = nullptr;
|
||||
ggml_tensor * ref_quant_first_cb = nullptr;
|
||||
ggml_tensor * ref_quant_rest_in_w = nullptr;
|
||||
ggml_tensor * ref_quant_rest_cb = nullptr; // acoustic codebooks, merged 3D
|
||||
|
||||
bool has_ref_encoder() const {
|
||||
return ref_quant_rest_cb != nullptr;
|
||||
}
|
||||
|
||||
// qwen3tts code2wav: RVQ codes -> raw PCM
|
||||
mtmd_code2wav c2w;
|
||||
std::vector<mtmd_drama3_layer> drama3_layers;
|
||||
|
||||
@@ -521,6 +521,87 @@ mtmd_subctx_enc_state * mtmd_subcontext::enc_state_init(int32_t n_frames_max) {
|
||||
return st.release();
|
||||
}
|
||||
|
||||
bool mtmd_subcontext::encode_kokoro(mtmd_subctx_encode_params * params) {
|
||||
const int n_embd = model.n_mmproj_embd();
|
||||
const int n_style = mtmd_graph_kokoro::n_style;
|
||||
if (params->gen_process != MTMD_SUBCTX_GEN_PROCESS_GEN_WAV || !params->codes || !params->feats || !params->gen_state || !params->out_audio) {
|
||||
LOG_ERR("%s: kokoro takes the phoneme tokens and their backbone states\n", __func__);
|
||||
return false;
|
||||
}
|
||||
const auto & tokens = *params->codes;
|
||||
const auto & feats = *params->feats;
|
||||
const int n_tok = (int) tokens.size();
|
||||
if (n_tok < 2 || feats.size() != (size_t) 2 * n_style + (size_t) n_tok * n_embd) {
|
||||
LOG_ERR("%s: expected a voice row of %d values, then %d values for each of the %d tokens\n", __func__, 2 * n_style, n_embd, n_tok);
|
||||
return false;
|
||||
}
|
||||
const int64_t n_vocab = model.cbx_tensors.at("a.gen.text.embedding.weight")->ne[1];
|
||||
for (int32_t t : tokens) {
|
||||
if (t < 0 || t >= n_vocab) {
|
||||
LOG_ERR("%s: phoneme token out of range (%d, vocabulary size %lld)\n", __func__, t, (long long) n_vocab);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
const std::vector<float> style_dec(feats.begin(), feats.begin() + n_style);
|
||||
const std::vector<float> style_pred(feats.begin() + n_style, feats.begin() + 2 * n_style);
|
||||
const float * bert = feats.data() + 2 * n_style;
|
||||
|
||||
auto run = [&](mtmd_subctx_kokoro_io & io) {
|
||||
mtmd_subctx_encode_params p = *params;
|
||||
p.kokoro = &io;
|
||||
p.out_audio = nullptr;
|
||||
return encode(&p);
|
||||
};
|
||||
|
||||
std::vector<float> text_cnn;
|
||||
{
|
||||
mtmd_subctx_kokoro_io io;
|
||||
io.stage = mtmd_graph_kokoro::STAGE_TEXT;
|
||||
io.n_tok = n_tok;
|
||||
io.tokens = &tokens;
|
||||
io.out0 = &text_cnn;
|
||||
if (!run(io)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<float> shared, asr;
|
||||
const int n_frames = mtmd_graph_kokoro::prosody(model, bert, n_embd, text_cnn.data(), n_tok, style_pred.data(), params->speed, shared, asr);
|
||||
|
||||
std::vector<float> f0, en;
|
||||
{
|
||||
mtmd_subctx_kokoro_io io;
|
||||
io.stage = mtmd_graph_kokoro::STAGE_F0N;
|
||||
io.n_frames = n_frames;
|
||||
io.x = &shared;
|
||||
io.style = &style_pred;
|
||||
io.out0 = &f0;
|
||||
io.out1 = &en;
|
||||
if (!run(io)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<float> har, spec;
|
||||
{
|
||||
mtmd_subctx_kokoro_io io;
|
||||
io.stage = mtmd_graph_kokoro::STAGE_DEC;
|
||||
io.n_frames = n_frames;
|
||||
io.n_har = mtmd_graph_kokoro::har_source(model, f0, params->gen_state->rng, har);
|
||||
io.x = &asr;
|
||||
io.style = &style_dec;
|
||||
io.f0 = &f0;
|
||||
io.en = &en;
|
||||
io.har = &har;
|
||||
io.out0 = &spec;
|
||||
if (!run(io)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
mtmd_graph_kokoro::istft(spec, *params->out_audio);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
const mtmd_image_f32_batch & imgs = *params->imgs;
|
||||
int n_batch_cur = imgs.entries.size();
|
||||
@@ -536,6 +617,10 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
warmup(*params->imgs);
|
||||
}
|
||||
|
||||
if (model.proj_type == PROJECTOR_TYPE_KOKORO && !params->kokoro) {
|
||||
return encode_kokoro(params);
|
||||
}
|
||||
|
||||
const bool needs_gen_state = params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_CODE ||
|
||||
params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ||
|
||||
params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TTS;
|
||||
@@ -675,9 +760,9 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
set_input_attn_mask(mask);
|
||||
};
|
||||
|
||||
// rope positions and attention mask of the mimi transformers (pocket-tts).
|
||||
// rope positions and attention mask of the mimi encoder transformer (pocket-tts, qwen3tts reference encoder).
|
||||
// the mask is causal with a sliding window, see _build_attention_mask() in the reference
|
||||
auto set_pockettts_tfm_inputs = [&]() {
|
||||
auto set_mimi_tfm_inputs = [&](int64_t max_samples) {
|
||||
const int64_t n_pos = ggml_nelements(get_inp_tensor("inp_pos"));
|
||||
GGML_ASSERT(n_pos > 0);
|
||||
std::vector<int32_t> positions((size_t) n_pos);
|
||||
@@ -686,9 +771,13 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
}
|
||||
set_input_i32("inp_pos", positions);
|
||||
|
||||
// the preprocessor truncates the waveform to keep this mask bounded
|
||||
const int64_t max_pos = (int64_t) mtmd_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate / 120;
|
||||
GGML_ASSERT(n_pos <= max_pos && "pocket-tts speaker reference too long for a dense mask");
|
||||
// the caller truncates the waveform to keep this mask bounded
|
||||
int64_t hop = 1;
|
||||
for (int32_t ratio : hparams.seanet_ratios) {
|
||||
hop *= ratio;
|
||||
}
|
||||
const int64_t max_pos = (max_samples + hop - 1) / hop;
|
||||
GGML_ASSERT(n_pos <= max_pos && "speaker reference too long for a dense mask");
|
||||
|
||||
const int64_t context = hparams.mimi_tfm_context;
|
||||
std::vector<float> mask((size_t) n_pos * n_pos, -INFINITY);
|
||||
@@ -1483,7 +1572,7 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
} break;
|
||||
case PROJECTOR_TYPE_POCKETTTS_SPKENC:
|
||||
{
|
||||
set_pockettts_tfm_inputs();
|
||||
set_mimi_tfm_inputs((int64_t) mtmd_hparams::pockettts_max_spk_seconds * hparams.audio_sample_rate);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_POCKETTTS_GEN:
|
||||
{
|
||||
@@ -1510,6 +1599,29 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
{
|
||||
set_input_f32("inp_feats", *params->feats);
|
||||
} break;
|
||||
case PROJECTOR_TYPE_KOKORO:
|
||||
{
|
||||
const auto * io = params->kokoro;
|
||||
GGML_ASSERT(io);
|
||||
switch (io->stage) {
|
||||
case mtmd_graph_kokoro::STAGE_TEXT:
|
||||
{
|
||||
std::vector<int32_t> tokens = *io->tokens;
|
||||
set_input_i32("inp_tokens", tokens);
|
||||
} break;
|
||||
case mtmd_graph_kokoro::STAGE_F0N:
|
||||
set_input_f32("inp_x", *io->x);
|
||||
set_input_f32("inp_style", *io->style);
|
||||
break;
|
||||
case mtmd_graph_kokoro::STAGE_DEC:
|
||||
set_input_f32("inp_asr", *io->x);
|
||||
set_input_f32("inp_style", *io->style);
|
||||
set_input_f32("inp_f0", *io->f0);
|
||||
set_input_f32("inp_n", *io->en);
|
||||
set_input_f32("inp_har", *io->har);
|
||||
break;
|
||||
}
|
||||
} break;
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
{
|
||||
set_input_f32("inp_feats", *params->feats);
|
||||
@@ -1887,6 +1999,10 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
} break;
|
||||
case PROJECTOR_TYPE_QWEN3TTS_GEN:
|
||||
{
|
||||
if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
|
||||
set_mimi_tfm_inputs((int64_t) mtmd_hparams::qwen3tts_max_ref_seconds * hparams.audio_sample_rate);
|
||||
break;
|
||||
}
|
||||
if (params->gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV) {
|
||||
GGML_ASSERT(params->codes != nullptr);
|
||||
|
||||
@@ -2393,6 +2509,22 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (model.proj_type == PROJECTOR_TYPE_KOKORO) {
|
||||
const auto * io = params->kokoro;
|
||||
auto read = [&](const char * name, std::vector<float> * out) {
|
||||
ggml_tensor * t = ggml_graph_get_tensor(gf, name);
|
||||
GGML_ASSERT(t != nullptr && out != nullptr);
|
||||
out->resize(ggml_nelements(t));
|
||||
ggml_backend_tensor_get(t, out->data(), 0, ggml_nbytes(t));
|
||||
};
|
||||
switch (io->stage) {
|
||||
case mtmd_graph_kokoro::STAGE_TEXT: read("out_text", io->out0); break;
|
||||
case mtmd_graph_kokoro::STAGE_F0N: read("out_f0", io->out0); read("out_n", io->out1); break;
|
||||
case mtmd_graph_kokoro::STAGE_DEC: read("out_spec", io->out0); break;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
if (embeddings != nullptr) {
|
||||
// sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one)
|
||||
const int n_tokens_out = embeddings->ne[1];
|
||||
@@ -2423,6 +2555,35 @@ bool mtmd_subcontext::encode(mtmd_subctx_encode_params * params) {
|
||||
// for audio gen models
|
||||
//
|
||||
|
||||
if (model.proj_type == PROJECTOR_TYPE_QWEN3TTS_GEN && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
|
||||
GGML_ASSERT(params->out_codes && params->out_code_embd);
|
||||
|
||||
// frame-major codes, the layout of GEN_WAV's input
|
||||
std::vector<std::vector<int32_t>> groups;
|
||||
for (int g = 0; ; g++) {
|
||||
ggml_tensor * t = ggml_graph_get_tensor(gf, ("out_ref_code_" + std::to_string(g)).c_str());
|
||||
if (!t) {
|
||||
break;
|
||||
}
|
||||
groups.emplace_back((size_t) ggml_nelements(t));
|
||||
ggml_backend_tensor_get(t, groups.back().data(), 0, ggml_nbytes(t));
|
||||
}
|
||||
GGML_ASSERT(!groups.empty());
|
||||
const size_t n_frames = groups[0].size();
|
||||
params->out_codes->resize(n_frames * groups.size());
|
||||
for (size_t f = 0; f < n_frames; f++) {
|
||||
for (size_t g = 0; g < groups.size(); g++) {
|
||||
(*params->out_codes)[f * groups.size() + g] = groups[g][f];
|
||||
}
|
||||
}
|
||||
|
||||
ggml_tensor * rows = ggml_graph_get_tensor(gf, "out_ref_rows");
|
||||
GGML_ASSERT(rows && rows->type == GGML_TYPE_F32 && (size_t) rows->ne[1] == n_frames);
|
||||
params->out_code_embd->resize((size_t) ggml_nelements(rows));
|
||||
ggml_backend_tensor_get(rows, params->out_code_embd->data(), 0, ggml_nbytes(rows));
|
||||
return true;
|
||||
}
|
||||
|
||||
if (model.proj_type == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == MTMD_SUBCTX_GEN_PROCESS_TOKENIZE) {
|
||||
ggml_tensor * fsq = ggml_graph_get_tensor(gf, "out_fsq");
|
||||
GGML_ASSERT(fsq != nullptr && params->out_codes);
|
||||
|
||||
@@ -114,6 +114,22 @@ struct mtmd_subctx_gen_state {
|
||||
}
|
||||
};
|
||||
|
||||
// kokoro: inputs and outputs of one graph stage, see mtmd_graph_kokoro
|
||||
struct mtmd_subctx_kokoro_io {
|
||||
int stage = 0;
|
||||
int n_tok = 0;
|
||||
int n_frames = 0;
|
||||
int n_har = 0;
|
||||
const std::vector<int32_t> * tokens = nullptr; // STAGE_TEXT
|
||||
const std::vector<float> * x = nullptr; // STAGE_F0N: shared LSTM output, STAGE_DEC: aligned text features
|
||||
const std::vector<float> * style = nullptr; // predictor or decoder half of the voice row
|
||||
const std::vector<float> * f0 = nullptr; // STAGE_DEC
|
||||
const std::vector<float> * en = nullptr;
|
||||
const std::vector<float> * har = nullptr;
|
||||
std::vector<float> * out0 = nullptr; // text features, F0, or STFT coefficients
|
||||
std::vector<float> * out1 = nullptr; // STAGE_F0N: energy
|
||||
};
|
||||
|
||||
struct mtmd_subctx_encode_params {
|
||||
int n_threads = 1;
|
||||
const mtmd_image_f32_batch * imgs = nullptr;
|
||||
@@ -148,6 +164,10 @@ struct mtmd_subctx_encode_params {
|
||||
int vocode_n_stft = 0;
|
||||
std::vector<float> * out_spec = nullptr;
|
||||
|
||||
// kokoro: set by encode_kokoro() for each of its graph stages
|
||||
const mtmd_subctx_kokoro_io * kokoro = nullptr;
|
||||
float speed = 1.0f; // kokoro: the predicted durations are divided by it
|
||||
|
||||
// audio gen: state of one sequence, encode() updates it in place
|
||||
struct mtmd_subctx_gen_state * gen_state = nullptr;
|
||||
|
||||
@@ -205,6 +225,9 @@ private:
|
||||
|
||||
mtmd_image_f32_batch get_dummy_batch() const;
|
||||
|
||||
// kokoro GEN_WAV: codes are the phoneme tokens, feats the voice row followed by the backbone state of each token
|
||||
bool encode_kokoro(mtmd_subctx_encode_params * params);
|
||||
|
||||
// only initialize backend buffers, but do not allocate them yet
|
||||
support_info_graph reserve_compute_meta(const mtmd_image_f32_batch & batch);
|
||||
};
|
||||
|
||||
+161
-3
@@ -204,6 +204,7 @@ struct mtmd_bitmap {
|
||||
std::shared_ptr<const mtmd_gen_ref> gen_ref; // gen-side reference of a predefined voice, if any
|
||||
bool mergeable = false; // [QWEN_VIDEO] set only on frames of the same video
|
||||
bool is_video = false; // see mtmd_bitmap_set_is_video()
|
||||
std::string transcript; // see mtmd_bitmap_set_transcript()
|
||||
|
||||
// lazy-loaded bitmap
|
||||
mtmd_bitmap_lazy_callback lazy_callback = nullptr;
|
||||
@@ -2056,6 +2057,12 @@ struct mtmd_tokenizer {
|
||||
// sanity check
|
||||
GGML_ASSERT(ctx->audio_preproc != nullptr);
|
||||
|
||||
// qwen3tts: a clip with a transcript also gives the reference codes, see mtmd_qwen3tts_ref_encode()
|
||||
mtmd_image_f32 ref_wav;
|
||||
bool with_ref_codes = ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_SPKENC &&
|
||||
ctx->ctx_gen_a && ctx->ctx_gen_a->model.has_ref_encoder() &&
|
||||
!bitmap->transcript.empty();
|
||||
|
||||
// preprocess audio
|
||||
auto source = std::make_shared<mtmd_audio_preproc_out>();
|
||||
{
|
||||
@@ -2081,6 +2088,19 @@ struct mtmd_tokenizer {
|
||||
LOG_ERR("%s: this model takes audio input only via mtmd_audio_stream\n", __func__);
|
||||
return 2;
|
||||
}
|
||||
if (with_ref_codes) {
|
||||
// a cut clip no longer matches its transcript, so a long clip gives no codes at all
|
||||
const size_t sample_rate = (size_t) ctx->ctx_gen_a->model.hparams.audio_sample_rate;
|
||||
const size_t max_samples = (size_t) mtmd_hparams::qwen3tts_max_ref_seconds * sample_rate;
|
||||
if (n_samples > max_samples) {
|
||||
LOG_WRN("%s: speaker reference is %.1f s, longer than %d s, the reference codes are skipped\n", __func__,
|
||||
(double) n_samples / sample_rate, mtmd_hparams::qwen3tts_max_ref_seconds);
|
||||
with_ref_codes = false;
|
||||
} else {
|
||||
ref_wav.set_size({(int) n_samples, 1}, false, true);
|
||||
ref_wav.cpy_buf(std::vector<float>(samples, samples + n_samples));
|
||||
}
|
||||
}
|
||||
*source = ctx->audio_preproc->preprocess(samples, n_samples);
|
||||
if (source->entries.empty()) {
|
||||
LOG_ERR("Unable to preprocess audio\n");
|
||||
@@ -2123,6 +2143,17 @@ struct mtmd_tokenizer {
|
||||
batch_f32.is_audio = true;
|
||||
batch_f32.entries.push_back(std::move(mel_f32));
|
||||
|
||||
// the reference waveform is a second entry, its frames follow the speaker row
|
||||
if (with_ref_codes && i == 0) {
|
||||
const auto & gen_hparams = ctx->ctx_gen_a->model.hparams;
|
||||
int64_t samples_per_frame = gen_hparams.mimi_downsample;
|
||||
for (int32_t ratio : gen_hparams.seanet_ratios) {
|
||||
samples_per_frame *= ratio;
|
||||
}
|
||||
n_tokens += (ref_wav.nx() + samples_per_frame - 1) / samples_per_frame;
|
||||
batch_f32.entries.push_back(std::move(ref_wav));
|
||||
}
|
||||
|
||||
mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens);
|
||||
audio_tokens->n_tokens = n_tokens;
|
||||
audio_tokens->batch_f32 = std::move(batch_f32);
|
||||
@@ -2470,6 +2501,51 @@ static bool mtmd_audio_batch_encode(mtmd_context * ctx, const mtmd_image_f32_bat
|
||||
return ctx->ctx_a->encode_batch(ctx->n_threads, &batch, out_embd);
|
||||
}
|
||||
|
||||
// qwen3tts: the speaker row, then one talker input row per reference frame
|
||||
// the reference codes go to ctx->gen_ref, so GEN_WAV continues from them, see mtmd_gen_state_set_ref()
|
||||
static int32_t mtmd_qwen3tts_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector<float> & out_embd) {
|
||||
const size_t n_embd = ctx->n_embd_out();
|
||||
const auto & entries = audio_tokens->batch_f32.entries;
|
||||
GGML_ASSERT(entries.size() == 2);
|
||||
|
||||
mtmd_image_f32_batch mel;
|
||||
mel.is_audio = true;
|
||||
mel.entries.push_back(entries[0]);
|
||||
|
||||
std::vector<float> speaker(n_embd);
|
||||
if (!mtmd_audio_batch_encode(ctx, mel, audio_tokens->source.get(), audio_tokens->source_idx, speaker)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
mtmd_image_f32_batch wav;
|
||||
wav.is_audio = true;
|
||||
wav.entries.push_back(entries[1]);
|
||||
|
||||
std::vector<int32_t> codes;
|
||||
std::vector<float> rows;
|
||||
|
||||
mtmd_subctx_encode_params params;
|
||||
params.imgs = &wav;
|
||||
params.n_threads = ctx->n_threads;
|
||||
params.gen_process = MTMD_SUBCTX_GEN_PROCESS_TOKENIZE;
|
||||
params.out_codes = &codes;
|
||||
params.out_code_embd = &rows;
|
||||
|
||||
if (!ctx->ctx_gen_a->encode(¶ms)) {
|
||||
LOG_ERR("%s: encode failed (reference codes)\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
if (rows.size() != (size_t) (audio_tokens->n_tokens - 1) * n_embd) {
|
||||
LOG_ERR("%s: %zu reference rows, expected %u\n", __func__, rows.size() / n_embd, audio_tokens->n_tokens - 1);
|
||||
return 1;
|
||||
}
|
||||
|
||||
out_embd = std::move(speaker);
|
||||
out_embd.insert(out_embd.end(), rows.begin(), rows.end());
|
||||
ctx->gen_ref.tokens = std::move(codes);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk * chunk, std::vector<float> & out_embd) {
|
||||
ctx->gen_ref = {}; // see mtmd_gen_state_set_ref()
|
||||
if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) {
|
||||
@@ -2524,6 +2600,9 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk
|
||||
if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_CHATTERBOX_SPKENC) {
|
||||
return cbx_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
|
||||
}
|
||||
if (ctx->ctx_a->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_SPKENC && chunk->tokens_audio->batch_f32.entries.size() == 2) {
|
||||
return mtmd_qwen3tts_ref_encode(ctx, chunk->tokens_audio.get(), out_embd);
|
||||
}
|
||||
out_embd.resize((size_t)chunk->tokens_audio->n_tokens * ctx->n_embd_out());
|
||||
bool ok = mtmd_audio_batch_encode(ctx, chunk->tokens_audio->batch_f32, chunk->tokens_audio->source.get(), chunk->tokens_audio->source_idx, out_embd);
|
||||
return ok ? 0 : 1;
|
||||
@@ -2615,6 +2694,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
|
||||
info.type = MTMD_GEN_AUDIO_TYPE_FISH_S2;
|
||||
info.sample_rate = 44100;
|
||||
break;
|
||||
case PROJECTOR_TYPE_KOKORO:
|
||||
info.type = MTMD_GEN_AUDIO_TYPE_KOKORO;
|
||||
info.sample_rate = 24000;
|
||||
break;
|
||||
default:
|
||||
info.type = MTMD_GEN_AUDIO_TYPE_NONE;
|
||||
break;
|
||||
@@ -3012,7 +3095,8 @@ mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * na
|
||||
|
||||
mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
|
||||
mtmd_gen_inp inp{};
|
||||
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
|
||||
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
|
||||
inp.speed = 1.0f;
|
||||
if (!ctx->ctx_gen_a) {
|
||||
return inp;
|
||||
}
|
||||
@@ -3031,6 +3115,7 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
|
||||
inp.temp = 0.3f;
|
||||
break;
|
||||
case PROJECTOR_TYPE_SOPRANO:
|
||||
case PROJECTOR_TYPE_KOKORO:
|
||||
inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
|
||||
break;
|
||||
case PROJECTOR_TYPE_MINIMAX_MUSIC3_GEN:
|
||||
@@ -3079,7 +3164,9 @@ struct mtmd_gen_state {
|
||||
std::vector<float> out_audio; // decoded PCM samples for the current frame (GEN_WAV)
|
||||
|
||||
// voice reference for GEN_WAV, from a speaker clip or a predefined voice (chatterbox)
|
||||
// qwen3tts: the reference codes, decoded with the first GEN_WAV call so the speech continues from them
|
||||
mtmd_gen_ref ref;
|
||||
bool ref_decoded = false;
|
||||
|
||||
// lfm3a: the codec graph yields STFT frames, the ISTFT runs on the host
|
||||
std::unique_ptr<mtmd_audio_streaming_istft> istft;
|
||||
@@ -3126,6 +3213,7 @@ void mtmd_gen_state_free(mtmd_gen_state * state) {
|
||||
|
||||
void mtmd_gen_state_reset(mtmd_gen_state * state) {
|
||||
state->gen_state->reset();
|
||||
state->ref_decoded = false;
|
||||
if (state->istft) {
|
||||
const auto * hparams = &state->sctx->model.hparams;
|
||||
state->istft = std::make_unique<mtmd_audio_streaming_istft>(hparams->audio_n_fft, hparams->audio_hop_len);
|
||||
@@ -3622,6 +3710,7 @@ int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) {
|
||||
return 1;
|
||||
}
|
||||
state->ref = ctx->gen_ref;
|
||||
state->ref_decoded = false;
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -3655,6 +3744,46 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
|
||||
}
|
||||
}
|
||||
|
||||
if (sctx->model.proj_type == PROJECTOR_TYPE_KOKORO) {
|
||||
if (inp->type != MTMD_GEN_PROCESS_TYPE_GEN_WAV || !inp->codes || inp->n_codes == 0 || !inp->feats || inp->n_feats == 0) {
|
||||
LOG_ERR("%s: kokoro requires the phoneme tokens and their features\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
if (!std::isfinite(inp->speed) || inp->speed <= 0.0f) {
|
||||
LOG_ERR("%s: speed must be > 0\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
const std::vector<int32_t> in_codes(inp->codes, inp->codes + inp->n_codes);
|
||||
const std::vector<float> in_feats(inp->feats, inp->feats + inp->n_feats);
|
||||
|
||||
// the batch entry is unused, present to satisfy the encode interface
|
||||
mtmd_image_f32 dummy;
|
||||
dummy.set_size({1, 1}, false, true);
|
||||
dummy.cpy_buf(std::vector<float>(1, 0.0f));
|
||||
mtmd_image_f32_batch batch;
|
||||
batch.is_audio = true;
|
||||
batch.entries.push_back(std::move(dummy));
|
||||
|
||||
mtmd_subctx_encode_params params;
|
||||
params.imgs = &batch;
|
||||
params.n_threads = ctx->n_threads;
|
||||
params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV;
|
||||
params.codes = &in_codes;
|
||||
params.feats = &in_feats;
|
||||
params.speed = inp->speed;
|
||||
params.out_audio = &state->out_audio;
|
||||
params.gen_state = state->gen_state;
|
||||
|
||||
if (!sctx->encode(¶ms)) {
|
||||
LOG_ERR("%s: encode failed (gen_wav)\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
|
||||
out->audio = state->out_audio.data();
|
||||
out->n_samples = state->out_audio.size();
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && sctx->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN) {
|
||||
return mtmd_gen_wav_lfm3a(ctx, state, inp, out);
|
||||
}
|
||||
@@ -3784,16 +3913,37 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
|
||||
params.imgs = &batch;
|
||||
params.n_threads = ctx->n_threads;
|
||||
params.gen_process = MTMD_SUBCTX_GEN_PROCESS_GEN_WAV;
|
||||
params.codes = has_codes ? &in_codes : nullptr;
|
||||
params.feats = has_feats ? &in_feats : nullptr;
|
||||
params.out_audio = &state->out_audio;
|
||||
params.gen_state = state->gen_state;
|
||||
|
||||
// qwen3tts: the first call puts the end of the reference before its codes, so the speech continues from it
|
||||
// a batch shorter than the window carries a wrong state, so the reference only fills the free part of this window
|
||||
size_t n_ref_used = 0;
|
||||
size_t n_frames = 0;
|
||||
if (sctx->model.proj_type == PROJECTOR_TYPE_QWEN3TTS_GEN && has_codes) {
|
||||
const size_t n_codes = (size_t) sctx->model.gen_code_head_w->ne[2] + 1;
|
||||
if (!state->ref.tokens.empty() && !state->ref_decoded) {
|
||||
state->ref_decoded = true;
|
||||
const size_t n_win = (size_t) sctx->model.hparams.wav_tfm_swa;
|
||||
const size_t n_in = in_codes.size() / n_codes;
|
||||
n_ref_used = std::min(state->ref.tokens.size() / n_codes, n_win > n_in ? n_win - n_in : 0);
|
||||
in_codes.insert(in_codes.begin(), state->ref.tokens.end() - (std::ptrdiff_t) (n_ref_used * n_codes), state->ref.tokens.end());
|
||||
}
|
||||
n_frames = in_codes.size() / n_codes;
|
||||
}
|
||||
|
||||
params.codes = has_codes ? &in_codes : nullptr;
|
||||
params.feats = has_feats ? &in_feats : nullptr;
|
||||
if (!sctx->encode(¶ms)) {
|
||||
LOG_ERR("%s: encode failed (code2wav)\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (n_ref_used > 0) {
|
||||
const size_t hop = state->out_audio.size() / n_frames;
|
||||
state->out_audio.erase(state->out_audio.begin(), state->out_audio.begin() + (std::ptrdiff_t) (n_ref_used * hop));
|
||||
}
|
||||
|
||||
out->audio = state->out_audio.data();
|
||||
out->n_samples = state->out_audio.size();
|
||||
|
||||
@@ -4103,6 +4253,14 @@ void mtmd_bitmap_set_is_video(mtmd_bitmap * bitmap, bool is_video) {
|
||||
bitmap->is_video = is_video;
|
||||
}
|
||||
|
||||
void mtmd_bitmap_set_transcript(mtmd_bitmap * bitmap, const char * transcript) {
|
||||
if (transcript) {
|
||||
bitmap->transcript = std::string(transcript);
|
||||
} else {
|
||||
bitmap->transcript.clear();
|
||||
}
|
||||
}
|
||||
|
||||
mtmd_bitmap * mtmd_bitmap_init_lazy(const mtmd_context * ctx,
|
||||
const char * id,
|
||||
void * user_data,
|
||||
|
||||
@@ -218,6 +218,8 @@ MTMD_API void mtmd_bitmap_set_mergeable(mtmd_bitmap * bitmap, bool merge
|
||||
// on a lazy bitmap: the expanded chunks are one video, wrapped by the model's video begin/end tokens
|
||||
// with MTMD_VIDEO_TEMPLATE_LFM3_OMNI, its audio bitmaps are the pieces of one soundtrack, in order
|
||||
MTMD_API void mtmd_bitmap_set_is_video(mtmd_bitmap * bitmap, bool is_video);
|
||||
// transcript of an audio bitmap is optional, some TTS models use it to clone the speaker voice more closely (--tts-speaker-text)
|
||||
MTMD_API void mtmd_bitmap_set_transcript(mtmd_bitmap * bitmap, const char * transcript);
|
||||
|
||||
// mtmd_bitmap lazy
|
||||
//
|
||||
@@ -476,6 +478,7 @@ enum mtmd_gen_audio_type {
|
||||
MTMD_GEN_AUDIO_TYPE_YUE2,
|
||||
MTMD_GEN_AUDIO_TYPE_MINIMAX_MUSIC3,
|
||||
MTMD_GEN_AUDIO_TYPE_FISH_S2,
|
||||
MTMD_GEN_AUDIO_TYPE_KOKORO,
|
||||
};
|
||||
|
||||
struct mtmd_gen_audio_info {
|
||||
@@ -524,10 +527,12 @@ struct mtmd_gen_inp {
|
||||
// pass either codes (discrete) or feats (continuous), depending on the pipeline
|
||||
// lfm3a GEN_CODE also takes codes: the codes revealed so far, -1 for a masked codebook
|
||||
// zonos3 GEN_WAV: -1 for a residual code not known yet, the frame is decoded without it
|
||||
// kokoro GEN_WAV takes both: codes are the phoneme tokens, feats is the voice row (256 values) then the backbone state of each token
|
||||
int32_t * codes;
|
||||
size_t n_codes;
|
||||
const float * feats;
|
||||
size_t n_feats;
|
||||
float speed; // kokoro: speaking speed, > 0 (default: 1.0)
|
||||
|
||||
// yue2 GEN_CODE: n_codes is the frame count, codes is not read (the backbone state holds them)
|
||||
// backbone_state: llama_state_seq_get_data() of the backbone_n_past positions (prompt, codes, end token)
|
||||
|
||||
@@ -862,11 +862,13 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For the list
|
||||
|
||||
`lang`: Language code for the utterance (model-dependent, e.g. `en`, `zh`). Optional.
|
||||
|
||||
`raw_prompt`: If `true`, `input` is passed to the model as is, without the text preprocessing of the model's `tts` chat template (prompt format, text normalization, text to phonemes). For example, a Kokoro model then takes a phoneme string. Default: `false`
|
||||
|
||||
`voice`: Voice for the utterance, one of the `voices` listed by `GET /props` (model-dependent, e.g. `female` or `male` for LFM3-Audio). Optional: a model with predefined voices speaks with the first of them when neither `voice` nor `speaker_ref` is given. It rejects any other name, and a predefined voice cannot be combined with `speaker_ref`.
|
||||
|
||||
`speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence. Rejected when the model has no voice encoder (e.g. Pocket TTS without voice cloning).
|
||||
|
||||
`speaker_ref_text`: Transcript of the reference audio, put in the prompt by some models (e.g. Fish Audio S2 Pro). Optional.
|
||||
`speaker_ref_text`: Transcript of the reference audio, put in the prompt by some models (e.g. Fish Audio S2 Pro, Qwen3-TTS). Optional.
|
||||
|
||||
`top_k`, `top_p`: Sampling params for both the backbone sampler and the acoustic code predictor. Optional, `0` or absent keeps the server `--top-k`/`--top-p` for the backbone and the model defaults for the code predictor.
|
||||
|
||||
@@ -1678,7 +1680,7 @@ Generates speech from text, for models that can generate audio. The request is c
|
||||
|
||||
`stream_format`: `audio` streams the response as it becomes available, see `stream` in `/tts` for the WAV header it writes. `sse` is rejected. The response is not streamed when omitted.
|
||||
|
||||
`speed`: Speaking speed, see `speed` in `/tts`. Only models that support it accept a value other than `1.0`. `loudness`, `cfg_scale` and `flow_steps` of `/tts` can be used as well.
|
||||
`speed`: Speaking speed, see `speed` in `/tts`. Only models that support it accept a value other than `1.0`. `loudness`, `cfg_scale`, `flow_steps`, `lang` and `raw_prompt` of `/tts` can be used as well.
|
||||
|
||||
`instructions`: Ignored.
|
||||
|
||||
|
||||
@@ -6984,7 +6984,9 @@ std::unique_ptr<server_res_generator> server_routes::handle_tts_impl(const serve
|
||||
// speaker_ref in the request overrides the default voice
|
||||
const bool has_speaker_ref = req.files.count("speaker_ref") > 0 || !json_value(body, "speaker_ref_b64", std::string()).empty();
|
||||
const std::string voice = json_value(body, "voice", has_speaker_ref ? std::string() : params.tts_voice);
|
||||
const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
|
||||
// raw_prompt: the input goes to the model as is (e.g. phonemes), the "tts" chat template is skipped
|
||||
const bool raw_prompt = json_value(body, "raw_prompt", false);
|
||||
const std::string tts_prompt = raw_prompt ? std::string() : common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
|
||||
// a model with predefined voices only speaks with one of them
|
||||
mtmd::bitmap_ptr voice_ref;
|
||||
if (!voice.empty() && info.n_voices > 0) {
|
||||
|
||||
@@ -41,3 +41,22 @@ def test_ignore_eos_false_no_logit_bias():
|
||||
assert res.status_code == 200
|
||||
logit_bias = res.body["generation_settings"]["logit_bias"]
|
||||
assert len(logit_bias) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("constraint", [
|
||||
{"grammar": 'root ::= "ok"'},
|
||||
{"json_schema": {"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}},
|
||||
])
|
||||
def test_ignore_eos_with_grammar_rejected(constraint):
|
||||
"""ignore_eos is not supported together with a grammar"""
|
||||
global server
|
||||
server.start()
|
||||
res = server.make_request("POST", "/completion", data={
|
||||
"n_predict": 8,
|
||||
"prompt": "Once upon a time",
|
||||
"ignore_eos": True,
|
||||
"temperature": 0.0,
|
||||
**constraint,
|
||||
})
|
||||
assert res.status_code == 400, res.body
|
||||
assert res.body["error"]["type"] == "invalid_request_error"
|
||||
|
||||
+7
-1
@@ -135,7 +135,13 @@ int main(int argc, char ** argv) {
|
||||
|
||||
// a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it
|
||||
auto tmpls = common_chat_templates_init(model, "");
|
||||
std::string prompt = common_chat_get_tts_prompt(tmpls.get(), params.prompt, params.tts_lang, params.tts_voice);
|
||||
std::string prompt;
|
||||
try {
|
||||
prompt = common_chat_get_tts_prompt(tmpls.get(), params.prompt, params.tts_lang, params.tts_voice);
|
||||
} catch (const std::exception & e) {
|
||||
LOG_ERR("failed to prepare the text to speak: %s\n", e.what());
|
||||
return 1;
|
||||
}
|
||||
if (prompt.empty()) {
|
||||
prompt = params.prompt;
|
||||
}
|
||||
|
||||
+8
-1
@@ -41,7 +41,7 @@
|
||||
let editor: MarkdownEditor | undefined = $state();
|
||||
let box: HTMLDivElement | undefined = $state();
|
||||
|
||||
let livePreview = $derived(settingsStore.config[SETTINGS_KEYS.LIVE_MARKDOWN_PREVIEW] !== false);
|
||||
let livePreview = $derived(settingsStore.config[SETTINGS_KEYS.LIVE_MARKDOWN_PREVIEW] === true);
|
||||
|
||||
// Rendered mention badges label paths by these, so the blocks render again on a change
|
||||
let renderKey = $derived(
|
||||
@@ -76,6 +76,13 @@
|
||||
|
||||
return event.defaultPrevented;
|
||||
}
|
||||
}),
|
||||
// CodeMirror turns spellcheck, autocorrect and autocapitalize off for code; the
|
||||
// message input is prose, so it gets a textarea's defaults back
|
||||
EditorView.contentAttributes.of({
|
||||
autocapitalize: 'on',
|
||||
autocorrect: 'on',
|
||||
spellcheck: 'true'
|
||||
})
|
||||
];
|
||||
|
||||
|
||||
@@ -126,7 +126,7 @@ export const SETTINGS_REGISTRY: SettingsSectionEntry[] = [
|
||||
type: SettingsFieldType.CHECKBOX
|
||||
},
|
||||
{
|
||||
defaultValue: true,
|
||||
defaultValue: false,
|
||||
help: 'Render the markdown in the message input as you write it: every line but the one you are on shows as it will look in the conversation. When disabled, the input shows the plain markdown text.',
|
||||
key: SETTINGS_KEYS.LIVE_MARKDOWN_PREVIEW,
|
||||
label: 'Live markdown preview in the message input',
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Guards the chat form input contract: a live-preview markdown editor whose value
|
||||
// stays the exact markdown source, with mention links shown as badges while their
|
||||
// block is edited and every block rendered once the editor loses focus. The form's
|
||||
// Guards the chat form input contract: a markdown editor whose value stays the exact
|
||||
// markdown source, with mention links shown as badges while their block is edited and,
|
||||
// with the live preview setting on, every block rendered once the editor loses focus. The form's
|
||||
// key and paste handlers run first and win when they handle the event.
|
||||
|
||||
import { EditorView } from '@codemirror/view';
|
||||
@@ -80,20 +80,24 @@ describe('ChatFormInput', () => {
|
||||
expect(editor.state.doc.toString()).toBe('a ');
|
||||
});
|
||||
|
||||
it('renders every block like a chat message once blurred', async () => {
|
||||
const screen = render(ChatFormInput, { value: '# Title\n\nSome **bold** text' });
|
||||
const editor = editorIn(screen.container);
|
||||
it('renders every block like a chat message once blurred, with the live preview on', async () => {
|
||||
settingsStore.updateConfig(SETTINGS_KEYS.LIVE_MARKDOWN_PREVIEW, true);
|
||||
|
||||
editor.contentDOM.blur();
|
||||
await vi.waitFor(() => {
|
||||
expect(screen.container.querySelector('.cm-md-block h1')?.textContent).toBe('Title');
|
||||
expect(screen.container.querySelector('.cm-md-block strong')?.textContent).toBe('bold');
|
||||
});
|
||||
try {
|
||||
const screen = render(ChatFormInput, { value: '# Title\n\nSome **bold** text' });
|
||||
const editor = editorIn(screen.container);
|
||||
|
||||
editor.contentDOM.blur();
|
||||
await vi.waitFor(() => {
|
||||
expect(screen.container.querySelector('.cm-md-block h1')?.textContent).toBe('Title');
|
||||
expect(screen.container.querySelector('.cm-md-block strong')?.textContent).toBe('bold');
|
||||
});
|
||||
} finally {
|
||||
settingsStore.updateConfig(SETTINGS_KEYS.LIVE_MARKDOWN_PREVIEW, false);
|
||||
}
|
||||
});
|
||||
|
||||
it('shows the plain markdown, with mention badges, while the live preview setting is off', async () => {
|
||||
settingsStore.updateConfig(SETTINGS_KEYS.LIVE_MARKDOWN_PREVIEW, false);
|
||||
|
||||
it('shows the plain markdown, with mention badges, by default', async () => {
|
||||
try {
|
||||
const screen = render(ChatFormInput, { value: `# Title\n\nsee ${MENTION}` });
|
||||
const editor = editorIn(screen.container);
|
||||
@@ -114,10 +118,19 @@ describe('ChatFormInput', () => {
|
||||
expect(screen.container.querySelector('.cm-md-block h1')?.textContent).toBe('Title')
|
||||
);
|
||||
} finally {
|
||||
settingsStore.updateConfig(SETTINGS_KEYS.LIVE_MARKDOWN_PREVIEW, true);
|
||||
settingsStore.updateConfig(SETTINGS_KEYS.LIVE_MARKDOWN_PREVIEW, false);
|
||||
}
|
||||
});
|
||||
|
||||
it('spellchecks, autocorrects and autocapitalizes like a textarea', () => {
|
||||
const screen = render(ChatFormInput, { value: '' });
|
||||
const content = editorIn(screen.container).contentDOM;
|
||||
|
||||
expect(content.getAttribute('spellcheck')).toBe('true');
|
||||
expect(content.getAttribute('autocorrect')).toBe('on');
|
||||
expect(content.getAttribute('autocapitalize')).toBe('on');
|
||||
});
|
||||
|
||||
it('lets the form handle a key first and keeps it from the editor', async () => {
|
||||
const onKeydown = vi.fn((event: KeyboardEvent) => {
|
||||
if (event.key === 'Enter') event.preventDefault();
|
||||
|
||||
@@ -70,3 +70,22 @@ shows in the app at once and the app keeps no copy of a component or a token. Fo
|
||||
- `npm run dev`: a small SvelteKit check page (`src/routes`)
|
||||
- `npm run check`: svelte-check
|
||||
- `npm run prepack`: build the library into `dist/` and run publint
|
||||
- `node scripts/npm-version-check.mjs`: what the release workflow would make of this folder, against the real registry
|
||||
|
||||
## Releases
|
||||
|
||||
The package is published from here by
|
||||
[`ui-lib-publish.yml`](../../../.github/workflows/ui-lib-publish.yml), which runs when a commit reaches `master` with
|
||||
`ui/lib/ui` in it. The version in `package.json` is the release: land a bump together with the change that should ship
|
||||
and the workflow type-checks, builds, lints and publishes that version to npm, with a provenance attestation. A library
|
||||
change that carries no bump publishes nothing and leaves a warning on the run.
|
||||
|
||||
- `npm version <patch|minor> --no-git-tag-version` and commit: it moves `package.json` and both version fields of the
|
||||
lockfile together.
|
||||
- Below `1.0`: a breaking change bumps the minor, a feature bumps the patch.
|
||||
- A version with a prerelease suffix (`0.5.0-beta.1`) publishes under the `next` tag, so `latest` stays a stable build.
|
||||
- Publishing needs no token in the repo: the job trades its OIDC identity for one, so the only place to look when a
|
||||
publish is refused is the package's trusted-publisher settings on npm. A publisher names a workflow that must already
|
||||
exist under `.github/workflows`, so it is added after this workflow lands, and it allows staged publishing until
|
||||
`npm publish` is permitted for it. Provenance is generated without asking, and only on a GitHub-hosted runner: a
|
||||
self-hosted one cannot publish this way.
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@ggml-org/llama-ui",
|
||||
"version": "0.3.0",
|
||||
"version": "0.4.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@ggml-org/llama-ui",
|
||||
"version": "0.3.0",
|
||||
"version": "0.4.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@ggml-org/llama-brand": "https://codeload.github.com/ggml-org/llama-brand/tar.gz/f05540643c1ca2a18d732ed6491c839686da9a3a",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@ggml-org/llama-ui",
|
||||
"version": "0.3.0",
|
||||
"version": "0.4.0",
|
||||
"description": "Shared UI library for llama.cpp frontend apps.",
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env node
|
||||
// Decides whether this library is a version npm does not have yet, for
|
||||
// .github/workflows/ui-lib-publish.yml. It reads package.json next to its own folder, so it
|
||||
// runs from anywhere.
|
||||
//
|
||||
// The version in package.json is the release: this takes it, asks the registry what it holds,
|
||||
// and says whether to publish, under which dist-tag, and why not when not. The version already
|
||||
// being on npm is the case that let a release drift once (the library moved, the version did
|
||||
// not), so it is reported as a verdict rather than passed over in silence.
|
||||
//
|
||||
// It writes its findings to $GITHUB_OUTPUT when that is set, and always exits 0 unless the
|
||||
// registry could not be read: the workflow decides between warning and failing.
|
||||
|
||||
import { readFileSync, appendFileSync } from 'node:fs';
|
||||
|
||||
const REGISTRY = process.env.NPM_REGISTRY_URL ?? 'https://registry.npmjs.org';
|
||||
|
||||
const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
|
||||
const { name, version } = pkg;
|
||||
|
||||
if (!name || !version) {
|
||||
console.error(`package.json holds no name or version (got ${name}@${version})`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// @scoped/name goes in the registry path with only the slash escaped
|
||||
const url = `${REGISTRY}/${name.replace('/', '%2F')}`;
|
||||
let doc;
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
if (res.status === 404) {
|
||||
doc = null; // never published
|
||||
} else if (!res.ok) {
|
||||
console.error(`${url} answered ${res.status}`);
|
||||
process.exit(1);
|
||||
} else {
|
||||
doc = await res.json();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`${url} could not be read: ${err.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const latest = doc?.['dist-tags']?.latest ?? null;
|
||||
const publishedHere = Boolean(doc?.versions?.[version]);
|
||||
// a hyphen marks a prerelease; a plus is build metadata, which is not one
|
||||
const prerelease = version.split('+')[0].includes('-');
|
||||
|
||||
let verdict = 'new';
|
||||
let shouldPublish = true;
|
||||
let note = `${name}@${version} is not on npm${latest ? `, which holds ${latest}` : ''}`;
|
||||
|
||||
if (publishedHere) {
|
||||
verdict = 'published';
|
||||
shouldPublish = false;
|
||||
note = `${name}@${version} is already on npm. Bump the version in ui/lib/ui/package.json to release these changes.`;
|
||||
} else if (latest && compare(version, latest) < 0) {
|
||||
// covers the prerelease of an already-released core: 0.4.0-beta.1 behind 0.4.0
|
||||
verdict = 'stale';
|
||||
shouldPublish = false;
|
||||
note = `${name}@${version} is older than the ${latest} on npm.`;
|
||||
}
|
||||
|
||||
const tag = prerelease ? 'next' : 'latest';
|
||||
|
||||
console.log(`package.json : ${name}@${version}`);
|
||||
console.log(`npm latest : ${latest ?? 'nothing published yet'}`);
|
||||
console.log(`that version : ${publishedHere ? 'already on npm' : 'not on npm'}`);
|
||||
console.log(`verdict : ${verdict} (publish: ${shouldPublish}, dist-tag: ${tag})`);
|
||||
console.log(` ${note}`);
|
||||
|
||||
if (process.env.GITHUB_OUTPUT) {
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
[
|
||||
`name=${name}`,
|
||||
`version=${version}`,
|
||||
`latest=${latest ?? ''}`,
|
||||
`verdict=${verdict}`,
|
||||
`should_publish=${shouldPublish}`,
|
||||
`tag=${tag}`,
|
||||
`note=${note}`,
|
||||
'',
|
||||
].join('\n'),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Order two semver versions: negative when a < b, positive when a > b, 0 when they are equal.
|
||||
* Written out rather than pulled in as a dependency for a CI check: prerelease identifiers
|
||||
* order before the release they lead up to (0.4.0-beta.1 < 0.4.0), fewer identifiers sort
|
||||
* first when one is a prefix of the other, numeric identifiers order before alphanumeric ones,
|
||||
* and build metadata is ignored (semver 11.4.x of the specification).
|
||||
*/
|
||||
function compare(a, b) {
|
||||
const [coreA, preA] = split(a);
|
||||
const [coreB, preB] = split(b);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
if (coreA[i] !== coreB[i]) return coreA[i] < coreB[i] ? -1 : 1;
|
||||
}
|
||||
if (preA.length === 0 && preB.length === 0) return 0;
|
||||
if (preA.length === 0) return 1;
|
||||
if (preB.length === 0) return -1;
|
||||
for (let i = 0; ; i++) {
|
||||
const x = preA[i];
|
||||
const y = preB[i];
|
||||
if (x === undefined && y === undefined) return 0;
|
||||
if (x === undefined) return -1;
|
||||
if (y === undefined) return 1;
|
||||
if (x === y) continue;
|
||||
const numX = /^\d+$/.test(x);
|
||||
const numY = /^\d+$/.test(y);
|
||||
if (numX && numY) return Number(x) < Number(y) ? -1 : 1;
|
||||
if (numX) return -1;
|
||||
if (numY) return 1;
|
||||
return x < y ? -1 : 1;
|
||||
}
|
||||
}
|
||||
|
||||
/** Split a version into its numeric core and its prerelease identifiers, dropping build metadata. */
|
||||
function split(version) {
|
||||
const withoutBuild = version.split('+')[0];
|
||||
const dash = withoutBuild.indexOf('-');
|
||||
const core = (dash === -1 ? withoutBuild : withoutBuild.slice(0, dash)).split('.').map((n) => Number(n) || 0);
|
||||
while (core.length < 3) core.push(0);
|
||||
const pre = dash === -1 ? [] : withoutBuild.slice(dash + 1).split('.');
|
||||
return [core, pre];
|
||||
}
|
||||
Reference in New Issue
Block a user