mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-19 01:07:40 -06:00
Compare commits
150
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
48a32194ee | ||
|
|
6da36139e8 | ||
|
|
61f182fcde | ||
|
|
03ec5b4326 | ||
|
|
7144124a5b | ||
|
|
43bd7a2a42 | ||
|
|
65758d01f5 | ||
|
|
a300a9c334 | ||
|
+4 |
4577932fab | ||
|
|
398f00d657 | ||
|
|
9e93cd8001 | ||
|
|
aaa807c568 | ||
|
|
401563f10e | ||
|
|
52975342ff | ||
|
|
3574df84b7 | ||
|
|
2e53a0bab7 | ||
|
|
744c68cd14 | ||
|
|
307e3f1551 | ||
|
|
9048bad8d6 | ||
|
|
a9d43ecc30 | ||
|
|
8c16babba5 | ||
|
|
ad9d9e8f50 | ||
|
|
955c4e8ee5 | ||
|
|
10152bc4a0 | ||
|
|
8777c0c833 | ||
|
|
918076a635 | ||
|
|
ebc9fb933e | ||
|
|
8b8ca4c98e | ||
|
|
08a9b35893 | ||
|
|
acad5e405c | ||
|
|
d3064fddce | ||
|
|
7f106b5459 | ||
|
|
d92ea9bb12 | ||
|
|
8d1599fca1 | ||
|
|
054a1babf3 | ||
|
|
513ad7ae07 | ||
|
|
67cf8967eb | ||
|
|
4df3f2959c | ||
|
|
a49db739fb | ||
|
|
eada89c315 | ||
|
|
8f647f1fdb | ||
|
|
dde9de82cb | ||
|
|
e0a9118fe9 | ||
|
|
140a279874 | ||
|
|
c145db2728 | ||
|
|
ede6106780 | ||
|
|
9214bdb554 | ||
|
|
d3c3e242c1 | ||
|
|
9581154a22 | ||
|
|
1ef8c5d684 | ||
|
|
e8fbcad855 | ||
|
|
95e837aaf1 | ||
|
|
d998c676ff | ||
|
|
92192d31e4 | ||
|
|
ad5cbfa911 | ||
|
|
d261e49e7e | ||
|
|
4ddf2b9470 | ||
|
|
4303cd94ae | ||
|
|
c7518dbec5 | ||
|
|
a68c6136bb | ||
|
|
3a7660371d | ||
|
|
e8698ac163 | ||
|
|
7804be0634 | ||
|
|
ef6bc6f367 | ||
|
|
a06044b934 | ||
|
|
62a6d3a302 | ||
|
|
ef3848f9d7 | ||
|
|
47fe133318 | ||
|
|
301db63370 | ||
|
|
a001af915e | ||
|
|
164ab018f9 | ||
|
|
5cb9516798 | ||
|
|
8bfd64b5ac | ||
|
|
e1925b3695 | ||
|
|
95db855bce | ||
|
|
1d2e880bad | ||
|
|
adf554d239 | ||
|
|
6b01a4767f | ||
|
|
f477c1b90a | ||
|
|
21fa9c043f | ||
|
|
a946eb541a | ||
|
|
05db09a4d3 | ||
|
|
9fba51d2c9 | ||
|
|
b4c0e00a4f | ||
|
|
1719dbb0ee | ||
|
|
fc504f91d8 | ||
|
|
18b74fa41d | ||
|
|
7ae3c6b231 | ||
|
|
f4d0673917 | ||
|
|
a681a53b60 | ||
|
|
75fd7f9ec9 | ||
|
|
a68799c609 | ||
|
|
56fd8a375f | ||
|
|
b54752df40 | ||
|
|
900f01ba09 | ||
|
|
de32b268c4 | ||
|
|
4e5116f476 | ||
|
|
278a89cb78 | ||
|
|
dd099a6bf3 | ||
|
|
73f53317eb | ||
|
|
429460dc0e | ||
|
|
58a4a6b49c | ||
|
|
5a4afd8077 | ||
|
|
6d3568e31d | ||
|
|
e574a17b06 | ||
|
|
5c008e0008 | ||
|
|
9f26fe11c7 | ||
|
|
cf9a9957da | ||
|
|
dca216ad63 | ||
|
|
f8cc5774e7 | ||
|
|
acf5d54562 | ||
|
|
3e41ae69db | ||
|
|
3bc612fe23 | ||
|
|
d8f30a5b6d | ||
|
|
e964175653 | ||
|
|
d20426bbbe | ||
|
|
19f31e61b0 | ||
|
|
07037ba133 | ||
|
|
0cc522e45d | ||
|
|
f08bca8a46 | ||
|
|
3c311796f2 | ||
|
|
8b4ac2496c | ||
|
|
1ae047c926 | ||
|
|
1c2adc75f6 | ||
|
|
a956a852c9 | ||
|
|
c6c0ec6d5c | ||
|
|
d0764c5ee2 | ||
|
|
3549c274dc | ||
|
|
c56be864f4 | ||
|
|
a9a5d216e6 | ||
|
|
9d925e7c02 | ||
|
|
e3483e40b0 | ||
|
|
4f2187e73d | ||
|
|
0cce7580b8 | ||
|
|
32bdc3a89e | ||
|
|
61dbf49228 | ||
|
|
9d25e0014c | ||
|
|
9183a552e5 | ||
|
|
ded522f830 | ||
|
|
5193ef1d4f | ||
|
|
4df9b47976 | ||
|
|
b79696f1f5 | ||
|
|
4f9dfa0e36 | ||
|
|
9daa0aa22b | ||
|
|
fe14cad51b | ||
|
|
31953bcf74 | ||
|
|
ae93877c8f | ||
|
|
7874988b45 | ||
|
|
8c2c4c39c5 | ||
|
|
108f73adfe |
@@ -19,12 +19,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${CANN_BASE_IMAGE} AS build
|
||||
@@ -40,7 +43,7 @@ WORKDIR /app
|
||||
# -- Copy project files --
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
# -- Set CANN environment variables (required for compilation) --
|
||||
# Using ENV instead of `source` allows environment variables to persist across the entire image layer
|
||||
|
||||
@@ -9,12 +9,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -28,7 +31,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON; \
|
||||
|
||||
@@ -17,12 +17,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_CUDA_DEV_CONTAINER} AS build
|
||||
@@ -40,7 +43,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "${CUDA_DOCKER_ARCH}" != "default" ]; then \
|
||||
export CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CUDA_DOCKER_ARCH}"; \
|
||||
|
||||
@@ -11,12 +11,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build
|
||||
@@ -41,7 +44,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "${GGML_SYCL_F16}" = "ON" ]; then \
|
||||
echo "GGML_SYCL_F16 is set" \
|
||||
|
||||
@@ -15,12 +15,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_MUSA_DEV_CONTAINER} AS build
|
||||
@@ -45,7 +48,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "${MUSA_DOCKER_ARCH}" != "default" ]; then \
|
||||
export CMAKE_ARGS="-DMUSA_ARCHITECTURES=${MUSA_DOCKER_ARCH}"; \
|
||||
|
||||
+138
-41
@@ -22,12 +22,19 @@
|
||||
spirv-headers,
|
||||
nodejs,
|
||||
importNpmLock,
|
||||
syclToolkit ? null,
|
||||
level-zero,
|
||||
ocl-icd,
|
||||
intel-graphics-compiler,
|
||||
intel-compute-runtime,
|
||||
makeBinaryWrapper,
|
||||
useBlas ?
|
||||
builtins.all (x: !x) [
|
||||
useCuda
|
||||
useMetalKit
|
||||
useRocm
|
||||
useVulkan
|
||||
useSycl
|
||||
]
|
||||
&& blas.meta.available,
|
||||
useCuda ? config.cudaSupport,
|
||||
@@ -38,11 +45,28 @@
|
||||
rocmGpuTargets ? builtins.concatStringsSep ";" rocmPackages.clr.gpuTargets,
|
||||
useVulkan ? false,
|
||||
useRpc ? false,
|
||||
useSycl ? false,
|
||||
# Rebuild, not a runtime switch. cmake default is off.
|
||||
syclF16 ? false,
|
||||
# null keeps JIT. AOT makes icpx run ocloc, which DPC++ does not ship.
|
||||
syclDeviceArch ? null,
|
||||
syclMaxParallelLinkJobs ? null,
|
||||
llamaVersion ? "0.0.0", # Arbitrary version, substituted by the flake
|
||||
|
||||
# It's necessary to consistently use backendStdenv when building with CUDA support,
|
||||
# otherwise we get libstdc++ errors downstream.
|
||||
effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv,
|
||||
# CUDA needs its backend stdenv or libstdc++ mismatches downstream.
|
||||
# SYCL needs icpx which is provided by the oneAPI stdenv
|
||||
effectiveStdenv ?
|
||||
if useCuda && useSycl then
|
||||
throw "llama-cpp: useCuda and useSycl are mutually exclusive"
|
||||
else if useCuda then
|
||||
cudaPackages.backendStdenv
|
||||
else if useSycl then
|
||||
if syclToolkit == null then
|
||||
throw "llama-cpp: useSycl requires syclToolkit, consider setting it to intel-oneapi-toolkit"
|
||||
else
|
||||
syclToolkit.stdenv
|
||||
else
|
||||
stdenv,
|
||||
enableStatic ? effectiveStdenv.hostPlatform.isStatic,
|
||||
precompileMetalShaders ? false,
|
||||
useWebUi ? true,
|
||||
@@ -65,7 +89,8 @@ let
|
||||
++ lib.optionals useMetalKit [ "MetalKit" ]
|
||||
++ lib.optionals useMpi [ "MPI" ]
|
||||
++ lib.optionals useRocm [ "ROCm" ]
|
||||
++ lib.optionals useVulkan [ "Vulkan" ];
|
||||
++ lib.optionals useVulkan [ "Vulkan" ]
|
||||
++ lib.optionals useSycl [ "SYCL" ];
|
||||
|
||||
pnameSuffix =
|
||||
strings.optionalString (suffices != [ ])
|
||||
@@ -108,6 +133,26 @@ let
|
||||
shaderc
|
||||
spirv-headers
|
||||
];
|
||||
|
||||
syclAot = useSycl && syclDeviceArch != null && syclDeviceArch != "";
|
||||
|
||||
# source ocloc binary from intel-compute-runtime
|
||||
oclocAot = runCommand "ocloc-aot" { nativeBuildInputs = [ makeBinaryWrapper ]; } ''
|
||||
mkdir -p "$out/bin"
|
||||
shopt -s nullglob
|
||||
exact=(${lib.getBin intel-compute-runtime}/bin/ocloc)
|
||||
vers=(${lib.getBin intel-compute-runtime}/bin/ocloc-*)
|
||||
if [ ''${#exact[@]} -eq 1 ]; then
|
||||
src=''${exact[0]}
|
||||
elif [ ''${#vers[@]} -eq 1 ]; then
|
||||
src=''${vers[0]}
|
||||
else
|
||||
echo "expected one ocloc binary from intel-compute-runtime" >&2
|
||||
exit 1
|
||||
fi
|
||||
makeWrapper "$src" "$out/bin/ocloc" \
|
||||
--prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath [ intel-graphics-compiler ]}
|
||||
'';
|
||||
in
|
||||
|
||||
effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
@@ -133,11 +178,13 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
src = lib.cleanSource ../../.;
|
||||
};
|
||||
|
||||
# Builds the webui locally, taking care not to require updating any sha256 hash.
|
||||
# Builds Llama Chat locally, taking care not to require updating any sha256 hash.
|
||||
webui = stdenvNoCC.mkDerivation {
|
||||
pname = "webui";
|
||||
version = llamaVersion;
|
||||
src = lib.cleanSource ../../tools/ui;
|
||||
# the app compiles Llama UI from its source in ../../lib/ui, so the source is ui/ as a whole
|
||||
src = lib.cleanSource ../../ui;
|
||||
sourceRoot = "source/apps/chat";
|
||||
|
||||
nativeBuildInputs = [
|
||||
nodejs
|
||||
@@ -146,7 +193,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
|
||||
# no sha256 required when using buildNodeModules
|
||||
npmDeps = importNpmLock.buildNodeModules {
|
||||
npmRoot = ../../tools/ui;
|
||||
npmRoot = ../../ui/apps/chat;
|
||||
inherit nodejs;
|
||||
};
|
||||
|
||||
@@ -156,8 +203,8 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
};
|
||||
|
||||
postPatch = lib.optionalString useWebUi ''
|
||||
cp -r ${finalAttrs.webui} tools/ui/dist
|
||||
chmod -R u+w tools/ui/dist
|
||||
cp -r ${finalAttrs.webui} ui/apps/chat/dist
|
||||
chmod -R u+w ui/apps/chat/dist
|
||||
'';
|
||||
|
||||
# With PR#6015 https://github.com/ggml-org/llama.cpp/pull/6015,
|
||||
@@ -168,6 +215,13 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
# see https://github.com/ggml-org/llama.cpp/pull/6118 for discussion
|
||||
__noChroot = effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders;
|
||||
|
||||
# Do not fortify under SYCL as it breaks compiled memcpy invocations
|
||||
# _FORTIFY_SOURCE rewrites memcpy -> __memcpy_chk
|
||||
hardeningDisable = optionals useSycl [
|
||||
"fortify"
|
||||
"fortify3"
|
||||
];
|
||||
|
||||
nativeBuildInputs =
|
||||
[
|
||||
cmake
|
||||
@@ -181,7 +235,12 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
autoAddDriverRunpath
|
||||
]
|
||||
++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ]
|
||||
++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
|
||||
++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ]
|
||||
++ optionals useSycl [
|
||||
autoAddDriverRunpath
|
||||
makeBinaryWrapper
|
||||
]
|
||||
++ optionals syclAot [ oclocAot ];
|
||||
|
||||
buildInputs =
|
||||
optionals effectiveStdenv.hostPlatform.isDarwin darwinBuildInputs
|
||||
@@ -190,39 +249,53 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
++ optionals useRocm rocmBuildInputs
|
||||
++ optionals useBlas [ blas ]
|
||||
++ optionals useVulkan vulkanBuildInputs
|
||||
++ optionals useSycl [
|
||||
level-zero
|
||||
ocl-icd
|
||||
intel-graphics-compiler
|
||||
]
|
||||
++ [ openssl ];
|
||||
|
||||
cmakeFlags =
|
||||
[
|
||||
(cmakeBool "LLAMA_BUILD_SERVER" true)
|
||||
(cmakeBool "LLAMA_BUILD_WEBUI" useWebUi)
|
||||
(cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
|
||||
(cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
|
||||
(cmakeBool "GGML_NATIVE" false)
|
||||
(cmakeBool "GGML_BLAS" useBlas)
|
||||
(cmakeBool "GGML_CUDA" useCuda)
|
||||
(cmakeBool "GGML_HIP" useRocm)
|
||||
(cmakeBool "GGML_METAL" useMetalKit)
|
||||
(cmakeBool "GGML_VULKAN" useVulkan)
|
||||
(cmakeBool "GGML_STATIC" enableStatic)
|
||||
(cmakeBool "GGML_RPC" useRpc)
|
||||
]
|
||||
++ optionals useCuda [
|
||||
(
|
||||
with cudaPackages.flags;
|
||||
cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
|
||||
builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
|
||||
)
|
||||
cmakeFlags = [
|
||||
(cmakeBool "LLAMA_BUILD_SERVER" true)
|
||||
(cmakeBool "LLAMA_BUILD_WEBUI" useWebUi)
|
||||
(cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
|
||||
(cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
|
||||
(cmakeBool "GGML_NATIVE" false)
|
||||
(cmakeBool "GGML_BLAS" useBlas)
|
||||
(cmakeBool "GGML_CUDA" useCuda)
|
||||
(cmakeBool "GGML_HIP" useRocm)
|
||||
(cmakeBool "GGML_METAL" useMetalKit)
|
||||
(cmakeBool "GGML_VULKAN" useVulkan)
|
||||
(cmakeBool "GGML_STATIC" enableStatic)
|
||||
(cmakeBool "GGML_RPC" useRpc)
|
||||
]
|
||||
++ optionals useCuda [
|
||||
(
|
||||
with cudaPackages.flags;
|
||||
cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
|
||||
builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
|
||||
)
|
||||
]
|
||||
++ optionals useRocm [
|
||||
(cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
|
||||
(cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmGpuTargets)
|
||||
]
|
||||
++ optionals useMetalKit [
|
||||
(lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
|
||||
(cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
|
||||
];
|
||||
)
|
||||
]
|
||||
++ optionals useRocm [
|
||||
(cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
|
||||
(cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmGpuTargets)
|
||||
]
|
||||
++ optionals useMetalKit [
|
||||
(lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
|
||||
(cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
|
||||
]
|
||||
++ optionals useSycl [
|
||||
(cmakeBool "GGML_SYCL" true)
|
||||
(cmakeBool "GGML_SYCL_F16" syclF16)
|
||||
]
|
||||
++ optionals syclAot [
|
||||
(cmakeFeature "GGML_SYCL_DEVICE_ARCH" syclDeviceArch)
|
||||
]
|
||||
++ optionals (syclAot && syclMaxParallelLinkJobs != null) [
|
||||
(cmakeFeature "GGML_SYCL_MAX_PARALLEL_LINK_JOBS" (toString syclMaxParallelLinkJobs))
|
||||
];
|
||||
|
||||
# Environment variables needed for ROCm
|
||||
env = optionalAttrs useRocm {
|
||||
@@ -237,11 +310,35 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
cp $src/include/llama.h $out/include/
|
||||
'';
|
||||
|
||||
# Fix SYCL binary calling missing important libraries
|
||||
# Also use Level Zero + sysman INSTEAD of OpenCL
|
||||
# due to OpenCL reporting free memory as total memory,
|
||||
# breaking `-fit` functionality
|
||||
postFixup = lib.optionalString useSycl ''
|
||||
driverLibs=${
|
||||
lib.makeLibraryPath [
|
||||
intel-graphics-compiler
|
||||
ocl-icd
|
||||
]
|
||||
}
|
||||
for b in "$out"/bin/*; do
|
||||
[ -f "$b" ] && [ -x "$b" ] || continue
|
||||
patchelf --print-interpreter "$b" >/dev/null 2>&1 || continue
|
||||
patchelf --add-needed libOpenCL.so.1 --add-needed libigc.so.2 \
|
||||
--add-rpath "$driverLibs" "$b"
|
||||
wrapProgram "$b" \
|
||||
--set-default ONEAPI_DEVICE_SELECTOR level_zero:gpu \
|
||||
--set-default ZES_ENABLE_SYSMAN 1
|
||||
done
|
||||
'';
|
||||
|
||||
meta = {
|
||||
# Configurations we don't want even the CI to evaluate. Results in the
|
||||
# "unsupported platform" messages. This is mostly a no-op, because
|
||||
# cudaPackages would've refused to evaluate anyway.
|
||||
badPlatforms = optionals useCuda lib.platforms.darwin;
|
||||
badPlatforms =
|
||||
optionals useCuda lib.platforms.darwin
|
||||
++ optionals useSycl (lib.subtractLists [ "x86_64-linux" ] lib.platforms.all);
|
||||
|
||||
# Configurations that are known to result in build failures. Can be
|
||||
# overridden by importing Nixpkgs with `allowBroken = true`.
|
||||
|
||||
@@ -3,11 +3,11 @@ ARG OPENVINO_VERSION_FULL=2026.4.1.22982.07f9c262b05
|
||||
ARG UBUNTU_VERSION=24.04
|
||||
|
||||
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
|
||||
ARG IGC_VERSION=v2.41.5
|
||||
ARG IGC_VERSION_FULL=2_2.41.5+22716
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.35.39758.10
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.35.39758.10-0
|
||||
ARG IGDGMM_VERSION=22.10.0
|
||||
ARG IGC_VERSION=v2.42.6
|
||||
ARG IGC_VERSION_FULL=2_2.42.6+23070
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.40.40196.8
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.40.40196.8-0
|
||||
ARG IGDGMM_VERSION=22.10.2
|
||||
|
||||
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
|
||||
ARG NPU_DRIVER_VERSION=v1.38.0
|
||||
@@ -28,12 +28,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
## Build Image
|
||||
@@ -83,7 +86,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
# Build Stage
|
||||
RUN bash -c "source ${OpenVINO_DIR}/setupvars.sh && \
|
||||
|
||||
@@ -17,12 +17,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
### Build image
|
||||
@@ -52,7 +55,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \
|
||||
cmake -S . -B build \
|
||||
|
||||
@@ -9,12 +9,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -31,7 +34,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN cmake -B build -DGGML_NATIVE=OFF -DGGML_VULKAN=ON -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON && \
|
||||
cmake --build build --config Release -j$(nproc)
|
||||
|
||||
@@ -9,12 +9,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -28,7 +31,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_ZENDNN=ON && \
|
||||
cmake --build build -j $(nproc)
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
|
||||
build*/
|
||||
|
||||
tools/ui/node_modules/
|
||||
ui/apps/chat/node_modules/
|
||||
|
||||
models/*
|
||||
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ insert_final_newline = unset
|
||||
[vendor/**.patch]
|
||||
trim_trailing_whitespace = unset
|
||||
|
||||
[tools/ui/**]
|
||||
[ui/**]
|
||||
indent_style = unset
|
||||
indent_size = unset
|
||||
end_of_line = unset
|
||||
|
||||
@@ -14,7 +14,9 @@ runs:
|
||||
run: |
|
||||
BUILD_NUMBER="$(git rev-list --count HEAD)"
|
||||
SHORT_HASH="$(git rev-parse --short=7 HEAD)"
|
||||
if [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
if [[ "$GITHUB_REF_TYPE" == "tag" && "${{ env.BRANCH_NAME }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "name=${{ env.BRANCH_NAME }}" >> $GITHUB_OUTPUT
|
||||
elif [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
|
||||
|
||||
+1
-2
@@ -48,7 +48,6 @@ examples:
|
||||
- all-globs-to-all-files:
|
||||
- '!tools/server/**'
|
||||
- '!tools/mtmd/**'
|
||||
- '!tools/ui/**'
|
||||
testing:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
@@ -72,7 +71,7 @@ android:
|
||||
server/ui:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- tools/ui/**
|
||||
- ui/**
|
||||
server:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
|
||||
@@ -98,7 +98,7 @@ jobs:
|
||||
|
||||
hip:
|
||||
runs-on: ubuntu-22.04
|
||||
container: rocm/dev-ubuntu-22.04:6.1.2
|
||||
container: rocm/dev-ubuntu-22.04:6.3.4
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -16,6 +16,10 @@ on:
|
||||
description: "Skip the s390x build target (useful for fast test runs that do not need the IBM Z runner)"
|
||||
type: boolean
|
||||
default: false
|
||||
skip_ref_check:
|
||||
description: "Skip the requirement to run from a tag (allows branch refs)"
|
||||
type: boolean
|
||||
default: false
|
||||
schedule:
|
||||
# Rebuild daily rather than on every push because it is expensive
|
||||
- cron: '12 4 * * *'
|
||||
@@ -38,6 +42,14 @@ jobs:
|
||||
source_tag: ${{ steps.srctag.outputs.name }}
|
||||
|
||||
steps:
|
||||
- name: Check manual run ref
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.skip_ref_check }}
|
||||
run: |
|
||||
if [[ "$GITHUB_REF_TYPE" != "tag" ]]; then
|
||||
echo "::error::Manual runs of this workflow must be started from a tag"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
@@ -170,7 +182,7 @@ jobs:
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
# - name: Set up QEMU
|
||||
# if: ${{ contains(matrix.config.platforms, 'linux/amd64') }}
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
BODY: ${{ github.event.pull_request.body }}
|
||||
HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'video demo') }}
|
||||
run: |
|
||||
# sections written by demo:pr (see tools/ui/demo/cli.ts): the managed comment
|
||||
# sections written by demo:pr (see ui/apps/chat/demo/cli.ts): the managed comment
|
||||
# markers, or a preview linking to a demo video uploaded for this PR
|
||||
has_demo=false
|
||||
if { grep -qF '<!-- demo:start -->' <<< "$BODY" && grep -qF '<!-- demo:end -->' <<< "$BODY"; } ||
|
||||
|
||||
@@ -130,7 +130,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -196,7 +196,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -274,7 +274,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -381,7 +381,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -479,7 +479,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v5
|
||||
@@ -571,7 +571,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -624,7 +624,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -672,7 +672,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -894,7 +894,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Install Ninja
|
||||
run: |
|
||||
@@ -1460,7 +1460,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -1532,7 +1532,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Free up disk space
|
||||
uses: ggml-org/free-disk-space@v1.3.1
|
||||
@@ -1647,7 +1647,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
|
||||
@@ -7,6 +7,40 @@ on:
|
||||
description: 'Commit SHA1 to build'
|
||||
required: false
|
||||
type: string
|
||||
spec_methods:
|
||||
description: 'Speculative methods to test'
|
||||
required: true
|
||||
default: 'compact'
|
||||
type: choice
|
||||
options:
|
||||
- compact
|
||||
- nightly
|
||||
- all
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/server-speculative.yml',
|
||||
'common/speculative.*',
|
||||
'src/models/dflash*.cpp',
|
||||
'src/models/dspark.cpp',
|
||||
'src/models/eagle3.cpp',
|
||||
'src/models/lilicorr.cpp',
|
||||
'src/models/qwen35.cpp',
|
||||
'tools/server/tests/unit/test_spec_methods.py'
|
||||
]
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/server-speculative.yml',
|
||||
'common/speculative.*',
|
||||
'src/models/dflash*.cpp',
|
||||
'src/models/dspark.cpp',
|
||||
'src/models/eagle3.cpp',
|
||||
'src/models/lilicorr.cpp',
|
||||
'src/models/qwen35.cpp',
|
||||
'tools/server/tests/unit/test_spec_methods.py'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
@@ -56,8 +90,9 @@ jobs:
|
||||
|
||||
- name: Test speculative decoding
|
||||
env:
|
||||
SPEC_METHODS: compact
|
||||
SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || format('{0}/spec-models', runner.temp) }}
|
||||
SPEC_METHODS: ${{ github.event.inputs.spec_methods || 'compact' }}
|
||||
# pull requests from forks do not get repository variables
|
||||
SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || '/mnt/cache/spec-models' }}
|
||||
N_GPU_LAYERS: 99
|
||||
run: |
|
||||
cd tools/server/tests
|
||||
|
||||
@@ -23,20 +23,20 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build application
|
||||
run: npm run build
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Upload built UI
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
retention-days: 1
|
||||
|
||||
@@ -37,26 +37,26 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build application
|
||||
env:
|
||||
LLAMA_BUILD_NUMBER: ${{ steps.version.outputs.ui_version }}
|
||||
run: npm run build
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run PWA unit tests (versioned build output)
|
||||
run: npx vitest --project=unit --run tests/unit/pwa.spec.ts
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Upload built UI
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
retention-days: 1
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
name: UI Lib Publish
|
||||
|
||||
# Publishes @ggml-org/llama-ui, the library in ui/lib/ui, to npm.
|
||||
#
|
||||
# The version in ui/lib/ui/package.json is the release. A commit that reaches master with a
|
||||
# version npm does not have builds the library and publishes it; a commit that changes the
|
||||
# library without bumping it publishes nothing and says so on the run, which is the drift that
|
||||
# let 0.3.0 sit a day behind master. Bump the version in the PR whose change should ship, and
|
||||
# this workflow does the rest.
|
||||
#
|
||||
# Auth: no npm token is stored here. The job exchanges its OIDC identity for one (npm trusted
|
||||
# publishing) and signs the tarball with a provenance attestation. release-publish.yml has done
|
||||
# the same exchange on every push to master since it landed, its actions/attest step being the
|
||||
# proof that this repository can be issued a token.
|
||||
#
|
||||
# One-time setup, before this can publish: on the package, Publishing settings, Trusted
|
||||
# Publisher, GitHub Actions, with organization ggml-org, repository llama.cpp, workflow file
|
||||
# ui-lib-publish.yml, and Environment left blank - this job names no environment, since creating
|
||||
# one needs repository admin. Merging this workflow before that exists leaves the publish step
|
||||
# red on master: the release is late rather than wrong, and re-running the job publishes it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: 'Build and pack, but do not publish'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-lib-publish.yml',
|
||||
'ui/lib/ui/**.*'
|
||||
]
|
||||
# the advisory half, so a library change without a bump is annotated on the PR rather than
|
||||
# discovered after the merge: the path filter keeps it off PRs that touch only Llama Chat
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ui-lib-publish.yml',
|
||||
'ui/lib/ui/**.*'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
# never cancel a run mid-publish: a version half-published cannot be taken back. GitHub holds
|
||||
# one pending run per group, so a newer push replaces an older queued one - no release is lost,
|
||||
# as each run reads the version from its own commit.
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
LIB_DIR: ui/lib/ui
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish @ggml-org/llama-ui
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: ubuntu-26.04-arm
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Check npm can publish with an OIDC token
|
||||
run: |
|
||||
npm -v
|
||||
node -e '
|
||||
const v = process.argv[1];
|
||||
const [major, minor] = v.split(".").map(Number);
|
||||
if (major < 11 || (major === 11 && minor < 5)) {
|
||||
console.error(`npm ${v} cannot publish with trusted publishing: 11.5.1 or newer is needed`);
|
||||
process.exit(1);
|
||||
}
|
||||
' "$(npm -v)"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Run type checking
|
||||
run: npm run check
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Build the package and lint it
|
||||
run: npm run prepack
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Compare the version with npm
|
||||
id: version
|
||||
# on a PR this is advisory and must not fail the job: a registry outage is not the
|
||||
# contributor's mistake. On master the publish decision rests on the answer, so it fails.
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node scripts/npm-version-check.mjs
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Note a library change that ships no new version
|
||||
if: github.event_name == 'pull_request' && always() && steps.version.outcome == 'success' && steps.version.outputs.should_publish != 'true'
|
||||
env:
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
run: echo "::warning::$NOTE"
|
||||
|
||||
- name: Publish
|
||||
# master only. npm's trusted publisher matches on the repository and the workflow file,
|
||||
# not the branch, so this condition is the only thing stopping a manual run from a PR
|
||||
# branch - one that anyone with write access can start - from publishing code that no
|
||||
# one reviewed. A dispatch on master remains the way to re-run a release.
|
||||
if: github.ref == 'refs/heads/master' && steps.version.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
# --ignore-scripts publishes the tarball prepack built and publint passed rather than
|
||||
# rebuilding it here; --provenance is implied by trusted publishing and named to be
|
||||
# explicit. A package published with nothing in it cannot be taken back, so refuse first.
|
||||
env:
|
||||
TAG: ${{ steps.version.outputs.tag }}
|
||||
run: |
|
||||
test -d dist || { echo "dist/ is missing: nothing was built, so nothing is published"; exit 1; }
|
||||
npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Verify it landed
|
||||
if: github.ref == 'refs/heads/master' && steps.version.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
NAME: ${{ steps.version.outputs.name }}
|
||||
VERSION: ${{ steps.version.outputs.version }}
|
||||
run: |
|
||||
# the registry can serve a new version some time after publish returns, so retry rather
|
||||
# than call a release that worked a failure
|
||||
for try in 1 2 3 4 5 6; do
|
||||
seen=$(npm view "$NAME@$VERSION" version 2>/dev/null || true)
|
||||
if [ "$seen" = "$VERSION" ]; then
|
||||
echo "$NAME@$VERSION is on npm"
|
||||
npm view "$NAME@$VERSION" dist-tags version dist.integrity
|
||||
exit 0
|
||||
fi
|
||||
echo "not visible yet (attempt $try of 6)"
|
||||
sleep 10
|
||||
done
|
||||
echo "::error::$NAME@$VERSION published without error but is not visible on the registry after 6 attempts. The publish did not fail: check the package on npm before publishing again."
|
||||
exit 1
|
||||
|
||||
- name: Report
|
||||
if: always()
|
||||
# every value reaches the shell through env, not through the script, so a package name
|
||||
# from a PR's package.json cannot inject into this step
|
||||
env:
|
||||
NAME: ${{ steps.version.outputs.name || '@ggml-org/llama-ui' }}
|
||||
VERSION: ${{ steps.version.outputs.version || 'unknown' }}
|
||||
LATEST: ${{ steps.version.outputs.latest }}
|
||||
VERDICT: ${{ steps.version.outputs.verdict || 'not reached' }}
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
SHOULD: ${{ steps.version.outputs.should_publish }}
|
||||
DRY: ${{ inputs.dry_run }}
|
||||
run: |
|
||||
if [ "$DRY" = "true" ]; then published="no (dry run)"
|
||||
elif [ "$SHOULD" = "true" ]; then published=yes
|
||||
else published=no
|
||||
fi
|
||||
{
|
||||
echo "### $NAME $VERSION"
|
||||
echo
|
||||
echo "| | |"
|
||||
echo "| --- | --- |"
|
||||
echo "| npm latest was | ${LATEST:-nothing published yet} |"
|
||||
echo "| verdict | \`$VERDICT\` |"
|
||||
echo "| published | $published |"
|
||||
echo
|
||||
echo "> $NOTE"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Mind the verdict
|
||||
if: always() && github.event_name != 'pull_request' && steps.version.outcome == 'success'
|
||||
# a version going backwards is a mistake; a version not bumped is usually one too, but it
|
||||
# is also what a stories-only commit looks like, so that only fails the run when the
|
||||
# repository variable UI_LIB_PUBLISH_FAIL_ON_UNBUMPED is set to true
|
||||
env:
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
VERDICT: ${{ steps.version.outputs.verdict }}
|
||||
FAIL_ON_UNBUMPED: ${{ vars.UI_LIB_PUBLISH_FAIL_ON_UNBUMPED }}
|
||||
run: |
|
||||
case "$VERDICT" in
|
||||
stale)
|
||||
echo "::error::$NOTE"
|
||||
exit 1 ;;
|
||||
published)
|
||||
if [ "$FAIL_ON_UNBUMPED" = "true" ]; then
|
||||
echo "::error::$NOTE"
|
||||
exit 1
|
||||
fi
|
||||
echo "::warning::$NOTE" ;;
|
||||
esac
|
||||
@@ -33,15 +33,15 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
run-id: ${{ inputs.run_id }}
|
||||
github-token: ${{ github.token }}
|
||||
|
||||
- name: Create distribution archive
|
||||
run: |
|
||||
tar -czf dist.tar.gz -C tools/ui/dist .
|
||||
tar -czf dist.tar.gz -C ui/apps/chat/dist .
|
||||
sha256sum dist.tar.gz > dist.tar.gz.sha256
|
||||
mv dist.tar.gz dist.tar.gz.sha256 tools/ui/dist/
|
||||
mv dist.tar.gz dist.tar.gz.sha256 ui/apps/chat/dist/
|
||||
|
||||
- name: Install Hugging Face Hub CLI
|
||||
run: pip install -U huggingface_hub
|
||||
@@ -52,12 +52,12 @@ jobs:
|
||||
- name: Sync built files to Hugging Face bucket (version tag)
|
||||
run: |
|
||||
# Upload the built files to the Hugging Face bucket under the release version
|
||||
hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet
|
||||
hf buckets sync ui/apps/chat/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet
|
||||
|
||||
- name: Sync built files to Hugging Face bucket (latest)
|
||||
run: |
|
||||
# Also upload to the 'latest' directory for fallback downloads
|
||||
hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet
|
||||
hf buckets sync ui/apps/chat/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet
|
||||
|
||||
- name: Verify upload
|
||||
run: |
|
||||
|
||||
@@ -17,7 +17,8 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -25,7 +26,8 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
@@ -67,33 +69,33 @@ jobs:
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Run type checking
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run linting
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run lint
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Client tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:client
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Unit tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:unit
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
@@ -109,25 +111,25 @@ jobs:
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Build Storybook
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run build-storybook
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run UI tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:ui -- --testTimeout=60000
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run E2E tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:e2e
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
name: UI Storybook
|
||||
|
||||
# Builds Llama UI and Llama Chat's Storybook (ui/apps/chat) and publishes it to its Hugging Face Space
|
||||
# whenever the UI changes on master, so the Space always shows the latest build.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-storybook.yml',
|
||||
'ui/apps/chat/**',
|
||||
'ui/lib/ui/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# only the newest build matters: a newer push replaces a deploy still in progress
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
name: Deploy to Hugging Face Spaces
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: hf-jobs-cpu-upgrade:ubuntu26_04
|
||||
|
||||
env:
|
||||
HF_SPACE: ${{ vars.HF_SPACE_UI_STORYBOOK || 'ggml-org/llama-ui-storybook' }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build Storybook
|
||||
run: npm run build-storybook
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Write the Space card
|
||||
run: |
|
||||
cat > ui/apps/chat/storybook-static/README.md << EOF
|
||||
---
|
||||
title: Llama UI Storybook
|
||||
emoji: 🦙
|
||||
colorFrom: gray
|
||||
colorTo: gray
|
||||
sdk: static
|
||||
pinned: false
|
||||
---
|
||||
|
||||
The Storybook of Llama UI and Llama Chat, built from [ggml-org/llama.cpp](https://github.com/ggml-org/llama.cpp) (\`ui/apps/chat\`) at ${GITHUB_SHA}.
|
||||
EOF
|
||||
|
||||
- name: Install Hugging Face Hub CLI
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y python3 python3-venv
|
||||
python3 -m venv .venv-hf
|
||||
.venv-hf/bin/pip install -U huggingface_hub==1.28.0
|
||||
|
||||
- name: Upload to the Space
|
||||
env:
|
||||
# a token with write access to the Space; HF_TOKEN_UI_STATIC_OUTPUT if that one has it
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_UI_STORYBOOK || secrets.HF_TOKEN_UI_STATIC_OUTPUT }}
|
||||
run: |
|
||||
# --delete '*' removes files the new build no longer has; the Space's .gitattributes is always kept
|
||||
.venv-hf/bin/hf upload "$HF_SPACE" ui/apps/chat/storybook-static . \
|
||||
--repo-type space \
|
||||
--delete '*' \
|
||||
--commit-message "Storybook from ggml-org/llama.cpp@${GITHUB_SHA::10}"
|
||||
+60
-17
@@ -13,7 +13,8 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -21,7 +22,8 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
@@ -66,46 +68,87 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
if: ${{ steps.node.conclusion == 'success' }}
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run type checking
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run linting
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run lint
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Check DESIGN.md is current with the tokens
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check:design
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npx playwright install --with-deps
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Client tests
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:client
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Unit tests (uses pre-built dist/ from ui-build)
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:unit
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
lib-checks:
|
||||
name: Library checks
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Run type checking
|
||||
run: npm run check
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Install Playwright browsers
|
||||
run: npx playwright install --with-deps chromium
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
# the unit tests and the Storybook stories (ui/lib/ui/tests/stories), which run as tests in Chromium
|
||||
- name: Run tests
|
||||
run: npm test
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Build the package
|
||||
run: npm run prepack
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
@@ -124,38 +167,38 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
if: ${{ steps.node.conclusion == 'success' }}
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Download built UI artifacts (reuses ui-build)
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npx playwright install --with-deps
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build Storybook
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run build-storybook
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run UI tests
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:ui -- --testTimeout=60000
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run E2E tests (uses pre-built dist/ from ui-build)
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:e2e
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# date: Wed Nov 4 12:03:31 EET 2026
|
||||
# date: Mon Nov 16 10:05:56 EET 2026
|
||||
# this file is auto-generated by scripts/gen-authors.sh
|
||||
|
||||
Нияз Гарифзянов <112617865+garrnizon@users.noreply.github.com>
|
||||
@@ -43,7 +43,9 @@ Aarni Koskela <akx@iki.fi>
|
||||
Aaron Miller <apage43@ninjawhale.com>
|
||||
Aaron Teo <57927438+taronaeo@users.noreply.github.com>
|
||||
Aaron Teo <aaron.teo1@ibm.com>
|
||||
Aaron Thomas <84493696+arxhr007@users.noreply.github.com>
|
||||
Aaryaman Vasishta <aaryaman.vasishta@amd.com>
|
||||
Aayan Karmali <44980366+StockerMC@users.noreply.github.com>
|
||||
Abheek Gulati <abheekg@hotmail.com>
|
||||
abhijain1204fujitsu <139222713+abhijain1204fujitsu@users.noreply.github.com>
|
||||
Abhijit Ramesh <abhijitramesh2k@gmail.com>
|
||||
@@ -112,10 +114,12 @@ Aleksander Grygier <admin@serveurperso.com>
|
||||
Aleksander Grygier <aleksander.grygier@gmail.com>
|
||||
Aleksander Grygier <thichthat@gmail.com>
|
||||
Aleksei Nikiforov <103434461+AlekseiNikiforovIBM@users.noreply.github.com>
|
||||
Alen Požun <kalyx.solutions@proton.me>
|
||||
Alessandro de Oliveira Faria (A.K.A.CABELO) <cabelo@opensuse.org>
|
||||
Alessandro98-git <61804547+Alessandro98-git@users.noreply.github.com>
|
||||
Alex <18387287+wadealexc@users.noreply.github.com>
|
||||
Alex <59368173+AlexGabbia@users.noreply.github.com>
|
||||
Alex <alexzkhr@gmail.com>
|
||||
Alex <awhill19@icloud.com>
|
||||
Alex Azarov <alex@azarov.by>
|
||||
Alex Azarov <alexander.azarov@mapbox.com>
|
||||
@@ -132,6 +136,7 @@ Alex Trotta <44127594+Ahajha@users.noreply.github.com>
|
||||
Alex Tuddenham <61622354+AlexsCode@users.noreply.github.com>
|
||||
Alex von Gluck IV <kallisti5@unixzen.com>
|
||||
Alex Wu <dindinw@users.noreply.github.com>
|
||||
Alex-JP-93 <Jianpeng.Li@amd.com>
|
||||
alex-spacemit <jinghui.huang@spacemit.com>
|
||||
Alexander Batischev <eual.jp@gmail.com>
|
||||
Alexander Heisler <126129661+heislera763@users.noreply.github.com>
|
||||
@@ -143,6 +148,7 @@ Alexis Williams <typedrat@users.noreply.github.com>
|
||||
alexpinel <93524949+alexpinel@users.noreply.github.com>
|
||||
Alfred <zxu3@clemson.edu>
|
||||
Ali Chraghi <63465728+alichraghi@users.noreply.github.com>
|
||||
Ali F. Al-Ramadan <55710790+aalramadan@users.noreply.github.com>
|
||||
Ali Nehzat <ali.nehzat@thanks.dev>
|
||||
Ali Tariq <ali.tariq@10xengineers.ai>
|
||||
Ali Tariq <alitariq4589@gmail.com>
|
||||
@@ -158,6 +164,7 @@ Aman Karki <amankarki151@gmail.com>
|
||||
Aman Karki <itsamankarki@gmail.com>
|
||||
amd-dwang <dong.wang@amd.com>
|
||||
amd-lalithnc <lalithnc@amd.com>
|
||||
AMIR <127450356+amirthn6533@users.noreply.github.com>
|
||||
Amir <amir_zia@outlook.com>
|
||||
amirai21 <89905406+amirai21@users.noreply.github.com>
|
||||
AmirAli Mirian <37371367+amiralimi@users.noreply.github.com>
|
||||
@@ -213,6 +220,7 @@ Antoine Viallon <antoine@lesviallon.fr>
|
||||
Anton I <inflyer@hotmail.com>
|
||||
Anton Mitkov <anton_b_mitkov@abv.bg>
|
||||
Anton Mitkov <anton.mitkov@codeplay.com>
|
||||
Anton Sokolchenko <wsevendays@gmail.com>
|
||||
Antonis Makropoulos <benuix@gmail.com>
|
||||
anubhavfujitsu <Anubhav.Srivastava@fujitsu.com>
|
||||
Anudit Nagar <nagaranudit@gmail.com>
|
||||
@@ -233,10 +241,12 @@ arlo-phoenix <140345165+arlo-phoenix@users.noreply.github.com>
|
||||
Armen Kaleshian <kriation@users.noreply.github.com>
|
||||
Arsen Arutunan <58118221+limloop@users.noreply.github.com>
|
||||
Arseniy Obolenskiy <good.doog.2012@gmail.com>
|
||||
ArshM17-NV <amaknojia@nvidia.com>
|
||||
Artem <guinmoon@gmail.com>
|
||||
Artem Zinnatullin <ceo@abstractny.gay>
|
||||
Artyom Lebedev <vagran.ast@gmail.com>
|
||||
aryantandon01 <80969509+aryantandon01@users.noreply.github.com>
|
||||
Asahi-Prv <246092086+Asahi-Prv@users.noreply.github.com>
|
||||
Asahi-Prv <asahihori7@gmail.com>
|
||||
asbelin <71226900+asbelin@users.noreply.github.com>
|
||||
Asbjørn Olling <asbjornolling@gmail.com>
|
||||
@@ -258,6 +268,7 @@ AustinMroz <austinmroz@utexas.edu>
|
||||
AUTOMATIC1111 <16777216c@gmail.com>
|
||||
automaticcat <daogiatuank54@gmail.com>
|
||||
awatuna <23447591+awatuna@users.noreply.github.com>
|
||||
Awss <gitj.18@alt.awss1i.com>
|
||||
b4b4o <zwbao@foxmail.com>
|
||||
Bach Le <bach@bullno1.com>
|
||||
BADR <contact@pythops.com>
|
||||
@@ -351,6 +362,7 @@ CarterLi999 <664681047@qq.com>
|
||||
Casey Primozic <casey@cprimozic.net>
|
||||
Casey Primozic <me@ameo.link>
|
||||
CausalLM <148736309+CausalLM@users.noreply.github.com>
|
||||
CC <65151926+realMoana@users.noreply.github.com>
|
||||
ccbinn <ccbinn@163.com>
|
||||
cduk <19917266+cduk@users.noreply.github.com>
|
||||
cebtenzzre <cebtenzzre@gmail.com>
|
||||
@@ -379,6 +391,7 @@ Cheng Shao <terrorjack@type.dance>
|
||||
Chenguang Li <757486878@qq.com>
|
||||
Chenguang Li <87689256+noemotiovon@users.noreply.github.com>
|
||||
chiheb ben cheikh <100644963+ChihebBENCHEIKH1@users.noreply.github.com>
|
||||
chinmaykamjith001 <chinmaykamjith@gmail.com>
|
||||
Chipmunk <101038159+CHIPMUNK-T0T@users.noreply.github.com>
|
||||
chiranko <96988916+chiranko@users.noreply.github.com>
|
||||
Chris Danis <cdanis@gmail.com>
|
||||
@@ -434,6 +447,7 @@ crasm <crasm@git.vczf.us>
|
||||
crat0z <11581854+crat0z@users.noreply.github.com>
|
||||
CRD716 <crd716@gmail.com>
|
||||
CrispStrobe <154636388+CrispStrobe@users.noreply.github.com>
|
||||
Cristian Adam <cristian.adam@gmail.com>
|
||||
Cristiano Pinto <140563307+crowmoed@users.noreply.github.com>
|
||||
crsawyer <7572190+crsawyer@users.noreply.github.com>
|
||||
Csaba Kecskemeti <csaba.kecskemeti@gmail.com>
|
||||
@@ -527,6 +541,7 @@ Djip007 <3705339+Djip007@users.noreply.github.com>
|
||||
Djip007 <djip.perois@free.fr>
|
||||
dm4 <dm4@secondstate.io>
|
||||
dm4 <sunrisedm4@gmail.com>
|
||||
Dmitrii Panchev <dmnobunaga@users.noreply.github.com>
|
||||
Dmitry Atamanov <data-man@users.noreply.github.com>
|
||||
Dmytro Minochkin <dmytro.minochkin@gmail.com>
|
||||
Dmytro Romanov <casteldazur@gmail.com>
|
||||
@@ -567,6 +582,7 @@ Ed Addario <29247825+EAddario@users.noreply.github.com>
|
||||
Ed Lee <edilee@mozilla.com>
|
||||
Ed Lepedus <ed.lepedus@googlemail.com>
|
||||
Eddie-Wang <wangjinheng1120@163.com>
|
||||
eddiechen <Echen1246@gmail.com>
|
||||
Eden <aa9736195201@gmail.com>
|
||||
eduardopessin <100053075+eduardopessin@users.noreply.github.com>
|
||||
Edward Taylor <edeetee@gmail.com>
|
||||
@@ -598,6 +614,7 @@ Erik Winter <93544977+ewintr@users.noreply.github.com>
|
||||
Ervin Áron Tasnádi <etasnadi@protonmail.com>
|
||||
Esko Toivonen <eskot98@gmail.com>
|
||||
Ethan Guo <ethanguo.dev@gmail.com>
|
||||
Ethan Lin <linsen458@gmail.com>
|
||||
Ethan Turner <eturner64@gmail.com>
|
||||
Ettore Di Giacinto <mudler@users.noreply.github.com>
|
||||
EugeoSynthesisThirtyTwo <gabriel.dhimoila@gmail.com>
|
||||
@@ -625,6 +642,7 @@ fanyang <fanyang89@outlook.com>
|
||||
Farbod Bijary <110523279+farbodbj@users.noreply.github.com>
|
||||
Fathi Boudra <fathi.boudra@linaro.org>
|
||||
Fattire <528174+fat-tire@users.noreply.github.com>
|
||||
Feiteng <lifeiteng0422@gmail.com>
|
||||
felix <felix314159@users.noreply.github.com>
|
||||
Felix <stenbackfelix@gmail.com>
|
||||
Felix Ye <felixyjs@gmail.com>
|
||||
@@ -632,6 +650,7 @@ fengerhu1 <2748250768@qq.com>
|
||||
feni6 <fenissova@proton.me>
|
||||
fidoriel <49869342+fidoriel@users.noreply.github.com>
|
||||
fiesh <fiesh@zefix.tv>
|
||||
figkcoci-debug <figkcoci@gmail.com>
|
||||
FINAL-Bench <kkms1116@koreacu.ac.kr>
|
||||
Finn Voorhees <finnvoorhees@gmail.com>
|
||||
Firat <firatkiral@gmail.com>
|
||||
@@ -646,6 +665,7 @@ Foad Abo Dahood <32059146+masterFoad@users.noreply.github.com>
|
||||
Folko-Ven <71110216+Folko-Ven@users.noreply.github.com>
|
||||
forforever73 <63285796+forforever73@users.noreply.github.com>
|
||||
Foul-Tarnished <107711110+Foul-Tarnished@users.noreply.github.com>
|
||||
fox5611 <66219952+fox5611@users.noreply.github.com>
|
||||
Francisco Herrera <ppaanncchhoo507@gmail.com>
|
||||
Francisco Melo <43780565+francis2tm@users.noreply.github.com>
|
||||
Francois Dugast <francois.dugast@intel.com>
|
||||
@@ -709,6 +729,7 @@ GiviMAD <GiviMAD@users.noreply.github.com>
|
||||
gliptic <gliptic@users.noreply.github.com>
|
||||
gn64 <yukikaze.jp@gmail.com>
|
||||
goerch <jhr.walter@t-online.de>
|
||||
Gotya-OFFCIAL <zhengfreeman@gmail.com>
|
||||
Govlzkoy <gotope@users.noreply.github.com>
|
||||
grahameth <96447521+grahameth@users.noreply.github.com>
|
||||
Gregor Jasny <gjasny@googlemail.com>
|
||||
@@ -749,6 +770,7 @@ Haowei Wu <breadcyanide@icloud.com>
|
||||
Haoxiang Fei <tonyfettes@tonyfettes.com>
|
||||
Harald Fernengel <harald.fernengel@here.com>
|
||||
Harapan Rachman <harapanrachman@gmail.com>
|
||||
Harkanwal Bedi <bedi1887@gmail.com>
|
||||
Harkirat Gill <harkirat.gill@amd.com>
|
||||
HarrisonSec <gzxharrison@gmail.com>
|
||||
Hatsune Miku <129688334+at8u@users.noreply.github.com>
|
||||
@@ -768,6 +790,7 @@ Henrik Forstén <henrik.forsten@gmail.com>
|
||||
Henry Linjamäki <henry.linjamaki@gmail.com>
|
||||
Henry Linjamäki <henry.mikael.linjamaki@intel.com>
|
||||
Henry147147 <44851451+Henry147147@users.noreply.github.com>
|
||||
Hera <58063257+hera2019@users.noreply.github.com>
|
||||
Herman Semenoff <GermanAizek@yandex.ru>
|
||||
Herman Semenov <GermanAizek@yandex.ru>
|
||||
Hesen Peng <hesen.peng@gmail.com>
|
||||
@@ -846,6 +869,7 @@ Imad Saddik <79410781+ImadSaddik@users.noreply.github.com>
|
||||
iMil <imil@NetBSD.org>
|
||||
Incarnas <119618389+bit-incarnas@users.noreply.github.com>
|
||||
InflexCZE <martin.pavlicek11@gmail.com>
|
||||
Iñigo Imaz <49730431+inimaz@users.noreply.github.com>
|
||||
Intel AI Get-to Market Customer Success and Solutions <ai.gtm.css@gmail.com>
|
||||
intelmatt <61025942+intelmatt@users.noreply.github.com>
|
||||
iohub <rickyang.pro@gmail.com>
|
||||
@@ -857,7 +881,9 @@ IsaacDynamo <61521674+IsaacDynamo@users.noreply.github.com>
|
||||
Ishaan Gandhi <Ishaangandhi@gmail.com>
|
||||
iSma <ismail.senhaji@gmail.com>
|
||||
Ismail <115064057+AlrIsmail@users.noreply.github.com>
|
||||
IsNot <209144869@qq.com>
|
||||
issixx <46835150+issixx@users.noreply.github.com>
|
||||
Iswarya Alex <47045679+iswaryaalex@users.noreply.github.com>
|
||||
itsnotoger <19309683+itsnotoger@users.noreply.github.com>
|
||||
itterative <190138728+itterative@users.noreply.github.com>
|
||||
Ivan <nekotekina@gmail.com>
|
||||
@@ -888,6 +914,7 @@ James Francis <6763899+JamesFranc@users.noreply.github.com>
|
||||
James O'Leary <65884233+jpohhhh@users.noreply.github.com>
|
||||
James Reynolds <magnusviri@users.noreply.github.com>
|
||||
jameswu2014 <545426914@qq.com>
|
||||
JamieBordoni <jamie.bordoni@protonmail.com>
|
||||
Jan Boon <jan.boon@kaetemi.be>
|
||||
Jan Boon <kaetemi@gmail.com>
|
||||
Jan Ekström <jeebjp@gmail.com>
|
||||
@@ -956,8 +983,10 @@ Jinyang He <hejinyang@loongson.cn>
|
||||
jinzihao <jinzihao1996@gmail.com>
|
||||
Jiří Podivín <66251151+jpodivin@users.noreply.github.com>
|
||||
Jiří Sejkora <Sejseloid@gmail.com>
|
||||
Jiří Svítil <32205472+mistrjirka@users.noreply.github.com>
|
||||
Jiwoong Song <55657581+jwsong98@users.noreply.github.com>
|
||||
JJJYmmm <92386084+JJJYmmm@users.noreply.github.com>
|
||||
jjperry69 <jjperry69@gmail.com>
|
||||
jklincn <985765408@qq.com>
|
||||
jklincn <jklincn@outlook.com>
|
||||
JM Robles <roblesjm@gmail.com>
|
||||
@@ -970,6 +999,7 @@ Joe Rowell <joerowell4@gmail.com>
|
||||
Joe Todd <joe.todd@codeplay.com>
|
||||
joecryptotoo <80373433+joecryptotoo@users.noreply.github.com>
|
||||
Joel <agnel.joel@gmail.com>
|
||||
Joel Fernandes <joelagnelf@nvidia.com>
|
||||
Johan <JohanAR@users.noreply.github.com>
|
||||
Johannes Gäßler <johannesg@5d6.de>
|
||||
Johannes Rudolph <johannes.rudolph@gmail.com>
|
||||
@@ -988,6 +1018,7 @@ Jonas J <111707981+John-194@users.noreply.github.com>
|
||||
Jonas Jankaitis <111707981+John-194@users.noreply.github.com>
|
||||
Jonas Wunderlich <32615971+jonas-w@users.noreply.github.com>
|
||||
Jonathan <47618606+jbuchananr@users.noreply.github.com>
|
||||
Jonathan Buck <jonathan.j.buck@gmail.com>
|
||||
Jonathan Clohessy <jonathan.clohessy@arm.com>
|
||||
Jonathan Graehl <99024+graehl@users.noreply.github.com>
|
||||
Jorge A <161275481+jorgealias@users.noreply.github.com>
|
||||
@@ -1097,6 +1128,7 @@ klosax <131523366+klosax@users.noreply.github.com>
|
||||
KnightYao <KnightYao@users.noreply.github.com>
|
||||
KokerZhou <111279477+KokerZhou@users.noreply.github.com>
|
||||
Kolen Cheung <ickc@users.noreply.github.com>
|
||||
Kong Jiale <2049447146@qq.com>
|
||||
kononnable <kononnable@gmail.com>
|
||||
Konrad Moren <kmoren@nvidia.com>
|
||||
konradmb <konradmb@o2.pl>
|
||||
@@ -1167,6 +1199,7 @@ li-lizhe <82501526+li-lizhe@users.noreply.github.com>
|
||||
liangliangchang <lichang@amd.com>
|
||||
liminfei-amd <91481003+liminfei-amd@users.noreply.github.com>
|
||||
limitedAtonement <limitedAtonement@users.noreply.github.com>
|
||||
Lines <linellazatin@gmail.com>
|
||||
lingyezhixing <144504450+lingyezhixing@users.noreply.github.com>
|
||||
linsen458-spec <linsen458@gmail.com>
|
||||
Linux User <linuxusergd@outlook.com>
|
||||
@@ -1227,6 +1260,7 @@ makomk <makosoft@googlemail.com>
|
||||
manayang <jackmanayang@gmail.com>
|
||||
manikbhandari <mbbhandarimanik2@gmail.com>
|
||||
Manuel <44313466+makuche@users.noreply.github.com>
|
||||
ManuelPDM <153872964+ManuelPDM@users.noreply.github.com>
|
||||
maor-ps <154728172+maor-ps@users.noreply.github.com>
|
||||
Maphist0 <28743569+Maphist0@users.noreply.github.com>
|
||||
Marc Köhlbrugge <subscriptions@marckohlbrugge.com>
|
||||
@@ -1336,6 +1370,7 @@ Michael Hueschen <m@mhueschen.dev>
|
||||
Michael Kesper <mkesper@schokokeks.org>
|
||||
Michael Klimenko <mklimenko29@gmail.com>
|
||||
Michael Lamothe <michael.lamothe@gmail.com>
|
||||
Michael Paulauski <mike10010100@gmail.com>
|
||||
Michael Podvitskiy <podvitskiymichael@gmail.com>
|
||||
Michael Potter <NanoTekGuy@Gmail.com>
|
||||
Michael Taylor <162068037+mctylr-gh@users.noreply.github.com>
|
||||
@@ -1455,6 +1490,7 @@ Nikolay Popov <131475237+npopov-vst@users.noreply.github.com>
|
||||
Nils Gladitz <nilsgladitz@gmail.com>
|
||||
Nindaleth <Nindaleth@users.noreply.github.com>
|
||||
ningshanwutuobang <ningshanwutuobang@gmail.com>
|
||||
Nisbenz <benazzaanis783@gmail.com>
|
||||
Noah <99681487+NoahOksuz@users.noreply.github.com>
|
||||
nold <Nold360@users.noreply.github.com>
|
||||
nopperl <54780682+nopperl@users.noreply.github.com>
|
||||
@@ -1462,6 +1498,7 @@ nullname <chraac@gmail.com>
|
||||
Nuno <rare-magma@posteo.eu>
|
||||
nuri <yoonuri1@gmail.com>
|
||||
nusu-github <29514220+nusu-github@users.noreply.github.com>
|
||||
nv-vankit <162085734+nv-vankit@users.noreply.github.com>
|
||||
nwyin <tommynguyen0512@gmail.com>
|
||||
o7si <32285332+o7si@users.noreply.github.com>
|
||||
Oğuzhan Akkaya <oakkaya@ymail.com>
|
||||
@@ -1485,6 +1522,7 @@ oobabooga <oobabooga4@gmail.com>
|
||||
opparco <parco.opaai@gmail.com>
|
||||
Ori Pekelman <ori@pekelman.com>
|
||||
Oscar Barenys <rtfss1@gmail.com>
|
||||
Oscar M. Herrera <oscar@kss-inc.com>
|
||||
OSecret <135510162+OLSecret@users.noreply.github.com>
|
||||
ostix360 <55257054+ostix360@users.noreply.github.com>
|
||||
Ouadie EL FAROUKI <ouadie.elfarouki@codeplay.com>
|
||||
@@ -1611,6 +1649,7 @@ Raj Hammeer Singh Hada <hammeerraj@gmail.com>
|
||||
Rajendra Matcha <matcraje@qti.qualcomm.com>
|
||||
Ralph Soika <ralph.soika@imixs.com>
|
||||
Raman Shinde <raman.shinde15@gmail.com>
|
||||
Ramzi <146260554+rmz-24@users.noreply.github.com>
|
||||
Rand Xie <randxiexyy29@gmail.com>
|
||||
Randall Fitzgerald <randall@dasaku.net>
|
||||
Random Fly <renfei8@live.cn>
|
||||
@@ -1651,6 +1690,7 @@ Rich Dougherty <rich@rd.nz>
|
||||
Richard <r-burton@hotmail.co.uk>
|
||||
Richard Davison <richard.davison1@gmail.com>
|
||||
Richard Kiss <him@richardkiss.com>
|
||||
Richard Palethorpe <io@richiejp.com>
|
||||
Richard Roberson <richardr1126@gmail.com>
|
||||
RichardScottOZ <rnmscott@netspace.net.au>
|
||||
Rick G <26732651+TheFlipbook@users.noreply.github.com>
|
||||
@@ -1695,6 +1735,7 @@ Roni <sulpher@gmx.net>
|
||||
Ronny Brendel <ronnyb@nvidia.com>
|
||||
Ronny Brendel <ronnybrendel@gmail.com>
|
||||
Ronsor <ronsor@ronsor.pw>
|
||||
Roshan Kumar <52444089+roshankumarg529@users.noreply.github.com>
|
||||
Rotem Dan <rotemdan@gmail.com>
|
||||
Rowan Hart <rowanbhart@gmail.com>
|
||||
rspOverflow <217881046+rspOverflow@users.noreply.github.com>
|
||||
@@ -1727,9 +1768,11 @@ Saba Fallah <10401143+sfallah@users.noreply.github.com>
|
||||
Saba Fallah <sabafallah@gmail.com>
|
||||
Sachin Desai <smdesai@gmail.com>
|
||||
Sachin Sharma <sachin@zettabolt.com>
|
||||
Saejin Kim-Heinert <saejinmh@gmail.com>
|
||||
Safi Ullah <safiullah.3915@gmail.com>
|
||||
safranowith <bsh155762@gmail.com>
|
||||
Said Taghadouini <84044788+staghado@users.noreply.github.com>
|
||||
SaidSultani <62264302+SaidSultani@users.noreply.github.com>
|
||||
Sait Furkan Teke <35101659+stfurkan@users.noreply.github.com>
|
||||
SakuraUmi <yukinon244@gmail.com>
|
||||
Salvador E. Tropea <stropea@inti.gob.ar>
|
||||
@@ -1754,6 +1797,7 @@ Sang-Kil Park <sang.park@42dot.ai>
|
||||
Sanjay Ahari <sanjayahari1704@gmail.com>
|
||||
Sarah Wu <sarahwu185@gmail.com>
|
||||
Sarah Wu <sarwu@nvidia.com>
|
||||
sargentmajord <jasonlather13@gmail.com>
|
||||
Sascha Rogmann <59577610+srogmann@users.noreply.github.com>
|
||||
sasha0552 <admin@sasha0552.org>
|
||||
Satinder Grewal <grewal.satinder@gmail.com>
|
||||
@@ -1786,6 +1830,7 @@ Sergiu <8598216+mzsergiu@users.noreply.github.com>
|
||||
serhii-nakon <57632032+serhii-nakon@users.noreply.github.com>
|
||||
Sertaç Özercan <852750+sozercan@users.noreply.github.com>
|
||||
seryogakovalyov <seryogakovalyov@gmail.com>
|
||||
Seungjun Bae <linkedlatch@gmail.com>
|
||||
Seungmin Kim <8457324+ehfd@users.noreply.github.com>
|
||||
SeungWon Jeong <65549245+redlion0929@users.noreply.github.com>
|
||||
Seyoung Jeong <seyoungjeong@gmail.com>
|
||||
@@ -1871,6 +1916,7 @@ SmartestWashingMachine <ottobizness@gmail.com>
|
||||
smugman-dot <wbsmoke101@gmail.com>
|
||||
SnA1lGo <44647694+skrandy@users.noreply.github.com>
|
||||
snadampal <87143774+snadampal@users.noreply.github.com>
|
||||
Sofian Elmotiem <sofianelmotiem@gmail.com>
|
||||
SoftwareRenderer <138734813+SoftwareRenderer@users.noreply.github.com>
|
||||
Someone <sergei.kozlukov@aalto.fi>
|
||||
Someone Serge <sergei.kozlukov@aalto.fi>
|
||||
@@ -1886,6 +1932,7 @@ sprayandwipe <spray.and.wipe@gmail.com>
|
||||
Sreeram <sreeram.sivasubramony@amd.com>
|
||||
SRHMorris <69468379+SRHMorris@users.noreply.github.com>
|
||||
Srihari-mcw <96763064+Srihari-mcw@users.noreply.github.com>
|
||||
Srijan Keshri <212402043+arcusbuilds@users.noreply.github.com>
|
||||
Srinivas Billa <nivibilla@gmail.com>
|
||||
srkizer <github.soreepeong.prqovzj1d@srkizer.net>
|
||||
ssweens <1149151+ssweens@users.noreply.github.com>
|
||||
@@ -1919,6 +1966,7 @@ Sutou Kouhei <kou@cozmixng.org>
|
||||
Svetlozar Georgiev <55534064+sgeor255@users.noreply.github.com>
|
||||
Swigler <124839156+Swigler@users.noreply.github.com>
|
||||
swittk <switt1995@gmail.com>
|
||||
sxq <18856096915@163.com>
|
||||
SXX <song_xiaoxi@126.com>
|
||||
SXX <sxx1136965276@gmail.com>
|
||||
Tai Duc Nguyen <taiducnguyen.drexel@gmail.com>
|
||||
@@ -2044,6 +2092,7 @@ Ulrich Drepper <drepper@gmail.com>
|
||||
unbounded <haakon@likedan.net>
|
||||
Unbug Lee <tidelgl@gmail.com>
|
||||
unraido <127105806+unraido@users.noreply.github.com>
|
||||
usrlocalben <benjamin@rqdq.com>
|
||||
uvos <carl@uvos.xyz>
|
||||
uvos <devnull@uvos.xyz>
|
||||
uvos <philipp@uvos.xyz>
|
||||
@@ -2084,6 +2133,7 @@ vk <89937361+itsvedantkumar@users.noreply.github.com>
|
||||
Vlad <spitfireage@gmail.com>
|
||||
Vladimir <bogdad@gmail.com>
|
||||
Vladimir Malyutin <first-leon@yandex.ru>
|
||||
Vladimir Sotnikov <bubert9@gmail.com>
|
||||
Vladimir Vuksanovic <109677816+vvuksanovic@users.noreply.github.com>
|
||||
Vladimir Zorin <vladimir@deviant.guru>
|
||||
Vladislav <vladplotnikov34@gmail.com>
|
||||
@@ -2139,6 +2189,7 @@ Wroclaw <wroclaw223@outlook.com>
|
||||
wsbagnsv1 <sclumpfpapa36@gmail.com>
|
||||
Wu Jian Ping <wujjpp@hotmail.com>
|
||||
Wu Jian Ping <wujp@greatld.com>
|
||||
WuXin <apuravmanhas123@gmail.com>
|
||||
wwoodsTM <104587230+wwoodsTM@users.noreply.github.com>
|
||||
Wyatt Caldwell <218154709+Detensable@users.noreply.github.com>
|
||||
wzy <32936898+Freed-Wu@users.noreply.github.com>
|
||||
@@ -2162,6 +2213,7 @@ Xinpeng Dou <15529241576@163.com>
|
||||
Xinpeng Dou <81913537+Dou-Git@users.noreply.github.com>
|
||||
xloem <0xloem@gmail.com>
|
||||
xris99 <79798089+xris99@users.noreply.github.com>
|
||||
Xuan Son Nguyen <son@huggingface.co>
|
||||
Xuan Son Nguyen <thichthat@gmail.com>
|
||||
Xuan-Son Nguyen <son@huggingface.co>
|
||||
Xuan-Son Nguyen <thichthat@gmail.com>
|
||||
@@ -2247,6 +2299,7 @@ zhangrunda <zhangrunda1234@outlook.com>
|
||||
zhangtao2-1 <478679312@qq.com>
|
||||
ZHAOKAI WANG <sanxianwei@163.com>
|
||||
Zhaolun Yin <129580161+ZhaolunYin@users.noreply.github.com>
|
||||
zhaoyshine <30611065+zhaoyshine@users.noreply.github.com>
|
||||
Zheng.Deng <32841220+dengzheng-cloud@users.noreply.github.com>
|
||||
zhentaoyu <zhentao.yu@intel.com>
|
||||
Zhenwei Jin <109658203+kylo5aby@users.noreply.github.com>
|
||||
|
||||
+1
-1
@@ -134,7 +134,7 @@ option(LLAMA_BUILD_TOOLS "llama: build tools"
|
||||
option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_SERVER "llama: build server example" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_APP "llama: build the unified binary" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_UI "llama: build the embedded Web UI for server" OFF)
|
||||
option(LLAMA_BUILD_UI "llama: build Llama Chat, the embedded chat app, for server" OFF)
|
||||
option(LLAMA_USE_PREBUILT_UI "llama: use prebuilt UI from HF Bucket when available" ON)
|
||||
option(LLAMA_CONNECT "llama: include the llama-connect tool (downloads a prebuilt binary)" OFF)
|
||||
|
||||
|
||||
+1
-1
@@ -102,7 +102,7 @@
|
||||
/tools/rpc/ @ggml-org/ggml-rpc
|
||||
/tools/server/* @ggml-org/llama-server # no subdir
|
||||
/tools/server/tests/ @ggml-org/llama-server
|
||||
/tools/ui/ @ggml-org/llama-ui
|
||||
/ui/ @ggml-org/llama-ui
|
||||
/tools/tokenize/ @ggerganov
|
||||
/tools/tts/ @ggerganov
|
||||
/vendor/ @ggerganov
|
||||
|
||||
@@ -16,6 +16,8 @@ The project differentiates between 3 levels of contributors:
|
||||
>
|
||||
> Detailed information regarding permissible and restricted uses of AI can be found in the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
### Requirements
|
||||
|
||||
If AI is used to generate any portion of the code, contributors must adhere to the following requirements:
|
||||
|
||||
1. Explicitly disclose the manner in which AI was employed.
|
||||
@@ -24,6 +26,16 @@ If AI is used to generate any portion of the code, contributors must adhere to t
|
||||
4. Be prepared to explain every line of code you submit when asked about it by a maintainer.
|
||||
5. It is strictly prohibited to use AI to write your posts for you (bug reports, feature requests, pull request descriptions, Github discussions, responding to humans, ...).
|
||||
|
||||
### Checklist
|
||||
|
||||
> [!IMPORTANT]
|
||||
>
|
||||
> **If you use AI, do all of the following:**
|
||||
>
|
||||
> 1. **Make sure [AGENTS.md](AGENTS.md) is loaded into your agent's context.** Skipping this step is one of the most common reasons PRs get rejected.
|
||||
> 2. **Search for related discussions, PRs and issues** to get the broader context before writing any code.
|
||||
> 3. **Run a local review pass before pushing anything**, using the [`code-review`](skills/code-review/SKILL.md) skill, once the change is implemented. Fix what it finds.
|
||||
|
||||
For more info, please refer to the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
# Pull requests (for contributors & collaborators)
|
||||
|
||||
@@ -13,10 +13,16 @@
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
|
||||
|
||||
[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
[v1.0.0](https://huggingface.co/blog/ggml-org/llama-cpp-v1) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
|
||||
</div>
|
||||
|
||||
## Description
|
||||
|
||||
The main goal of `llama.cpp` is to enable LLM (and MLLM) inference with minimal setup and state-of-the-art performance on a wide range of hardware - locally and in the cloud.
|
||||
|
||||
The `llama.cpp` project is built on top of the [ggml](https://github.com/ggml-org/ggml) library.
|
||||
|
||||
## Quick start
|
||||
|
||||
A few options to get `llama.cpp` installed on your machine:
|
||||
@@ -51,50 +57,12 @@ llama serve -hf ggml-org/Qwen3.5-0.8B-GGUF
|
||||
<i>VLM session with <b>llama cli</b></i>
|
||||
</td>
|
||||
<td align="center">
|
||||
<img width="1392" height="958" alt="Built-in web UI against `llama serve` running Qwen 3.6" src="https://github.com/user-attachments/assets/b402f972-2e32-4def-8771-8d849f08cf2e" />
|
||||
<i>Built-in web UI against <b>llama serve</b></i>
|
||||
<img width="1392" height="958" alt="Llama Chat, the built-in chat app, against `llama serve` running Qwen 3.6" src="https://github.com/user-attachments/assets/b402f972-2e32-4def-8771-8d849f08cf2e" />
|
||||
<i>Llama Chat, the built-in chat app, against <b>llama serve</b></i>
|
||||
</td>
|
||||
</tr>
|
||||
<table>
|
||||
|
||||
## Description
|
||||
|
||||
The main goal of `llama.cpp` is to enable LLM (and VLM) inference with minimal setup and state-of-the-art performance on
|
||||
a wide range of hardware - locally and in the cloud.
|
||||
|
||||
- Plain C/C++ implementation without any dependencies
|
||||
- Apple silicon is a first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks
|
||||
- AVX, AVX2, AVX512 and AMX support for x86 architectures
|
||||
- RVV, ZVFH, ZFH, ZICBOP and ZIHINTPAUSE support for RISC-V architectures
|
||||
- 1.5-bit, 2-bit, 3-bit, 4-bit, 5-bit, 6-bit, and 8-bit integer quantization for faster inference and reduced memory use
|
||||
- Custom CUDA kernels for running LLMs on NVIDIA GPUs (support for AMD GPUs via HIP and Moore Threads GPUs via MUSA)
|
||||
- Vulkan and SYCL backend support
|
||||
- CPU+GPU hybrid inference to partially accelerate models larger than the total VRAM capacity
|
||||
|
||||
The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-org/ggml) library.
|
||||
|
||||
## Supported backends
|
||||
|
||||
| Backend | Target devices |
|
||||
| --- | --- |
|
||||
| [BLAS](docs/build.md#blas-build) | All |
|
||||
| [BLIS](docs/backend/BLIS.md) | All |
|
||||
| [CANN](docs/build.md#cann) | Huawei Ascend NPU |
|
||||
| [CUDA](docs/build.md#cuda) | Nvidia GPU |
|
||||
| [HIP](docs/build.md#hip) | AMD GPU |
|
||||
| [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |
|
||||
| [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
|
||||
| [MUSA](docs/build.md#musa) | Moore Threads GPU |
|
||||
| [Metal](docs/build.md#metal-build) | Apple Silicon |
|
||||
| [OpenCL](docs/backend/OPENCL.md) | Adreno GPU |
|
||||
| [OpenVINO [In Progress]](docs/backend/OPENVINO.md) | Intel CPUs, GPUs, and NPUs |
|
||||
| [RPC](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) | All |
|
||||
| [SYCL](docs/backend/SYCL.md) | Intel GPU |
|
||||
| [VirtGPU](docs/backend/VirtGPU.md) | VirtGPU APIR |
|
||||
| [Vulkan](docs/build.md#vulkan) | GPU |
|
||||
| [WebGPU](docs/build.md#webgpu) | All |
|
||||
| [ZenDNN](docs/build.md#zendnn) | AMD CPU |
|
||||
|
||||
## Documentation
|
||||
|
||||
#### Tools
|
||||
@@ -102,6 +70,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or
|
||||
- [cli](tools/cli/README.md)
|
||||
- [completion](tools/completion/README.md)
|
||||
- [server](tools/server/README.md)
|
||||
- [Text-to-speech (TTS) models](docs/multimodal/tts.md)
|
||||
- [GBNF grammars](grammars/README.md)
|
||||
|
||||
#### Development
|
||||
|
||||
+1
-1
@@ -46,7 +46,7 @@ Only vulnerabilities that fall within these parts of the project are considered
|
||||
- `ggml/**/*`
|
||||
- `gguf-py/**/*`
|
||||
- `tools/server/*`, **excluding** the following topics:
|
||||
- Web UI
|
||||
- Llama Chat, the built-in chat app (`ui/apps/chat`)
|
||||
- Features marked as experimental
|
||||
- Features not recommended for use in untrusted environments (e.g., router, MCP)
|
||||
- Bugs that can lead to Denial-of-Service attack
|
||||
|
||||
@@ -122,6 +122,8 @@ if [ ! -z ${GG_BUILD_VULKAN} ]; then
|
||||
MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION="${MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION}/SPIRV-Headers/SPIRV-HeadersConfig.cmake"
|
||||
if [[ -f "${MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION}" || -h "${MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION}" ]]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DSPIRV-Headers_DIR=${MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION}/SPIRV-Headers"
|
||||
# SPIRV-Headers walks up three directories and appends /include, yielding /usr/local/lib/include; the extra vulkan/ level likely breaks its prefix calculation.
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN_SPIRV_INCLUDE_DIR=/usr/local/include"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
+27
-7
@@ -349,6 +349,14 @@ common_models_handler common_models_handler_init(const common_params & params, l
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH) != params.speculative.types.end();
|
||||
|
||||
const bool spec_type_draft_dflash2 = std::find(params.speculative.types.begin(),
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH2) != params.speculative.types.end();
|
||||
|
||||
const bool spec_type_draft_lilicorr = std::find(params.speculative.types.begin(),
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_LILICORR) != params.speculative.types.end();
|
||||
|
||||
const bool spec_type_draft_eagle3 = std::find(params.speculative.types.begin(),
|
||||
params.speculative.types.end(),
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3) != params.speculative.types.end();
|
||||
@@ -373,7 +381,7 @@ common_models_handler common_models_handler_init(const common_params & params, l
|
||||
opts.offline = params.offline;
|
||||
opts.download_mtp = spec_type_draft_mtp;
|
||||
opts.download_eagle3 = spec_type_draft_eagle3;
|
||||
opts.download_dflash = spec_type_draft_dflash;
|
||||
opts.download_dflash = spec_type_draft_dflash || spec_type_draft_dflash2 || spec_type_draft_lilicorr;
|
||||
opts.download_dspark = spec_type_draft_dspark;
|
||||
opts.download_mmproj = use_mmproj && !params.no_mmproj
|
||||
&& params.mmproj.path.empty() && params.mmproj.url.empty();
|
||||
@@ -904,6 +912,10 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
|
||||
throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");
|
||||
}
|
||||
|
||||
if (params.sampling.ignore_eos && !common_grammar_value(params.sampling.grammar).empty()) {
|
||||
throw std::invalid_argument("error: --ignore-eos is not supported with --grammar or --json-schema\n");
|
||||
}
|
||||
|
||||
const bool skip_model_download =
|
||||
// server will call common_params_handle_models() later, so we skip it here
|
||||
ctx_arg.ex == LLAMA_EXAMPLE_SERVER ||
|
||||
@@ -2821,7 +2833,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
const std::regex regex{ R"([,/]+)" };
|
||||
std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
|
||||
std::vector<std::string> split_arg{ it, {} };
|
||||
if (split_arg.size() >= llama_max_devices()) {
|
||||
if (split_arg.size() > llama_max_devices()) {
|
||||
throw std::invalid_argument(
|
||||
string_format("got %zu input configs, but system only has %zu devices", split_arg.size(), llama_max_devices())
|
||||
);
|
||||
@@ -2887,7 +2899,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
const std::regex regex{ R"([,/]+)" };
|
||||
std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
|
||||
std::vector<std::string> split_arg{ it, {} };
|
||||
if (split_arg.size() >= llama_max_devices()) {
|
||||
if (split_arg.size() > llama_max_devices()) {
|
||||
throw std::invalid_argument(
|
||||
string_format("got %zu input configs, but system only has %zu devices", split_arg.size(), llama_max_devices())
|
||||
);
|
||||
@@ -3408,7 +3420,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui-mcp-proxy", "--webui-mcp-proxy"},
|
||||
{"--no-ui-mcp-proxy", "--no-webui-mcp-proxy"},
|
||||
"experimental: whether to enable MCP CORS proxy, for the Web UI to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)\n"
|
||||
"experimental: whether to enable MCP CORS proxy, for Llama Chat to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)\n"
|
||||
"note: for security reasons, this will limit --cors-origins to localhost by default",
|
||||
[](common_params & params, bool value) {
|
||||
params.ui_mcp_proxy = value;
|
||||
@@ -3477,7 +3489,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui", "--webui"},
|
||||
{"--no-ui", "--no-webui"},
|
||||
string_format("whether to enable the Web UI (default: %s)", params.ui ? "enabled" : "disabled"),
|
||||
string_format("whether to enable Llama Chat, the built-in chat app (default: %s)", params.ui ? "enabled" : "disabled"),
|
||||
[](common_params & params, bool value) {
|
||||
params.ui = value;
|
||||
}
|
||||
@@ -4494,7 +4506,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--tts-lang"}, "FNAME",
|
||||
"language (ISO 639-1) for audio generation\n"
|
||||
"see tts/README.md for per-model usage notes",
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_lang = value;
|
||||
}
|
||||
@@ -4502,7 +4514,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--tts-voice"}, "VOICE",
|
||||
"voice for audio generation (model-dependent)\n"
|
||||
"see tts/README.md for per-model usage notes",
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_voice = value;
|
||||
}
|
||||
@@ -4514,6 +4526,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.tts_speaker_file = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-speaker-text"}, "TEXT",
|
||||
"transcript of --tts-speaker-file, for models that need it to clone the voice\n"
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_speaker_text = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
|
||||
//
|
||||
// diffusion params
|
||||
|
||||
@@ -562,7 +562,8 @@ common_peg_parser common_chat_peg_builder::python_or_json_value() {
|
||||
common_peg_parser common_chat_peg_builder::python_style_tool_calls(
|
||||
const ordered_json & tools,
|
||||
bool parallel_tool_calls,
|
||||
bool allow_json_literals) {
|
||||
bool allow_json_literals,
|
||||
bool allow_undeclared) {
|
||||
if (!tools.is_array() || tools.empty()) {
|
||||
return eps();
|
||||
}
|
||||
@@ -620,6 +621,18 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls(
|
||||
tool_choices |= rule("tool-" + std::to_string(i), tool_parser);
|
||||
}
|
||||
|
||||
// last, so declared tools win when parsing
|
||||
if (allow_undeclared) {
|
||||
auto ident = rule("tool-undeclared-ident", chars("[a-zA-Z_]", 1, 1) + chars("[a-zA-Z0-9_.]", 0, -1));
|
||||
auto value = tool_arg_value(allow_json_literals ? python_or_json_value() : python_value());
|
||||
auto arg = tool_arg(tool_arg_open(tool_arg_name(ident) + literal("=")) + value + tool_arg_close(eps()));
|
||||
auto args = optional(arg + zero_or_more("," + space() + arg));
|
||||
|
||||
tool_choices |= rule("tool-undeclared", tool(tool_open(tool_name(ident) + literal("(")) +
|
||||
space() + tool_args(args) + space() + tool_close(literal(")"))
|
||||
));
|
||||
}
|
||||
|
||||
if (parallel_tool_calls) {
|
||||
return "[" + space() + tool_choices + zero_or_more("," + space() + tool_choices) + space() + "]";
|
||||
}
|
||||
|
||||
@@ -161,9 +161,11 @@ class common_chat_peg_builder : public common_peg_parser_builder {
|
||||
|
||||
// Helper for Python-style function call format: name(arg1="value1", arg2=123)
|
||||
// Used by LFM2 and similar templates
|
||||
// allow_undeclared: also accept calls to names not in tools (any identifier, any arguments)
|
||||
common_peg_parser python_style_tool_calls(const common_json & tools,
|
||||
bool parallel_tool_calls,
|
||||
bool allow_json_literals);
|
||||
bool allow_json_literals,
|
||||
bool allow_undeclared = false);
|
||||
|
||||
private:
|
||||
// Python values plus JSON true/false/null.
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
#include "jinja/runtime.h"
|
||||
#include "jinja/value.h"
|
||||
#include "json.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <climits>
|
||||
@@ -857,8 +858,45 @@ static value tts_norm_builtin_sub(const func_args & args) {
|
||||
return mk_val<value_string>(string(res, text_val.all_parts_are_input()));
|
||||
}
|
||||
|
||||
void common_chat_tts_norm_add_builtins(context & ctx) {
|
||||
bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key, std::string & value) {
|
||||
const auto it = std::lower_bound(dict.keys.begin(), dict.keys.end(), key);
|
||||
if (it == dict.keys.end() || *it != key) {
|
||||
return false;
|
||||
}
|
||||
const size_t idx = it - dict.keys.begin();
|
||||
if (idx >= dict.values.size()) {
|
||||
throw std::runtime_error("tts_dict: malformed dictionary, a key has no value");
|
||||
}
|
||||
value = dict.values[idx];
|
||||
return true;
|
||||
}
|
||||
|
||||
void common_chat_tts_norm_add_builtins(context & ctx, std::shared_ptr<const common_tts_dicts> dicts, common_tts_g2p g2p) {
|
||||
ctx.set_val("tts_norm_scan", mk_val<value_func>("tts_norm_scan", tts_norm_builtin_scan));
|
||||
ctx.set_val("tts_norm_sub", mk_val<value_func>("tts_norm_sub", tts_norm_builtin_sub));
|
||||
ctx.set_val("tts_dict", mk_val<value_func>("tts_dict", [dicts](const func_args & args) -> value {
|
||||
args.ensure_count(2, 2);
|
||||
args.ensure_vals<value_string, value_string>();
|
||||
const std::string name = args.get_pos(0)->as_string().str();
|
||||
const auto it = dicts ? dicts->find(name) : common_tts_dicts::const_iterator();
|
||||
if (!dicts || it == dicts->end()) {
|
||||
throw std::runtime_error("tts_dict: unknown dictionary '" + name + "'");
|
||||
}
|
||||
std::string doc;
|
||||
if (!common_tts_dict_find(it->second, args.get_pos(1)->as_string().str(), doc)) {
|
||||
return mk_val<value_none>();
|
||||
}
|
||||
return value_from_json(common_json::parse(doc), false);
|
||||
}));
|
||||
// none when the caller gives no letter-to-sound model, or the model has no pronunciation
|
||||
ctx.set_val("tts_g2p", mk_val<value_func>("tts_g2p", [g2p](const func_args & args) -> value {
|
||||
args.ensure_count(1, 1);
|
||||
args.ensure_vals<value_string>();
|
||||
const std::string out = g2p ? g2p(args.get_pos(0)->as_string().str()) : std::string();
|
||||
if (out.empty()) {
|
||||
return mk_val<value_none>();
|
||||
}
|
||||
return mk_val<value_string>(out);
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
+22
-3
@@ -1,6 +1,8 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
@@ -11,9 +13,10 @@
|
||||
// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this
|
||||
// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input
|
||||
//
|
||||
// two functions are exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
|
||||
// functions exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
|
||||
// tts_norm_scan(text, rules) -> list of segments {rule, text, groups}
|
||||
// tts_norm_sub(text, pattern, repl) -> string
|
||||
// tts_dict(name, key) -> value of key in a dictionary of the model, or none
|
||||
|
||||
namespace jinja {
|
||||
struct context;
|
||||
@@ -57,6 +60,22 @@ std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & te
|
||||
// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent)
|
||||
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl);
|
||||
|
||||
// add tts_norm_scan() and tts_norm_sub() to the context of a task template
|
||||
// a dictionary of the model, e.g. a pronunciation lexicon
|
||||
struct common_tts_dict {
|
||||
std::vector<std::string> keys; // sorted by their bytes
|
||||
std::vector<std::string> values; // the JSON document of each key
|
||||
};
|
||||
|
||||
using common_tts_dicts = std::map<std::string, common_tts_dict>;
|
||||
|
||||
// binary search of key in a dictionary; on success, value is set to the JSON document
|
||||
bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key, std::string & value);
|
||||
|
||||
// pronunciation of a word by a letter-to-sound model, as phonemes separated by spaces (e.g. CMUdict notation)
|
||||
// empty when there is none; the notation is the one of the model, its "tts" template knows it
|
||||
using common_tts_g2p = std::function<std::string(const std::string & word)>;
|
||||
|
||||
// add tts_norm_scan(), tts_norm_sub(), tts_dict() and tts_g2p() to the context of a task template
|
||||
// they are not jinja global builtins on purpose: chat templates are not affected
|
||||
void common_chat_tts_norm_add_builtins(jinja::context & ctx);
|
||||
void common_chat_tts_norm_add_builtins(jinja::context & ctx, std::shared_ptr<const common_tts_dicts> dicts = nullptr,
|
||||
common_tts_g2p g2p = nullptr);
|
||||
|
||||
+51
-18
@@ -318,6 +318,7 @@ struct common_chat_templates {
|
||||
std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)
|
||||
std::unique_ptr<common_chat_template> template_tool_use;
|
||||
std::unique_ptr<common_chat_template> template_tts; // the prompt of a text to speak
|
||||
std::shared_ptr<common_tts_dicts> tts_dicts; // dictionaries of the model for tts_dict(), by name
|
||||
};
|
||||
|
||||
common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {
|
||||
@@ -806,10 +807,11 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
|
||||
}
|
||||
|
||||
// renders a task template (e.g. "tts") with the given variables
|
||||
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) {
|
||||
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars, const std::shared_ptr<common_tts_dicts> & tts_dicts,
|
||||
const common_tts_g2p & g2p = nullptr) {
|
||||
jinja::context ctx(tmpl.source());
|
||||
// text normalization helpers: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md
|
||||
common_chat_tts_norm_add_builtins(ctx);
|
||||
// text normalization helpers: tts_norm_scan(), tts_norm_sub(), tts_dict() and tts_g2p(), see docs/tts-norm-patterns.md
|
||||
common_chat_tts_norm_add_builtins(ctx, tts_dicts, g2p);
|
||||
jinja::global_from_json(ctx, vars, false);
|
||||
ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
|
||||
ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
|
||||
@@ -827,6 +829,11 @@ static std::string string_lower(std::string s) {
|
||||
return s;
|
||||
}
|
||||
|
||||
// VibeVoice-ASR: default system prompt of the template written by the converter
|
||||
static bool is_vibevoice_asr_template(const std::string & src) {
|
||||
return src.find("transcribes audio input into text output in JSON format") != std::string::npos;
|
||||
}
|
||||
|
||||
// LFM3-Audio: language code -> name
|
||||
static std::string lfm3_language_name(const std::string & language) {
|
||||
static const std::map<std::string, std::string> names = {
|
||||
@@ -847,7 +854,8 @@ static std::string lfm3_asr_system_prompt(const std::string & language) {
|
||||
return "Perform ASR in " + lfm3_language_name(language) + ".";
|
||||
}
|
||||
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language,
|
||||
double audio_length, const std::string & context) {
|
||||
common_chat_prompt_preset asr_preset;
|
||||
asr_preset.system = "";
|
||||
asr_preset.user = "Transcribe audio to text";
|
||||
@@ -861,6 +869,17 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates
|
||||
asr_preset.system = lfm3_asr_system_prompt(language);
|
||||
asr_preset.user = "";
|
||||
asr_preset.has_language = true;
|
||||
} else if (is_vibevoice_asr_template(src)) {
|
||||
// without the audio length the model does not know where the audio ends and loops
|
||||
const char * keys = "transcribe it with these keys: Start time, End time, Speaker ID, Content";
|
||||
if (context.empty()) {
|
||||
asr_preset.user = string_format("\nThis is a %.2f seconds audio, please %s", audio_length, keys);
|
||||
} else {
|
||||
asr_preset.user = string_format("\nThis is a %.2f seconds audio, with extra info: %s\n\nPlease %s", audio_length, context.c_str(), keys);
|
||||
asr_preset.has_context = true;
|
||||
}
|
||||
asr_preset.media_first = true;
|
||||
asr_preset.needs_audio_length = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -868,7 +887,7 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates
|
||||
}
|
||||
|
||||
std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
|
||||
const std::string & language, const std::string & voice) {
|
||||
const std::string & language, const std::string & voice, const common_tts_g2p & g2p) {
|
||||
if (!chat_templates || !chat_templates->template_tts) {
|
||||
return "";
|
||||
}
|
||||
@@ -876,7 +895,7 @@ std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templa
|
||||
{"messages", json::array({{{"role", "user"}, {"content", text}}})},
|
||||
{"language", language},
|
||||
{"voice", voice},
|
||||
});
|
||||
}, chat_templates->tts_dicts, g2p);
|
||||
}
|
||||
|
||||
std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
|
||||
@@ -955,18 +974,6 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
|
||||
"{%- if false %}");
|
||||
}
|
||||
|
||||
// TODO @aldehir : this is a temporary fix, pending Minja changes
|
||||
// Ref: https://github.com/ggml-org/llama.cpp/pull/17713#issuecomment-3631342664
|
||||
if (default_template_src.find("[TOOL_CALLS]") != std::string::npos
|
||||
// search for the error message and patch it
|
||||
&& default_template_src.find("if (message['content'] is none or") != std::string::npos) {
|
||||
string_replace_all(default_template_src,
|
||||
"{%- if (message['content'] is none or message['content'] == '' or "
|
||||
"message['content']|length == 0) and (message['tool_calls'] is not defined or "
|
||||
"message['tool_calls'] is none or message['tool_calls']|length == 0) %}",
|
||||
"{%- if false %}");
|
||||
}
|
||||
|
||||
std::string token_bos = bos_token_override;
|
||||
std::string token_eos = eos_token_override;
|
||||
bool add_bos = false;
|
||||
@@ -1019,6 +1026,25 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
|
||||
LOG_ERR("%s: failed to parse the tts chat template (ignoring it): %s\n", __func__, e.what());
|
||||
}
|
||||
}
|
||||
// dictionaries for tts_dict(), listed by name in tokenizer.tts_dicts
|
||||
tmpls->tts_dicts = std::make_shared<common_tts_dicts>();
|
||||
for (int32_t i = 0; i < llama_model_meta_arr_n(model, "tokenizer.tts_dicts"); i++) {
|
||||
const std::string name = llama_model_meta_arr_str(model, "tokenizer.tts_dicts", i);
|
||||
const std::string key = "tokenizer.tts_dict." + name + ".keys";
|
||||
const std::string val_key = "tokenizer.tts_dict." + name + ".values";
|
||||
const int32_t n = llama_model_meta_arr_n(model, key.c_str());
|
||||
if (n < 0 || llama_model_meta_arr_n(model, val_key.c_str()) != n) {
|
||||
LOG_ERR("%s: the tts dictionary '%s' has no value for each of its keys (ignoring it)\n", __func__, name.c_str());
|
||||
continue;
|
||||
}
|
||||
auto & dict = (*tmpls->tts_dicts)[name];
|
||||
dict.keys.reserve(n);
|
||||
dict.values.reserve(n);
|
||||
for (int32_t j = 0; j < n; j++) {
|
||||
dict.keys.emplace_back(llama_model_meta_arr_str(model, key.c_str(), j));
|
||||
dict.values.emplace_back(llama_model_meta_arr_str(model, val_key.c_str(), j));
|
||||
}
|
||||
}
|
||||
}
|
||||
return tmpls;
|
||||
}
|
||||
@@ -1439,6 +1465,12 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
return common_chat_params_init_qwen3_coder;
|
||||
}
|
||||
|
||||
// Inkling 1 and 2
|
||||
if (is_inkling_template(src)) {
|
||||
LOG_DBG("Using specialized template: Inkling\n");
|
||||
return common_chat_params_init_inkling;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
@@ -1971,5 +2003,6 @@ common_json common_chat_templates_get_caps(const common_chat_templates * chat_te
|
||||
: chat_templates->template_default->caps;
|
||||
common_json result = caps.to_map();
|
||||
result["reasoning_effort_levels"] = caps.reasoning_effort_levels;
|
||||
result["reasoning_effort_rejected"] = caps.reasoning_effort_rejected;
|
||||
return result;
|
||||
}
|
||||
|
||||
+11
-3
@@ -491,15 +491,23 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
|
||||
struct common_chat_prompt_preset {
|
||||
std::string system;
|
||||
std::string user;
|
||||
bool has_language = false; // the preset already names the requested language
|
||||
bool has_language = false; // the preset already names the requested language
|
||||
bool has_context = false; // the user text already holds the context
|
||||
bool media_first = false; // the media goes before the user text
|
||||
bool needs_audio_length = false; // the user text depends on the audio length, call again with it
|
||||
};
|
||||
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "");
|
||||
// audio_length: in seconds, 0 if unknown
|
||||
// context: extra info from the user (e.g. names, hotwords), only used by some presets, see has_context
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "",
|
||||
double audio_length = 0.0, const std::string & context = "");
|
||||
|
||||
// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
|
||||
// language and voice are optional, the template defines their defaults
|
||||
// g2p (optional): the letter-to-sound model of the template's tts_g2p(), e.g. mtmd_gen_audio_g2p()
|
||||
std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
|
||||
const std::string & language = "", const std::string & voice = "");
|
||||
const std::string & language = "", const std::string & voice = "",
|
||||
const std::function<std::string(const std::string & word)> & g2p = nullptr);
|
||||
|
||||
// system prompt line for the speech language and voice (e.g. LFM3-Audio), empty if not used
|
||||
std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
|
||||
|
||||
+20
-12
@@ -174,18 +174,20 @@ enum common_params_sampling_config : uint64_t {
|
||||
};
|
||||
|
||||
enum common_speculative_type {
|
||||
COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head)
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values
|
||||
COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head)
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH2, // DFlash2 speculative decoding (DFlash + selector)
|
||||
COMMON_SPECULATIVE_TYPE_DRAFT_LILICORR, // LiliCorr speculative decoding (DFlash + scorer)
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_MOD,
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache
|
||||
COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type
|
||||
COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache
|
||||
COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type
|
||||
};
|
||||
|
||||
// Grammar type enumeration
|
||||
@@ -414,7 +416,12 @@ struct common_params_speculative {
|
||||
|
||||
uint32_t need_n_rs_seq() const {
|
||||
bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {
|
||||
return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
|
||||
return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH2 ||
|
||||
t == COMMON_SPECULATIVE_TYPE_DRAFT_LILICORR;
|
||||
});
|
||||
|
||||
return needs_rs_seq ? draft.n_max : 0u;
|
||||
@@ -789,6 +796,7 @@ struct common_params {
|
||||
std::string tts_lang = "";
|
||||
std::string tts_voice = "";
|
||||
std::string tts_speaker_file = "";
|
||||
std::string tts_speaker_text = "";
|
||||
|
||||
bool is_gen_docs = false; // whether we are running inside llama-gen-docs
|
||||
};
|
||||
|
||||
+11
-1
@@ -627,9 +627,14 @@ caps caps_get(jinja::program & prog) {
|
||||
// levels sharing a render are aliases and the highest one is kept
|
||||
const auto unknown = render("REASONING_EFFORT_UNKNOWN");
|
||||
std::vector<std::string> renders;
|
||||
std::vector<std::string> rejected;
|
||||
for (const auto & level : candidates) {
|
||||
const auto rendered = render(level);
|
||||
if (!rendered || rendered == unknown) {
|
||||
if (!rendered) {
|
||||
rejected.push_back(level);
|
||||
continue;
|
||||
}
|
||||
if (rendered == unknown) {
|
||||
continue;
|
||||
}
|
||||
const auto it = std::find(renders.begin(), renders.end(), *rendered);
|
||||
@@ -640,6 +645,11 @@ caps caps_get(jinja::program & prog) {
|
||||
result.reasoning_effort_levels.push_back(level);
|
||||
}
|
||||
}
|
||||
|
||||
// a template that renders no level fails the probe for another reason, so its failures say nothing about the levels
|
||||
if (!result.reasoning_effort_levels.empty()) {
|
||||
result.reasoning_effort_rejected = rejected;
|
||||
}
|
||||
}
|
||||
|
||||
JJ_DEBUG("%s\n", result.to_string().c_str());
|
||||
|
||||
@@ -23,6 +23,9 @@ struct caps {
|
||||
// reasoning effort levels the template renders distinctly, from lowest to highest
|
||||
std::vector<std::string> reasoning_effort_levels;
|
||||
|
||||
// reasoning effort levels the template raises on
|
||||
std::vector<std::string> reasoning_effort_rejected;
|
||||
|
||||
// one of the 2 content capabilities must be true
|
||||
bool supports_string_content = true;
|
||||
bool supports_typed_content = false;
|
||||
|
||||
@@ -263,41 +263,6 @@ lexer_result lexer::tokenize(const std::string & source) {
|
||||
|
||||
char ch = src[pos];
|
||||
|
||||
bool is_closing_block = ch == '-' && next_pos_is( {'%', '}'} );
|
||||
|
||||
// Check for unary operators
|
||||
if (!is_closing_block && (ch == '-' || ch == '+')) {
|
||||
start_pos = pos;
|
||||
token::type last_token_type = tokens.empty() ? token::eof : tokens.back().t;
|
||||
if (last_token_type == token::text || last_token_type == token::eof) {
|
||||
throw lexer_exception(std::string("unexpected character: ") + ch, source, pos);
|
||||
}
|
||||
switch (last_token_type) {
|
||||
case token::identifier:
|
||||
case token::numeric_literal:
|
||||
case token::string_literal:
|
||||
case token::close_paren:
|
||||
case token::close_square_bracket:
|
||||
// Part of a binary operator
|
||||
// a - 1, 1 - 1, true - 1, "apple" - 1, (1) - 1, a[1] - 1
|
||||
// Continue parsing normally
|
||||
break;
|
||||
default: {
|
||||
// Is part of a unary operator
|
||||
// (-1), [-1], (1 + -1), not -1, -apple
|
||||
++pos; // Consume the operator
|
||||
|
||||
// Check for numbers following the unary operator
|
||||
std::string num = consume_numeric();
|
||||
std::string value = std::string(1, ch) + num;
|
||||
token::type t = num.empty() ? token::unary_operator : token::numeric_literal;
|
||||
// JJ_DEBUG("consumed unary operator or numeric literal: '%s'", value.c_str());
|
||||
tokens.push_back({t, value, start_pos});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try to match one of the tokens in the mapping table
|
||||
bool matched = false;
|
||||
for (const auto & [seq, typ] : ordered_mapping_table) {
|
||||
|
||||
@@ -39,7 +39,6 @@ struct token {
|
||||
multiplicative_binary_operator, // * / %
|
||||
power_operator, // **
|
||||
comparison_binary_operator, // < > <= >= == !=
|
||||
unary_operator, // ! - +
|
||||
comment, // {# ... #}
|
||||
};
|
||||
type t;
|
||||
@@ -74,7 +73,6 @@ static std::string type_to_string(token::type t) {
|
||||
case token::multiplicative_binary_operator: return "multiplicative_binary_operator";
|
||||
case token::power_operator: return "power_operator";
|
||||
case token::comparison_binary_operator: return "comparison_binary_operator";
|
||||
case token::unary_operator: return "unary_operator";
|
||||
case token::comment: return "comment";
|
||||
default: return "unknown";
|
||||
}
|
||||
@@ -141,6 +139,7 @@ struct lexer {
|
||||
{"~", token::additive_binary_operator},
|
||||
{"**", token::power_operator},
|
||||
{"*", token::multiplicative_binary_operator},
|
||||
{"//", token::multiplicative_binary_operator},
|
||||
{"/", token::multiplicative_binary_operator},
|
||||
{"%", token::multiplicative_binary_operator},
|
||||
// Assignment operator
|
||||
|
||||
@@ -482,7 +482,8 @@ private:
|
||||
}
|
||||
|
||||
statement_ptr parse_unary_expression() {
|
||||
if (is(token::unary_operator)) {
|
||||
// a sign where an operand is expected is unary: -x, 1 + -x, x if c else -1
|
||||
if (is(token::additive_binary_operator) && peek().value != "~") {
|
||||
depth_guard guard(*this);
|
||||
size_t start_pos = current;
|
||||
auto op = next();
|
||||
|
||||
+95
-49
@@ -89,21 +89,22 @@ value statement::execute(context & ctx) const {
|
||||
}
|
||||
|
||||
value identifier::execute_impl(context & ctx) const {
|
||||
auto it = ctx.get_val(val);
|
||||
auto builtins = global_builtins();
|
||||
if (!it->is_undefined()) {
|
||||
auto it = ctx.find_val(key);
|
||||
if (it && !it->is_undefined()) {
|
||||
if (ctx.is_get_stats) {
|
||||
value_t::stats_t::mark_used(it);
|
||||
}
|
||||
JJ_DEBUG("Identifier '%s' found, type = %s", val.c_str(), it->type().c_str());
|
||||
return it;
|
||||
} else if (builtins.find(val) != builtins.end()) {
|
||||
JJ_DEBUG("Identifier '%s' found in builtins", val.c_str());
|
||||
return mk_val<value_func>(val, builtins.at(val));
|
||||
} else {
|
||||
JJ_DEBUG("Identifier '%s' not found, returning undefined", val.c_str());
|
||||
return mk_val<value_undefined>(val);
|
||||
}
|
||||
const auto & builtins = global_builtins();
|
||||
auto builtin = builtins.find(val);
|
||||
if (builtin != builtins.end()) {
|
||||
JJ_DEBUG("Identifier '%s' found in builtins", val.c_str());
|
||||
return mk_val<value_func>(val, builtin->second);
|
||||
}
|
||||
JJ_DEBUG("Identifier '%s' not found, returning undefined", val.c_str());
|
||||
return mk_val<value_undefined>(val);
|
||||
}
|
||||
|
||||
value object_literal::execute_impl(context & ctx) const {
|
||||
@@ -190,11 +191,30 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
|
||||
if (is_float) {
|
||||
return mk_val<value_float>(res);
|
||||
} else {
|
||||
return mk_val<value_int>(static_cast<int64_t>(res));
|
||||
return mk_val<value_int>(float_to_int(res));
|
||||
}
|
||||
} else if (op.value == "/") {
|
||||
JJ_DEBUG("Division operation: %f / %f", a, b);
|
||||
return mk_val<value_float>(a / b);
|
||||
} else if (op.value == "//") {
|
||||
// python semantics: round toward -inf
|
||||
if (is_val<value_int>(left_val) && is_val<value_int>(right_val)) {
|
||||
const int64_t x = left_val->as_int();
|
||||
const int64_t y = right_val->as_int();
|
||||
if (y == 0) {
|
||||
throw std::runtime_error("integer division by zero");
|
||||
}
|
||||
if (y == -1) {
|
||||
// INT64_MIN / -1 is UB, wrap around instead
|
||||
return mk_val<value_int>((int64_t) (0 - (uint64_t) x));
|
||||
}
|
||||
int64_t q = x / y;
|
||||
if (x % y != 0 && (x < 0) != (y < 0)) {
|
||||
q -= 1;
|
||||
}
|
||||
return mk_val<value_int>(q);
|
||||
}
|
||||
return mk_val<value_float>(std::floor(a / b));
|
||||
} else if (op.value == "%") {
|
||||
double rem = std::fmod(a, b);
|
||||
JJ_DEBUG("Modulo operation: %f %% %f = %f", a, b, rem);
|
||||
@@ -202,7 +222,7 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
|
||||
if (is_float) {
|
||||
return mk_val<value_float>(rem);
|
||||
} else {
|
||||
return mk_val<value_int>(static_cast<int64_t>(rem));
|
||||
return mk_val<value_int>(float_to_int(rem));
|
||||
}
|
||||
} else if (op.value == "<") {
|
||||
JJ_DEBUG("Comparison operation: %f < %f is %d", a, b, a < b);
|
||||
@@ -273,12 +293,17 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
|
||||
// String membership
|
||||
if (is_val<value_string>(left_val) && is_val<value_string>(right_val)) {
|
||||
// case: "a" in "abc"
|
||||
bool member = test_is_in();
|
||||
if (op.value == "in") {
|
||||
return mk_val<value_bool>(member);
|
||||
} else if (op.value == "not in") {
|
||||
return mk_val<value_bool>(!member);
|
||||
if (op.value == "in" || op.value == "not in") {
|
||||
bool member = test_is_in();
|
||||
return mk_val<value_bool>(op.value == "in" ? member : !member);
|
||||
}
|
||||
// case: "a" < "b"; UTF-8 byte order is the same as codepoint order, like python
|
||||
const std::string a = left_val->as_string().str();
|
||||
const std::string b = right_val->as_string().str();
|
||||
if (op.value == "<") { return mk_val<value_bool>(a < b); }
|
||||
if (op.value == ">") { return mk_val<value_bool>(a > b); }
|
||||
if (op.value == "<=") { return mk_val<value_bool>(a <= b); }
|
||||
if (op.value == ">=") { return mk_val<value_bool>(a >= b); }
|
||||
}
|
||||
|
||||
// Value key in object
|
||||
@@ -693,7 +718,7 @@ value set_statement::execute_impl(context & ctx) const {
|
||||
}
|
||||
auto obj_ptr = cast_val<value_object>(object);
|
||||
JJ_DEBUG("Setting object property '%s' with value type %s", prop_name.c_str(), rhs->type().c_str());
|
||||
obj_ptr->insert(prop_name, rhs);
|
||||
obj_ptr->insert(cached_val<value_string>(ctx, member->static_property), rhs);
|
||||
|
||||
} else {
|
||||
throw std::runtime_error("Invalid LHS inside assignment expression: " + assignee->type());
|
||||
@@ -786,13 +811,14 @@ value call_statement::execute_impl(context & ctx) const {
|
||||
}
|
||||
auto * callee_func = cast_val<value_func>(callee_val);
|
||||
|
||||
context caller_ctx(ctx); // new scope for caller execution
|
||||
// caller() can be saved and called after this scope ended, it owns a snapshot of the variables
|
||||
auto caller_ctx = std::make_shared<context>(ctx.snapshot());
|
||||
|
||||
const func_handler func = [this, caller_ctx = std::move(caller_ctx)](const func_args & args) -> value {
|
||||
const func_handler func = [this, caller_ctx](const func_args & args) -> value {
|
||||
if (args.ctx.macro_depth >= JINJA_RECURSIVE_LIMIT) {
|
||||
throw std::runtime_error("macro calls are nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
|
||||
}
|
||||
context block_ctx(caller_ctx); // new scope for block execution
|
||||
context block_ctx(*caller_ctx); // new scope for block execution
|
||||
block_ctx.macro_depth = args.ctx.macro_depth + 1;
|
||||
|
||||
bind_parameters("caller", this->caller_args, args, block_ctx);
|
||||
@@ -813,6 +839,19 @@ value call_statement::execute_impl(context & ctx) const {
|
||||
return callee_func->invoke(args);
|
||||
}
|
||||
|
||||
member_expression::member_expression(statement_ptr && object, statement_ptr && property, bool computed) :
|
||||
object(std::move(object)),
|
||||
property(std::move(property)),
|
||||
computed(computed) {
|
||||
chk_type<expression>(this->object);
|
||||
chk_type<expression>(this->property);
|
||||
if (!computed) {
|
||||
if (const auto * id = cast_stmt<identifier>(this->property)) {
|
||||
static_property = id->key;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
value member_expression::execute_impl(context & ctx) const {
|
||||
value object = this->object->execute(ctx);
|
||||
|
||||
@@ -821,14 +860,13 @@ value member_expression::execute_impl(context & ctx) const {
|
||||
// syntax: obj[expr]
|
||||
JJ_DEBUG("Member expression, computing property type %s", this->property->type().c_str());
|
||||
|
||||
int64_t arr_size = 0;
|
||||
if (is_val<value_array>(object)) {
|
||||
arr_size = object->as_array().size();
|
||||
} else if (is_val<value_string>(object)) {
|
||||
arr_size = object->as_string().length();
|
||||
}
|
||||
|
||||
if (is_stmt<slice_expression>(this->property)) {
|
||||
int64_t arr_size = 0;
|
||||
if (is_val<value_array>(object)) {
|
||||
arr_size = static_cast<int64_t>(object->as_array().size());
|
||||
} else if (is_val<value_string>(object)) {
|
||||
arr_size = static_cast<int64_t>(cast_val<value_string>(object)->val_str.length());
|
||||
}
|
||||
auto s = cast_stmt<slice_expression>(this->property);
|
||||
value step_val = s->step_expr ? s->step_expr->execute(ctx) : mk_val<value_int>(1);
|
||||
value start_val = s->start_expr ? s->start_expr->execute(ctx) : (step_val->as_int() < 0 ? mk_val<value_int>(arr_size - 1) : mk_val<value_int>(0));
|
||||
@@ -850,7 +888,7 @@ value member_expression::execute_impl(context & ctx) const {
|
||||
}
|
||||
} else if (is_stmt<integer_literal>(this->property)) {
|
||||
// syntax: obj.index
|
||||
property = mk_val<value_int>(cast_stmt<integer_literal>(this->property)->val);
|
||||
property = this->property->execute(ctx);
|
||||
if (property->as_int() < 0) {
|
||||
throw std::runtime_error("Static member property cannot be negative");
|
||||
}
|
||||
@@ -859,8 +897,8 @@ value member_expression::execute_impl(context & ctx) const {
|
||||
if (!is_stmt<identifier>(this->property)) {
|
||||
throw std::runtime_error("Static member property must be an identifier");
|
||||
}
|
||||
property = mk_val<value_string>(cast_stmt<identifier>(this->property)->val);
|
||||
std::string prop = property->as_string().str();
|
||||
property = cached_val<value_string>(ctx, static_property);
|
||||
const auto & prop = cast_stmt<identifier>(this->property)->val;
|
||||
JJ_DEBUG("Member expression, object type %s, static property '%s'", object->type().c_str(), prop.c_str());
|
||||
|
||||
// behavior of jinja2: obj having prop as a built-in function AND 'prop', as an object key,
|
||||
@@ -868,51 +906,56 @@ value member_expression::execute_impl(context & ctx) const {
|
||||
// while obj['prop'] returns the property value.
|
||||
// example: {"obj": {"items": 123}} -> obj.items is the built-in function, obj['items'] is 123
|
||||
|
||||
value val = try_builtin_func(ctx, prop, object, true);
|
||||
if (!is_val<value_undefined>(val)) {
|
||||
value val = find_builtin_func(ctx, prop, object);
|
||||
if (val) {
|
||||
return val;
|
||||
}
|
||||
// else, fallthrough to normal property access below
|
||||
}
|
||||
|
||||
JJ_DEBUG("Member expression on object type %s, property type %s", object->type().c_str(), property->type().c_str());
|
||||
value val = mk_val<value_undefined>("object_property");
|
||||
value val;
|
||||
|
||||
if (property->is_undefined()) {
|
||||
JJ_DEBUG("%s", "Member expression property is undefined, returning undefined");
|
||||
return val;
|
||||
return mk_val<value_undefined>("object_property");
|
||||
}
|
||||
|
||||
ensure_key_type_allowed(property);
|
||||
|
||||
if (is_val<value_undefined>(object)) {
|
||||
JJ_DEBUG("%s", "Accessing property on undefined object, returning undefined");
|
||||
return val;
|
||||
return mk_val<value_undefined>("object_property");
|
||||
|
||||
} else if (is_val<value_object>(object)) {
|
||||
auto key = property->as_string().str();
|
||||
val = object->at(property, val);
|
||||
if (is_val<value_undefined>(val)) {
|
||||
val = try_builtin_func(ctx, key, object, true);
|
||||
val = object->find(property);
|
||||
if (!val || is_val<value_undefined>(val)) {
|
||||
val = try_builtin_func(ctx, property->as_string().str(), object, true);
|
||||
}
|
||||
JJ_DEBUG("Accessed property '%s' value, got type: %s", key.c_str(), val->type().c_str());
|
||||
JJ_DEBUG("Accessed property '%s' value, got type: %s", property->as_string().str().c_str(), val->type().c_str());
|
||||
|
||||
} else if (is_val<value_array>(object) || is_val<value_string>(object)) {
|
||||
if (is_val<value_int>(property) || is_val<value_bool>(property)) {
|
||||
int64_t index = property->as_int();
|
||||
JJ_DEBUG("Accessing %s index %d", object->type().c_str(), (int)index);
|
||||
if (is_val<value_array>(object)) {
|
||||
auto & arr = object->as_array();
|
||||
const auto * arr = cast_val<value_array>(object);
|
||||
const auto count = arr->as_array().size();
|
||||
if (index < 0) {
|
||||
index += static_cast<int64_t>(arr.size());
|
||||
}
|
||||
if (index >= 0 && index < static_cast<int64_t>(arr.size())) {
|
||||
val = arr[index];
|
||||
// Avoid negating INT64_MIN.
|
||||
const uint64_t distance = static_cast<uint64_t>(-(index + 1)) + 1;
|
||||
if (distance <= count) {
|
||||
val = arr->as_array().at(static_cast<size_t>(count - distance));
|
||||
}
|
||||
} else if (static_cast<uint64_t>(index) < count) {
|
||||
val = arr->as_array().at(static_cast<size_t>(index));
|
||||
}
|
||||
} else { // value_string
|
||||
auto str = object->as_string().str();
|
||||
if (index >= 0 && index < static_cast<int64_t>(str.size())) {
|
||||
val = mk_val<value_string>(std::string(1, str[index]));
|
||||
if (index >= 0) {
|
||||
const auto byte = cast_val<value_string>(object)->val_str.byte_at(static_cast<size_t>(index));
|
||||
if (byte) {
|
||||
val = mk_val<value_string>(std::string(1, *byte));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -930,7 +973,10 @@ value member_expression::execute_impl(context & ctx) const {
|
||||
val = try_builtin_func(ctx, key, object, true);
|
||||
}
|
||||
|
||||
if (ctx.is_get_stats && val && object && property) {
|
||||
if (!val) {
|
||||
val = mk_val<value_undefined>("object_property");
|
||||
}
|
||||
if (ctx.is_get_stats && object && property) {
|
||||
value_t::stats_t::mark_used(val);
|
||||
value_t::stats_t::mark_used(object);
|
||||
value_t::stats_t::mark_used(property);
|
||||
|
||||
+59
-20
@@ -81,12 +81,43 @@ struct context {
|
||||
current_time(parent.current_time),
|
||||
is_get_stats(parent.is_get_stats),
|
||||
macro_depth(parent.macro_depth),
|
||||
// inherit variables (for example, when entering a new scope)
|
||||
env(mk_val<value_object>(*parent.env)) {}
|
||||
// child scope: own variables only, lookups fall back to the parent (which must outlive this scope)
|
||||
parent(&parent),
|
||||
env(mk_val<value_object>()) {
|
||||
env->has_builtins = false;
|
||||
}
|
||||
|
||||
context(context &&) = default;
|
||||
|
||||
// copy with all visible variables flattened in, for a scope that can outlive its parents (caller() of a call block)
|
||||
context snapshot() const {
|
||||
std::vector<const context *> chain;
|
||||
for (const context * c = this; c; c = c->parent) {
|
||||
chain.push_back(c);
|
||||
}
|
||||
context res(*this);
|
||||
res.parent = nullptr;
|
||||
for (auto it = chain.rbegin(); it != chain.rend(); ++it) {
|
||||
for (const auto & kv : (*it)->env->as_ordered_object()) {
|
||||
res.env->insert(kv.first, kv.second);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
// returns nullptr if not found
|
||||
value find_val(const value & key) {
|
||||
for (const context * c = this; c; c = c->parent) {
|
||||
if (value v = c->env->find(key)) {
|
||||
return v;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
value get_val(const std::string & name) {
|
||||
value default_val = mk_val<value_undefined>(name);
|
||||
return env->at(name, default_val);
|
||||
auto result = find_val(mk_val<value_string>(name));
|
||||
return result ? result : mk_val<value_undefined>(name);
|
||||
}
|
||||
|
||||
void set_val(const std::string & name, const value & val) {
|
||||
@@ -102,9 +133,16 @@ struct context {
|
||||
}
|
||||
|
||||
private:
|
||||
const context * parent = nullptr;
|
||||
value_object env;
|
||||
};
|
||||
|
||||
// values cached in the AST are shared across renders, but stats mutate values, so stats mode gets a fresh copy
|
||||
template<typename T>
|
||||
value cached_val(const context & ctx, const value & cached) {
|
||||
return ctx.is_get_stats ? mk_val<T>(*cast_val<T>(cached)) : cached;
|
||||
}
|
||||
|
||||
// utils for visiting AST nodes
|
||||
static std::vector<const statement *> stmts_to_ptr(const statements & stmts) {
|
||||
std::vector<const statement *> children;
|
||||
@@ -328,11 +366,8 @@ struct member_expression : public expression {
|
||||
statement_ptr property;
|
||||
bool computed; // true if obj[expr] and false if obj.prop
|
||||
|
||||
member_expression(statement_ptr && object, statement_ptr && property, bool computed)
|
||||
: object(std::move(object)), property(std::move(property)), computed(computed) {
|
||||
chk_type<expression>(this->object);
|
||||
chk_type<expression>(this->property);
|
||||
}
|
||||
value static_property; // set if property is an identifier, see cached_val()
|
||||
member_expression(statement_ptr && object, statement_ptr && property, bool computed);
|
||||
std::string type() const override { return "MemberExpression"; }
|
||||
value execute_impl(context & ctx) const override;
|
||||
void visit(context & ctx) const override {
|
||||
@@ -367,7 +402,8 @@ struct call_expression : public expression {
|
||||
*/
|
||||
struct identifier : public expression {
|
||||
std::string val;
|
||||
explicit identifier(const std::string & val) : val(val) {}
|
||||
const value key; // lookup key, safe to share across renders as lookups do not touch stats
|
||||
explicit identifier(const std::string & val) : val(val), key(mk_val<value_string>(val)) {}
|
||||
std::string type() const override { return "Identifier"; }
|
||||
value execute_impl(context & ctx) const override;
|
||||
};
|
||||
@@ -376,28 +412,31 @@ struct identifier : public expression {
|
||||
|
||||
struct integer_literal : public expression {
|
||||
int64_t val;
|
||||
explicit integer_literal(int64_t val) : val(val) {}
|
||||
const value constant; // see cached_val()
|
||||
explicit integer_literal(int64_t val) : val(val), constant(mk_val<value_int>(val)) {}
|
||||
std::string type() const override { return "IntegerLiteral"; }
|
||||
value execute_impl(context &) const override {
|
||||
return mk_val<value_int>(val);
|
||||
value execute_impl(context & ctx) const override {
|
||||
return cached_val<value_int>(ctx, constant);
|
||||
}
|
||||
};
|
||||
|
||||
struct float_literal : public expression {
|
||||
double val;
|
||||
explicit float_literal(double val) : val(val) {}
|
||||
const value constant; // see cached_val()
|
||||
explicit float_literal(double val) : val(val), constant(mk_val<value_float>(val)) {}
|
||||
std::string type() const override { return "FloatLiteral"; }
|
||||
value execute_impl(context &) const override {
|
||||
return mk_val<value_float>(val);
|
||||
value execute_impl(context & ctx) const override {
|
||||
return cached_val<value_float>(ctx, constant);
|
||||
}
|
||||
};
|
||||
|
||||
struct string_literal : public expression {
|
||||
std::string val;
|
||||
explicit string_literal(const std::string & val) : val(val) {}
|
||||
const value constant; // see cached_val()
|
||||
explicit string_literal(const std::string & val) : val(val), constant(mk_val<value_string>(val)) {}
|
||||
std::string type() const override { return "StringLiteral"; }
|
||||
value execute_impl(context &) const override {
|
||||
return mk_val<value_string>(val);
|
||||
value execute_impl(context & ctx) const override {
|
||||
return cached_val<value_string>(ctx, constant);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -748,7 +787,7 @@ static void gather_string_parts_recursive(const value & val, value_string & part
|
||||
} else if (is_val<value_array>(val) && depth >= JINJA_RECURSIVE_LIMIT) {
|
||||
throw std::runtime_error("value is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
|
||||
} else if (is_val<value_array>(val)) {
|
||||
auto items = cast_val<value_array>(val)->as_array();
|
||||
const auto & items = cast_val<value_array>(val)->as_array();
|
||||
for (const auto & item : items) {
|
||||
gather_string_parts_recursive(item, parts, depth + 1);
|
||||
}
|
||||
|
||||
+51
-6
@@ -43,14 +43,59 @@ void string::mark_input() {
|
||||
}
|
||||
|
||||
std::string string::str() const {
|
||||
if (parts.size() == 1) {
|
||||
return parts[0].val;
|
||||
}
|
||||
std::ostringstream oss;
|
||||
std::string result;
|
||||
result.reserve(length());
|
||||
for (const auto & part : parts) {
|
||||
oss << part.val;
|
||||
result += part.val;
|
||||
}
|
||||
return oss.str();
|
||||
return result;
|
||||
}
|
||||
|
||||
bool string::empty() const {
|
||||
for (const auto & part : parts) {
|
||||
if (!part.val.empty()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::optional<char> string::byte_at(size_t index) const {
|
||||
for (const auto & part : parts) {
|
||||
if (index < part.val.size()) {
|
||||
return part.val[index];
|
||||
}
|
||||
index -= part.val.size();
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
bool string::equals(const string & other) const {
|
||||
size_t part = 0, offset = 0;
|
||||
for (const auto & left : parts) {
|
||||
std::string_view text(left.val);
|
||||
while (!text.empty()) {
|
||||
while (part < other.parts.size() && offset == other.parts[part].val.size()) {
|
||||
++part;
|
||||
offset = 0;
|
||||
}
|
||||
if (part == other.parts.size()) {
|
||||
return false;
|
||||
}
|
||||
const auto & right = other.parts[part].val;
|
||||
const auto count = std::min(text.size(), right.size() - offset);
|
||||
if (text.substr(0, count) != std::string_view(right).substr(offset, count)) {
|
||||
return false;
|
||||
}
|
||||
text.remove_prefix(count);
|
||||
offset += count;
|
||||
}
|
||||
}
|
||||
while (part < other.parts.size() && offset == other.parts[part].val.size()) {
|
||||
++part;
|
||||
offset = 0;
|
||||
}
|
||||
return part == other.parts.size();
|
||||
}
|
||||
|
||||
size_t string::length() const {
|
||||
|
||||
@@ -39,6 +39,9 @@ struct string {
|
||||
|
||||
std::string str() const;
|
||||
size_t length() const;
|
||||
bool empty() const;
|
||||
bool equals(const string & other) const;
|
||||
std::optional<char> byte_at(size_t index) const;
|
||||
void hash_update(hasher & hash) const noexcept;
|
||||
bool all_parts_are_input() const;
|
||||
bool is_uppercase() const;
|
||||
|
||||
+77
-33
@@ -276,19 +276,17 @@ static value tojson(const func_args & args) {
|
||||
if (args.ctx.is_get_stats) {
|
||||
// mark as used (recursively) for stats
|
||||
auto val_input = args.get_pos(0);
|
||||
value_t::stats_t::mark_used(const_cast<value&>(val_input), true);
|
||||
value_t::stats_t::mark_used(val_input, true);
|
||||
}
|
||||
if (is_val<value_int>(val_indent)) {
|
||||
indent = static_cast<int>(val_indent->as_int());
|
||||
}
|
||||
if (val_sort->as_bool()) { // undefined == false
|
||||
throw not_implemented_exception("tojson sort_keys=true not implemented");
|
||||
}
|
||||
const bool sort_keys = val_sort->as_bool(); // undefined == false
|
||||
const bool ensure_ascii = val_ascii->as_bool(); // undefined == false
|
||||
auto separators = (is_val<value_array>(val_separators) ? val_separators : mk_val<value_array>())->as_array();
|
||||
std::string item_sep = separators.size() > 0 ? separators[0]->as_string().str() : (indent < 0 ? ", " : ",");
|
||||
std::string key_sep = separators.size() > 1 ? separators[1]->as_string().str() : ": ";
|
||||
std::string json_str = value_to_json(args.get_pos(0), indent, item_sep, key_sep);
|
||||
std::string json_str = value_to_json(args.get_pos(0), indent, item_sep, key_sep, sort_keys);
|
||||
if (ensure_ascii) {
|
||||
json_str = json_ensure_ascii_preserving_format(json_str);
|
||||
}
|
||||
@@ -507,7 +505,12 @@ const func_builtins & global_builtins() {
|
||||
}},
|
||||
{"test_is_divisibleby", [](const func_args & args) -> value {
|
||||
args.ensure_vals<value_int, value_int>();
|
||||
bool res = args.get_pos(0)->val_int % args.get_pos(1)->val_int == 0;
|
||||
const int64_t a = args.get_pos(0)->as_int();
|
||||
const int64_t b = args.get_pos(1)->as_int();
|
||||
if (b == 0) {
|
||||
throw std::runtime_error("divisibleby: division by zero");
|
||||
}
|
||||
bool res = b == -1 || a % b == 0; // avoid INT64_MIN % -1 overflow
|
||||
return mk_val<value_bool>(res);
|
||||
}},
|
||||
{"test_is_string", test_type_fn<value_string>},
|
||||
@@ -519,11 +522,11 @@ const func_builtins & global_builtins() {
|
||||
{"test_is_mapping", test_type_fn<value_object>},
|
||||
{"test_is_lower", [](const func_args & args) -> value {
|
||||
args.ensure_vals<value_string>();
|
||||
return mk_val<value_bool>(args.get_pos(0)->val_str.is_lowercase());
|
||||
return mk_val<value_bool>(cast_val<value_string>(args.get_pos(0))->val_str.is_lowercase());
|
||||
}},
|
||||
{"test_is_upper", [](const func_args & args) -> value {
|
||||
args.ensure_vals<value_string>();
|
||||
return mk_val<value_bool>(args.get_pos(0)->val_str.is_uppercase());
|
||||
return mk_val<value_bool>(cast_val<value_string>(args.get_pos(0))->val_str.is_uppercase());
|
||||
}},
|
||||
{"test_is_none", test_type_fn<value_none>},
|
||||
{"test_is_defined", [](const func_args & args) -> value {
|
||||
@@ -571,7 +574,7 @@ const func_builtins & global_builtins() {
|
||||
{"test_is_test", [](const func_args & args) -> value {
|
||||
args.ensure_vals<value_string>();
|
||||
auto & builtins = global_builtins();
|
||||
std::string test_name = args.get_pos(0)->val_str.str();
|
||||
std::string test_name = cast_val<value_string>(args.get_pos(0))->val_str.str();
|
||||
auto it = builtins.find("test_is_" + test_name);
|
||||
bool res = it != builtins.end();
|
||||
return mk_val<value_bool>(res);
|
||||
@@ -613,18 +616,26 @@ const func_builtins & global_builtins() {
|
||||
return builtins;
|
||||
}
|
||||
|
||||
value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing) {
|
||||
value find_builtin_func(context & ctx, const std::string & name, value & input) {
|
||||
JJ_DEBUG("Trying built-in function '%s' for type %s", name.c_str(), input->type().c_str());
|
||||
if (ctx.is_get_stats) {
|
||||
value_t::stats_t::mark_used(input);
|
||||
input->stats.ops.insert(name);
|
||||
}
|
||||
auto builtins = input->get_builtins();
|
||||
const auto & builtins = input->get_builtins();
|
||||
auto it = builtins.find(name);
|
||||
if (it != builtins.end()) {
|
||||
JJ_DEBUG("Binding built-in '%s'", name.c_str());
|
||||
return mk_val<value_func>(name, it->second, input);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing) {
|
||||
value func = find_builtin_func(ctx, name, input);
|
||||
if (func) {
|
||||
return func;
|
||||
}
|
||||
if (undef_on_missing) {
|
||||
return mk_val<value_undefined>(name);
|
||||
}
|
||||
@@ -700,7 +711,7 @@ const func_builtins & value_float_t::get_builtins() const {
|
||||
}},
|
||||
{"int", [](const func_args & args) -> value {
|
||||
args.ensure_vals<value_float>();
|
||||
int64_t val = static_cast<int64_t>(args.get_pos(0)->as_float());
|
||||
int64_t val = float_to_int(args.get_pos(0)->as_float());
|
||||
return mk_val<value_int>(val);
|
||||
}},
|
||||
{"float", [](const func_args & args) -> value {
|
||||
@@ -831,7 +842,7 @@ const func_builtins & value_string_t::get_builtins() const {
|
||||
--maxsplit;
|
||||
}
|
||||
auto res = mk_val<value_string>(str);
|
||||
res->val_str.mark_input_based_on(args.get_pos(0)->val_str);
|
||||
res->val_str.mark_input_based_on(cast_val<value_string>(args.get_pos(0))->val_str);
|
||||
result->push_back(std::move(res));
|
||||
return result;
|
||||
}},
|
||||
@@ -858,7 +869,7 @@ const func_builtins & value_string_t::get_builtins() const {
|
||||
--maxsplit;
|
||||
}
|
||||
auto res = mk_val<value_string>(str);
|
||||
res->val_str.mark_input_based_on(args.get_pos(0)->val_str);
|
||||
res->val_str.mark_input_based_on(cast_val<value_string>(args.get_pos(0))->val_str);
|
||||
result->push_back(std::move(res));
|
||||
result->reverse();
|
||||
return result;
|
||||
@@ -891,7 +902,7 @@ const func_builtins & value_string_t::get_builtins() const {
|
||||
}
|
||||
}
|
||||
auto res = mk_val<value_string>(str);
|
||||
res->val_str.mark_input_based_on(args.get_pos(0)->val_str);
|
||||
res->val_str.mark_input_based_on(cast_val<value_string>(args.get_pos(0))->val_str);
|
||||
return res;
|
||||
}},
|
||||
{"format", [](const func_args & args) -> value {
|
||||
@@ -1088,8 +1099,28 @@ const func_builtins & value_bool_t::get_builtins() const {
|
||||
return builtins;
|
||||
}
|
||||
|
||||
[[noreturn]] static value array_sum_not_implemented(const func_args &) {
|
||||
throw not_implemented_exception("Array sum builtin not implemented");
|
||||
static value array_sum(const func_args & args) {
|
||||
args.ensure_kwargs({"attribute", "start"});
|
||||
args.ensure_count(1, 3);
|
||||
args.ensure_vals<value_array>();
|
||||
value undef = mk_val<value_undefined>();
|
||||
value attribute = args.get_kwarg_or_pos("attribute", 1);
|
||||
value start = args.get_kwarg_or_pos("start", 2);
|
||||
// same as the + operator: compute in double, int result if all operands are int
|
||||
bool is_float = is_val<value_float>(start);
|
||||
double sum = start->is_undefined() ? 0.0 : start->as_float();
|
||||
for (const auto & item : args.get_pos(0)->as_array()) {
|
||||
value v = attribute->is_undefined() ? item : get_attribute(item, attribute, undef);
|
||||
if (v->is_undefined() && !attribute->is_undefined()) {
|
||||
throw raised_exception("sum: " + item->type() + " has no attribute '" + attribute->as_string().str() + "'");
|
||||
}
|
||||
if (!is_val<value_int>(v) && !is_val<value_float>(v)) {
|
||||
throw raised_exception("sum: unsupported operand type " + v->type());
|
||||
}
|
||||
is_float = is_float || is_val<value_float>(v);
|
||||
sum += v->as_float();
|
||||
}
|
||||
return is_float ? value(mk_val<value_float>(sum)) : value(mk_val<value_int>(float_to_int(sum)));
|
||||
}
|
||||
|
||||
const func_builtins & value_array_t::get_builtins() const {
|
||||
@@ -1158,7 +1189,7 @@ const func_builtins & value_array_t::get_builtins() const {
|
||||
{"select", select_or_reject<false, false>},
|
||||
{"rejectattr", select_or_reject<true, true>},
|
||||
{"reject", select_or_reject<true, false>},
|
||||
{"sum", array_sum_not_implemented},
|
||||
{"sum", array_sum},
|
||||
{"join", [](const func_args & args) -> value {
|
||||
args.ensure_kwargs({"d", "attribute"});
|
||||
args.ensure_count(1, 3);
|
||||
@@ -1191,7 +1222,7 @@ const func_builtins & value_array_t::get_builtins() const {
|
||||
if (args.ctx.is_get_stats) {
|
||||
// mark as used (recursively) for stats
|
||||
auto val_input = args.get_pos(0);
|
||||
value_t::stats_t::mark_used(const_cast<value&>(val_input), true);
|
||||
value_t::stats_t::mark_used(val_input, true);
|
||||
}
|
||||
return mk_val<value_string>(args.get_pos(0)->as_string());
|
||||
}},
|
||||
@@ -1439,7 +1470,7 @@ const func_builtins & value_object_t::get_builtins() const {
|
||||
if (args.ctx.is_get_stats) {
|
||||
// mark as used (recursively) for stats
|
||||
auto val_input = args.get_pos(0);
|
||||
value_t::stats_t::mark_used(const_cast<value&>(val_input), true);
|
||||
value_t::stats_t::mark_used(val_input, true);
|
||||
}
|
||||
return mk_val<value_string>(args.get_pos(0)->as_string());
|
||||
}},
|
||||
@@ -1643,6 +1674,11 @@ bool value_compare(const value & a, const value & b, value_compare_op op) {
|
||||
return result;
|
||||
}
|
||||
|
||||
template<>
|
||||
value value_from_json(const common_json & json_val, bool mark_input) {
|
||||
return from_json(json_val, mark_input);
|
||||
}
|
||||
|
||||
template<>
|
||||
void global_from_json(context & ctx, const common_json & json_obj, bool mark_input) {
|
||||
// printf("global_from_json: %s\n" , json_obj.dump(2).c_str());
|
||||
@@ -1728,7 +1764,7 @@ static std::string float_json(double val) {
|
||||
|
||||
// recursively convert value to JSON string
|
||||
// TODO: avoid circular references
|
||||
static void value_to_json_internal(std::ostringstream & oss, const value & val, int curr_lvl, int indent, const std::string_view item_sep, const std::string_view key_sep) {
|
||||
static void value_to_json_internal(std::ostringstream & oss, const value & val, int curr_lvl, int indent, const std::string_view item_sep, const std::string_view key_sep, bool sort_keys) {
|
||||
auto indent_str = [indent, curr_lvl]() -> std::string {
|
||||
return (indent > 0) ? std::string(curr_lvl * indent, ' ') : "";
|
||||
};
|
||||
@@ -1775,7 +1811,7 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
|
||||
oss << newline();
|
||||
for (size_t i = 0; i < arr.size(); ++i) {
|
||||
oss << indent_str() << (indent > 0 ? std::string(indent, ' ') : "");
|
||||
value_to_json_internal(oss, arr[i], curr_lvl + 1, indent, item_sep, key_sep);
|
||||
value_to_json_internal(oss, arr[i], curr_lvl + 1, indent, item_sep, key_sep, sort_keys);
|
||||
if (i < arr.size() - 1) {
|
||||
oss << item_sep;
|
||||
}
|
||||
@@ -1786,17 +1822,25 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
|
||||
oss << "]";
|
||||
} else if (is_val<value_object>(val)) {
|
||||
const auto & obj = val->as_ordered_object(); // IMPORTANT: need to keep exact order
|
||||
std::vector<std::pair<std::string, value>> items;
|
||||
items.reserve(obj.size());
|
||||
for (const auto & pair : obj) {
|
||||
items.emplace_back(is_val<value_float>(pair.first) ? float_json(pair.first->as_float()) : pair.first->as_string().str(), pair.second);
|
||||
}
|
||||
if (sort_keys) {
|
||||
// same order as python json.dumps(sort_keys=True): utf-8 byte order is code point order
|
||||
std::stable_sort(items.begin(), items.end(), [](const auto & a, const auto & b) { return a.first < b.first; });
|
||||
}
|
||||
oss << "{";
|
||||
if (!obj.empty()) {
|
||||
if (!items.empty()) {
|
||||
oss << newline();
|
||||
size_t i = 0;
|
||||
for (const auto & pair : obj) {
|
||||
for (const auto & [key, item] : items) {
|
||||
oss << indent_str() << (indent > 0 ? std::string(indent, ' ') : "");
|
||||
const std::string key = is_val<value_float>(pair.first) ? float_json(pair.first->as_float()) : pair.first->as_string().str();
|
||||
value_to_json_internal(oss, mk_val<value_string>(key), curr_lvl + 1, indent, item_sep, key_sep);
|
||||
value_to_json_internal(oss, mk_val<value_string>(key), curr_lvl + 1, indent, item_sep, key_sep, sort_keys);
|
||||
oss << key_sep;
|
||||
value_to_json_internal(oss, pair.second, curr_lvl + 1, indent, item_sep, key_sep);
|
||||
if (i < obj.size() - 1) {
|
||||
value_to_json_internal(oss, item, curr_lvl + 1, indent, item_sep, key_sep, sort_keys);
|
||||
if (i < items.size() - 1) {
|
||||
oss << item_sep;
|
||||
}
|
||||
oss << newline();
|
||||
@@ -1810,9 +1854,9 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
|
||||
}
|
||||
}
|
||||
|
||||
std::string value_to_json(const value & val, int indent, const std::string_view item_sep, const std::string_view key_sep) {
|
||||
std::string value_to_json(const value & val, int indent, const std::string_view item_sep, const std::string_view key_sep, bool sort_keys) {
|
||||
std::ostringstream oss;
|
||||
value_to_json_internal(oss, val, 0, indent, item_sep, key_sep);
|
||||
value_to_json_internal(oss, val, 0, indent, item_sep, key_sep, sort_keys);
|
||||
JJ_DEBUG("value_to_json: result=%s", oss.str().c_str());
|
||||
return oss.str();
|
||||
}
|
||||
@@ -1833,18 +1877,18 @@ std::string value_to_string_repr(const value & val) {
|
||||
}
|
||||
|
||||
// stats utility
|
||||
void value_t::stats_t::mark_used(value & val, bool deep, int depth) {
|
||||
void value_t::stats_t::mark_used(const value & val, bool deep, int depth) {
|
||||
val->stats.used = true;
|
||||
if (deep && (is_val<value_array>(val) || is_val<value_object>(val)) && depth >= JINJA_RECURSIVE_LIMIT) {
|
||||
throw std::runtime_error("value is nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
|
||||
}
|
||||
if (deep) {
|
||||
if (is_val<value_array>(val)) {
|
||||
for (auto & item : val->val_arr) {
|
||||
for (auto & item : val->as_array()) {
|
||||
mark_used(item, deep, depth + 1);
|
||||
}
|
||||
} else if (is_val<value_object>(val)) {
|
||||
for (auto & pair : val->val_obj) {
|
||||
for (auto & pair : val->as_ordered_object()) {
|
||||
mark_used(pair.first, deep, depth + 1);
|
||||
mark_used(pair.second, deep, depth + 1);
|
||||
}
|
||||
|
||||
+142
-90
@@ -21,6 +21,20 @@ namespace jinja {
|
||||
struct value_t;
|
||||
using value = std::shared_ptr<value_t>;
|
||||
|
||||
enum class value_kind {
|
||||
INTEGER,
|
||||
REAL,
|
||||
STRING,
|
||||
BOOLEAN,
|
||||
ARRAY,
|
||||
TUPLE,
|
||||
OBJECT,
|
||||
NONE,
|
||||
UNDEFINED,
|
||||
FUNCTION,
|
||||
KEYWORD
|
||||
};
|
||||
bool has_value_kind(const value_t * val, value_kind kind);
|
||||
|
||||
// Helper to check the type of a value
|
||||
template<typename T>
|
||||
@@ -34,12 +48,12 @@ struct extract_pointee<std::shared_ptr<U>> {
|
||||
template<typename T>
|
||||
bool is_val(const value & ptr) {
|
||||
using PointeeType = typename extract_pointee<T>::type;
|
||||
return dynamic_cast<const PointeeType*>(ptr.get()) != nullptr;
|
||||
return has_value_kind(ptr.get(), PointeeType::kind_id);
|
||||
}
|
||||
template<typename T>
|
||||
bool is_val(const value_t * ptr) {
|
||||
using PointeeType = typename extract_pointee<T>::type;
|
||||
return dynamic_cast<const PointeeType*>(ptr) != nullptr;
|
||||
return has_value_kind(ptr, PointeeType::kind_id);
|
||||
}
|
||||
template<typename T, typename... Args>
|
||||
std::shared_ptr<typename extract_pointee<T>::type> mk_val(Args&&... args) {
|
||||
@@ -49,12 +63,12 @@ std::shared_ptr<typename extract_pointee<T>::type> mk_val(Args&&... args) {
|
||||
template<typename T>
|
||||
const typename extract_pointee<T>::type * cast_val(const value & ptr) {
|
||||
using PointeeType = typename extract_pointee<T>::type;
|
||||
return dynamic_cast<const PointeeType*>(ptr.get());
|
||||
return is_val<T>(ptr) ? static_cast<const PointeeType *>(ptr.get()) : nullptr;
|
||||
}
|
||||
template<typename T>
|
||||
typename extract_pointee<T>::type * cast_val(value & ptr) {
|
||||
using PointeeType = typename extract_pointee<T>::type;
|
||||
return dynamic_cast<PointeeType*>(ptr.get());
|
||||
return is_val<T>(ptr) ? static_cast<PointeeType *>(ptr.get()) : nullptr;
|
||||
}
|
||||
// End Helper
|
||||
|
||||
@@ -91,6 +105,10 @@ struct context; // forward declaration
|
||||
template<typename T_JSON>
|
||||
void global_from_json(context & ctx, const T_JSON & json_obj, bool mark_input);
|
||||
|
||||
// convert a single JSON value, same rules as global_from_json()
|
||||
template<typename T_JSON>
|
||||
value value_from_json(const T_JSON & json_val, bool mark_input);
|
||||
|
||||
//
|
||||
// base value type
|
||||
//
|
||||
@@ -105,14 +123,7 @@ enum value_compare_op { eq, ge, gt, lt, ne };
|
||||
bool value_compare(const value & a, const value & b, value_compare_op op);
|
||||
|
||||
struct value_t {
|
||||
int64_t val_int;
|
||||
double val_flt;
|
||||
string val_str;
|
||||
|
||||
std::vector<value> val_arr;
|
||||
std::vector<std::pair<value, value>> val_obj;
|
||||
|
||||
func_handler val_func;
|
||||
const value_kind kind;
|
||||
|
||||
// only used if ctx.is_get_stats = true
|
||||
struct stats_t {
|
||||
@@ -120,10 +131,10 @@ struct value_t {
|
||||
// ops can be builtin calls or operators: "array_access", "object_access"
|
||||
std::set<std::string> ops;
|
||||
// utility to recursively mark value and its children as used
|
||||
static void mark_used(value & val, bool deep = false, int depth = 0);
|
||||
static void mark_used(const value & val, bool deep = false, int depth = 0);
|
||||
} stats;
|
||||
|
||||
value_t() = default;
|
||||
explicit value_t(value_kind kind) : kind(kind) {}
|
||||
value_t(const value_t &) = default;
|
||||
virtual ~value_t() = default;
|
||||
|
||||
@@ -141,6 +152,10 @@ struct value_t {
|
||||
virtual bool is_undefined() const { return false; }
|
||||
virtual const func_builtins & get_builtins() const { throw_type_error("has no builtins"); }
|
||||
|
||||
virtual value find(const value & key) {
|
||||
value missing;
|
||||
return at(key, missing);
|
||||
}
|
||||
virtual bool has_key(const value &) { throw_type_error("is not an object value"); }
|
||||
virtual void insert(const value & /* key */, const value & /* val */) { throw_type_error("is not an object value"); }
|
||||
virtual value & at(const value & /* key */, value & /* default_val */) { throw_type_error("is not an object value"); }
|
||||
@@ -172,15 +187,23 @@ protected:
|
||||
virtual bool nonequal(const value_t & other) const { return !equivalent(other); }
|
||||
};
|
||||
|
||||
inline bool has_value_kind(const value_t * val, value_kind kind) {
|
||||
return val && (val->kind == kind ||
|
||||
(kind == value_kind::ARRAY && (val->kind == value_kind::TUPLE)));
|
||||
}
|
||||
|
||||
//
|
||||
// utils
|
||||
//
|
||||
|
||||
const func_builtins & global_builtins();
|
||||
// returns nullptr if input has no built-in named `name`
|
||||
value find_builtin_func(context & ctx, const std::string & name, value & input);
|
||||
// returns value_undefined(name) if not found and undef_on_missing is set, throws otherwise
|
||||
value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false);
|
||||
value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args);
|
||||
|
||||
std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": ");
|
||||
std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": ", bool sort_keys = false);
|
||||
|
||||
// Note: only used for debugging purposes
|
||||
std::string value_to_string_repr(const value & val);
|
||||
@@ -214,11 +237,28 @@ struct value_equality {
|
||||
// primitive value types
|
||||
//
|
||||
|
||||
// double to int64, saturating and with nan as 0; a plain cast is undefined outside the int64 range
|
||||
inline int64_t float_to_int(double v) {
|
||||
if (std::isnan(v)) {
|
||||
return 0;
|
||||
}
|
||||
if (v >= 9223372036854775808.0) {
|
||||
return INT64_MAX;
|
||||
}
|
||||
if (v <= -9223372036854775808.0) {
|
||||
return INT64_MIN;
|
||||
}
|
||||
return static_cast<int64_t>(v);
|
||||
}
|
||||
|
||||
struct value_int_t : public value_t {
|
||||
value_int_t(int64_t v) {
|
||||
static constexpr auto kind_id = value_kind::INTEGER;
|
||||
int64_t val_int;
|
||||
double val_flt;
|
||||
value_int_t(int64_t v) : value_t(kind_id) {
|
||||
val_int = v;
|
||||
val_flt = static_cast<double>(v);
|
||||
if (static_cast<int64_t>(val_flt) != v) {
|
||||
if (float_to_int(val_flt) != v) {
|
||||
val_flt = v < 0 ? -INFINITY : INFINITY;
|
||||
}
|
||||
}
|
||||
@@ -239,21 +279,22 @@ struct value_int_t : public value_t {
|
||||
}
|
||||
protected:
|
||||
virtual bool equivalent(const value_t & other) const override {
|
||||
return other.is_numeric() && val_int == other.val_int && val_flt == other.val_flt;
|
||||
return other.is_numeric() && val_int == other.as_int() && val_flt == other.as_float();
|
||||
}
|
||||
virtual bool nonequal(const value_t & other) const override {
|
||||
return !(typeid(*this) == typeid(other) && val_int == other.val_int);
|
||||
return !(kind == other.kind && val_int == other.as_int());
|
||||
}
|
||||
};
|
||||
using value_int = std::shared_ptr<value_int_t>;
|
||||
|
||||
|
||||
struct value_float_t : public value_t {
|
||||
value val;
|
||||
value_float_t(double v) {
|
||||
static constexpr auto kind_id = value_kind::REAL;
|
||||
int64_t val_int;
|
||||
double val_flt;
|
||||
value_float_t(double v) : value_t(kind_id) {
|
||||
val_flt = v;
|
||||
val_int = std::isfinite(v) ? static_cast<int64_t>(v) : 0;
|
||||
val = mk_val<value_int>(val_int);
|
||||
val_int = std::isfinite(v) ? float_to_int(v) : 0;
|
||||
}
|
||||
virtual std::string type() const override { return "Float"; }
|
||||
virtual double as_float() const override { return val_flt; }
|
||||
@@ -272,7 +313,7 @@ struct value_float_t : public value_t {
|
||||
virtual bool is_hashable() const override { return true; }
|
||||
virtual hasher unique_hash() const noexcept override {
|
||||
if (static_cast<double>(val_int) == val_flt) {
|
||||
return val->unique_hash();
|
||||
return value_int_t(val_int).unique_hash();
|
||||
} else {
|
||||
return hasher(typeid(*this))
|
||||
.update(&val_int, sizeof(val_int))
|
||||
@@ -281,19 +322,22 @@ struct value_float_t : public value_t {
|
||||
}
|
||||
protected:
|
||||
virtual bool equivalent(const value_t & other) const override {
|
||||
return other.is_numeric() && val_int == other.val_int && val_flt == other.val_flt;
|
||||
return other.is_numeric() && val_int == other.as_int() && val_flt == other.as_float();
|
||||
}
|
||||
virtual bool nonequal(const value_t & other) const override {
|
||||
return !(typeid(*this) == typeid(other) && val_flt == other.val_flt);
|
||||
return !(kind == other.kind && val_flt == other.as_float());
|
||||
}
|
||||
};
|
||||
using value_float = std::shared_ptr<value_float_t>;
|
||||
|
||||
|
||||
struct value_string_t : public value_t {
|
||||
value_string_t() { val_str = string(); }
|
||||
value_string_t(const std::string & v) { val_str = string(v); }
|
||||
value_string_t(const string & v) { val_str = v; }
|
||||
static constexpr auto kind_id = value_kind::STRING;
|
||||
string val_str;
|
||||
value_string_t() : value_t(kind_id) {}
|
||||
value_string_t(const std::string & v) : value_t(kind_id), val_str(v) {}
|
||||
value_string_t(const string & v) : value_t(kind_id), val_str(v) {}
|
||||
value_string_t(string && v) : value_t(kind_id), val_str(std::move(v)) {}
|
||||
virtual std::string type() const override { return "String"; }
|
||||
virtual string as_string() const override { return val_str; }
|
||||
virtual std::string as_repr() const override {
|
||||
@@ -304,7 +348,7 @@ struct value_string_t : public value_t {
|
||||
return ss.str();
|
||||
}
|
||||
virtual bool as_bool() const override {
|
||||
return val_str.length() > 0;
|
||||
return !val_str.empty();
|
||||
}
|
||||
virtual const func_builtins & get_builtins() const override;
|
||||
virtual bool is_hashable() const override { return true; }
|
||||
@@ -320,51 +364,50 @@ struct value_string_t : public value_t {
|
||||
}
|
||||
protected:
|
||||
virtual bool equivalent(const value_t & other) const override {
|
||||
return typeid(*this) == typeid(other) && val_str.str() == other.val_str.str();
|
||||
return kind == other.kind && val_str.equals(static_cast<const value_string_t &>(other).val_str);
|
||||
}
|
||||
};
|
||||
using value_string = std::shared_ptr<value_string_t>;
|
||||
|
||||
|
||||
struct value_bool_t : public value_t {
|
||||
value val;
|
||||
value_bool_t(bool v) {
|
||||
static constexpr auto kind_id = value_kind::BOOLEAN;
|
||||
int64_t val_int;
|
||||
double val_flt;
|
||||
value_bool_t(bool v) : value_t(kind_id) {
|
||||
val_int = static_cast<int64_t>(v);
|
||||
val_flt = static_cast<double>(v);
|
||||
val = mk_val<value_int>(val_int);
|
||||
}
|
||||
virtual std::string type() const override { return "Boolean"; }
|
||||
virtual int64_t as_int() const override { return val_int; }
|
||||
virtual double as_float() const override { return val_flt; }
|
||||
virtual bool as_bool() const override { return val_int; }
|
||||
virtual string as_string() const override { return std::string(val_int ? "True" : "False"); }
|
||||
virtual const func_builtins & get_builtins() const override;
|
||||
virtual bool is_numeric() const override { return true; }
|
||||
virtual bool is_hashable() const override { return true; }
|
||||
virtual hasher unique_hash() const noexcept override {
|
||||
return val->unique_hash();
|
||||
return value_int_t(val_int).unique_hash();
|
||||
}
|
||||
protected:
|
||||
virtual bool equivalent(const value_t & other) const override {
|
||||
return other.is_numeric() && val_int == other.val_int && val_flt == other.val_flt;
|
||||
return other.is_numeric() && val_int == other.as_int() && val_flt == other.as_float();
|
||||
}
|
||||
virtual bool nonequal(const value_t & other) const override {
|
||||
return !(typeid(*this) == typeid(other) && val_int == other.val_int);
|
||||
return !(kind == other.kind && val_int == other.as_int());
|
||||
}
|
||||
};
|
||||
using value_bool = std::shared_ptr<value_bool_t>;
|
||||
|
||||
|
||||
struct value_array_t : public value_t {
|
||||
value_array_t() = default;
|
||||
value_array_t(value & v) {
|
||||
val_arr = v->val_arr;
|
||||
}
|
||||
value_array_t(std::vector<value> && arr) {
|
||||
val_arr = arr;
|
||||
}
|
||||
value_array_t(const std::vector<value> & arr) {
|
||||
val_arr = arr;
|
||||
}
|
||||
static constexpr auto kind_id = value_kind::ARRAY;
|
||||
std::vector<value> val_arr;
|
||||
|
||||
value_array_t() : value_t(kind_id) {}
|
||||
value_array_t(value & v) : value_t(kind_id), val_arr(v->as_array()) {}
|
||||
value_array_t(std::vector<value> && arr) : value_t(kind_id), val_arr(std::move(arr)) {}
|
||||
value_array_t(const std::vector<value> & arr) : value_t(kind_id), val_arr(arr) {}
|
||||
void reverse() {
|
||||
if (is_immutable()) {
|
||||
throw std::runtime_error("Attempting to modify immutable type");
|
||||
@@ -462,24 +505,27 @@ struct value_array_t : public value_t {
|
||||
return hash;
|
||||
}
|
||||
protected:
|
||||
explicit value_array_t(value_kind kind) : value_t(kind) {}
|
||||
|
||||
virtual bool equivalent(const value_t & other) const override {
|
||||
return typeid(*this) == typeid(other) && is_hashable() && other.is_hashable() && std::equal(val_arr.begin(), val_arr.end(), other.val_arr.begin(), other.val_arr.end(), value_equivalence());
|
||||
return kind == other.kind && is_hashable() && other.is_hashable() && std::equal(val_arr.begin(), val_arr.end(), other.as_array().begin(), other.as_array().end(), value_equivalence());
|
||||
}
|
||||
};
|
||||
using value_array = std::shared_ptr<value_array_t>;
|
||||
|
||||
|
||||
struct value_tuple_t : public value_array_t {
|
||||
value_tuple_t(value & v) {
|
||||
val_arr = v->val_arr;
|
||||
static constexpr auto kind_id = value_kind::TUPLE;
|
||||
value_tuple_t(value & v) : value_array_t(kind_id) {
|
||||
val_arr = v->as_array();
|
||||
}
|
||||
value_tuple_t(std::vector<value> && arr) {
|
||||
value_tuple_t(std::vector<value> && arr) : value_array_t(kind_id) {
|
||||
val_arr = std::move(arr);
|
||||
}
|
||||
value_tuple_t(const std::vector<value> & arr) : value_array_t(kind_id) {
|
||||
val_arr = arr;
|
||||
}
|
||||
value_tuple_t(const std::vector<value> & arr) {
|
||||
val_arr = arr;
|
||||
}
|
||||
value_tuple_t(const std::pair<value, value> & pair) {
|
||||
value_tuple_t(const std::pair<value, value> & pair) : value_array_t(kind_id) {
|
||||
val_arr.push_back(pair.first);
|
||||
val_arr.push_back(pair.second);
|
||||
}
|
||||
@@ -490,21 +536,23 @@ using value_tuple = std::shared_ptr<value_tuple_t>;
|
||||
|
||||
|
||||
struct value_object_t : public value_t {
|
||||
std::unordered_map<value, value, value_hasher, value_equivalence> unordered;
|
||||
static constexpr auto kind_id = value_kind::OBJECT;
|
||||
std::vector<std::pair<value, value>> val_obj;
|
||||
std::unordered_map<value, size_t, value_hasher, value_equivalence> key_index; // key -> index in val_obj
|
||||
bool has_builtins = true; // context and loop objects do not have builtins
|
||||
value_object_t() = default;
|
||||
value_object_t(value & v) {
|
||||
val_obj = v->val_obj;
|
||||
for (const auto & pair : val_obj) {
|
||||
unordered[pair.first] = pair.second;
|
||||
value_object_t() : value_t(kind_id) {}
|
||||
value_object_t(value & v) : value_t(kind_id) {
|
||||
val_obj = v->as_ordered_object();
|
||||
for (size_t i = 0; i < val_obj.size(); i++) {
|
||||
key_index[val_obj[i].first] = i;
|
||||
}
|
||||
}
|
||||
value_object_t(const std::map<value, value> & obj) {
|
||||
value_object_t(const std::map<value, value> & obj) : value_t(kind_id) {
|
||||
for (const auto & pair : obj) {
|
||||
insert(pair.first, pair.second);
|
||||
}
|
||||
}
|
||||
value_object_t(const std::vector<std::pair<value, value>> & obj) {
|
||||
value_object_t(const std::vector<std::pair<value, value>> & obj) : value_t(kind_id) {
|
||||
for (const auto & pair : obj) {
|
||||
insert(pair.first, pair.second);
|
||||
}
|
||||
@@ -527,45 +575,40 @@ struct value_object_t : public value_t {
|
||||
return ss.str();
|
||||
}
|
||||
virtual bool as_bool() const override {
|
||||
return !unordered.empty();
|
||||
return !val_obj.empty();
|
||||
}
|
||||
virtual bool has_key(const value & key) override {
|
||||
if (!key->is_immutable() || !key->is_hashable()) {
|
||||
throw std::runtime_error("Object key of unhashable type: " + key->type());
|
||||
}
|
||||
return unordered.find(key) != unordered.end();
|
||||
return key_index.find(key) != key_index.end();
|
||||
}
|
||||
virtual void insert(const value & key, const value & val) override {
|
||||
bool replaced = false;
|
||||
if (is_immutable()) {
|
||||
throw std::runtime_error("Attempting to modify immutable type");
|
||||
}
|
||||
if (has_key(key)) {
|
||||
// if key exists, replace value in ordered list instead of appending
|
||||
for (auto & pair : val_obj) {
|
||||
if (*(pair.first) == *key) {
|
||||
pair.second = val;
|
||||
replaced = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!key->is_immutable() || !key->is_hashable()) {
|
||||
throw std::runtime_error("Object key of unhashable type: " + key->type());
|
||||
}
|
||||
unordered[key] = val;
|
||||
if (!replaced) {
|
||||
const auto entry = key_index.try_emplace(key, val_obj.size());
|
||||
if (entry.second) {
|
||||
val_obj.push_back({key, val});
|
||||
} else {
|
||||
val_obj[entry.first->second].second = val;
|
||||
}
|
||||
}
|
||||
virtual value & at(const value & key, value & default_val) override {
|
||||
if (!has_key(key)) {
|
||||
return default_val;
|
||||
if (!key->is_immutable() || !key->is_hashable()) {
|
||||
throw std::runtime_error("Object key of unhashable type: " + key->type());
|
||||
}
|
||||
return unordered.at(key);
|
||||
const auto found = key_index.find(key);
|
||||
return found == key_index.end() ? default_val : val_obj[found->second].second;
|
||||
}
|
||||
virtual value & at(const value & key) override {
|
||||
if (!has_key(key)) {
|
||||
throw std::runtime_error("Key '" + key->as_string().str() + "' not found in value of type " + type());
|
||||
}
|
||||
return unordered.at(key);
|
||||
return val_obj[key_index.at(key)].second;
|
||||
}
|
||||
virtual value & at(const std::string & key, value & default_val) override {
|
||||
value key_val = mk_val<value_string>(key);
|
||||
@@ -599,7 +642,7 @@ struct value_object_t : public value_t {
|
||||
}
|
||||
protected:
|
||||
virtual bool equivalent(const value_t & other) const override {
|
||||
return typeid(*this) == typeid(other) && is_hashable() && other.is_hashable() && std::equal(val_obj.begin(), val_obj.end(), other.val_obj.begin(), other.val_obj.end(), value_equivalence());
|
||||
return kind == other.kind && is_hashable() && other.is_hashable() && std::equal(val_obj.begin(), val_obj.end(), other.as_ordered_object().begin(), other.as_ordered_object().end(), value_equivalence());
|
||||
}
|
||||
};
|
||||
using value_object = std::shared_ptr<value_object_t>;
|
||||
@@ -609,6 +652,8 @@ using value_object = std::shared_ptr<value_object_t>;
|
||||
//
|
||||
|
||||
struct value_none_t : public value_t {
|
||||
static constexpr auto kind_id = value_kind::NONE;
|
||||
value_none_t() : value_t(kind_id) {}
|
||||
virtual std::string type() const override { return "None"; }
|
||||
virtual bool is_none() const override { return true; }
|
||||
virtual bool as_bool() const override { return false; }
|
||||
@@ -621,14 +666,15 @@ struct value_none_t : public value_t {
|
||||
}
|
||||
protected:
|
||||
virtual bool equivalent(const value_t & other) const override {
|
||||
return typeid(*this) == typeid(other);
|
||||
return kind == other.kind;
|
||||
}
|
||||
};
|
||||
using value_none = std::shared_ptr<value_none_t>;
|
||||
|
||||
struct value_undefined_t : public value_t {
|
||||
static constexpr auto kind_id = value_kind::UNDEFINED;
|
||||
std::string hint; // for debugging, to indicate where undefined came from
|
||||
value_undefined_t(const std::string & h = "") : hint(h) {}
|
||||
value_undefined_t(const std::string & h = "") : value_t(kind_id), hint(h) {}
|
||||
virtual std::string type() const override { return hint.empty() ? "Undefined" : "Undefined (hint: '" + hint + "')"; }
|
||||
virtual bool is_undefined() const override { return true; }
|
||||
virtual bool as_bool() const override { return false; }
|
||||
@@ -705,12 +751,14 @@ private:
|
||||
};
|
||||
|
||||
struct value_func_t : public value_t {
|
||||
static constexpr auto kind_id = value_kind::FUNCTION;
|
||||
func_handler val_func;
|
||||
std::string name;
|
||||
value arg0; // bound "this" argument, if any
|
||||
value_func_t(const std::string & name, const func_handler & func) : name(name) {
|
||||
value_func_t(const std::string & name, const func_handler & func) : value_t(kind_id), name(name) {
|
||||
val_func = func;
|
||||
}
|
||||
value_func_t(const std::string & name, const func_handler & func, const value & arg_this) : name(name), arg0(arg_this) {
|
||||
value_func_t(const std::string & name, const func_handler & func, const value & arg_this) : value_t(kind_id), name(name), arg0(arg_this) {
|
||||
val_func = func;
|
||||
}
|
||||
virtual value invoke(const func_args & args) const override {
|
||||
@@ -735,18 +783,22 @@ protected:
|
||||
// Note: this is unused for now, we don't support function as object keys
|
||||
// compare function pointers
|
||||
// (val_func == other.val_func does not work as std::function::operator== is only used for nullptr check)
|
||||
if (kind != other.kind) {
|
||||
return false;
|
||||
}
|
||||
const auto target_this = this->val_func.target<func_hptr>();
|
||||
const auto target_other = other.val_func.target<func_hptr>();
|
||||
return typeid(*this) == typeid(other) && target_this == target_other;
|
||||
const auto target_other = static_cast<const value_func_t &>(other).val_func.target<func_hptr>();
|
||||
return target_this == target_other;
|
||||
}
|
||||
};
|
||||
using value_func = std::shared_ptr<value_func_t>;
|
||||
|
||||
// special value for kwarg
|
||||
struct value_kwarg_t : public value_t {
|
||||
static constexpr auto kind_id = value_kind::KEYWORD;
|
||||
std::string key;
|
||||
value val;
|
||||
value_kwarg_t(const std::string & k, const value & v) : key(k), val(v) {}
|
||||
value_kwarg_t(const std::string & k, const value & v) : value_t(kind_id), key(k), val(v) {}
|
||||
virtual std::string type() const override { return "KwArg"; }
|
||||
virtual std::string as_repr() const override { return type(); }
|
||||
virtual bool is_hashable() const override { return true; }
|
||||
@@ -760,7 +812,7 @@ struct value_kwarg_t : public value_t {
|
||||
protected:
|
||||
virtual bool equivalent(const value_t & other) const override {
|
||||
const value_kwarg_t & other_val = static_cast<const value_kwarg_t &>(other);
|
||||
return typeid(*this) == typeid(other) && key == other_val.key && val == other_val.val;
|
||||
return kind == other.kind && key == other_val.key && val == other_val.val;
|
||||
}
|
||||
};
|
||||
using value_kwarg = std::shared_ptr<value_kwarg_t>;
|
||||
|
||||
@@ -303,7 +303,8 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4;
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = "<|channel>thought";
|
||||
data.thinking_end_tags = {"<channel|>"};
|
||||
// a tool call may start before the thought channel is closed; <channel|> stays first, the budget sampler forces it
|
||||
data.thinking_end_tags = {"<channel|>", "<|tool_call>"};
|
||||
|
||||
data.preserved_tokens = {
|
||||
"<|channel>",
|
||||
@@ -334,9 +335,10 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
|
||||
auto start = p.rule("start", p.optional(p.literal("<|turn>model\n")));
|
||||
|
||||
if (extract_reasoning) {
|
||||
p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("<channel|>")) + p.literal("<channel|>"));
|
||||
// the model may start a tool call before closing the thought channel
|
||||
p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until_one_of({"<channel|>", "<|tool_call>"})) + (p.literal("<channel|>") | p.peek(p.literal("<|tool_call>"))));
|
||||
} else {
|
||||
p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until("<channel|>") + p.literal("<channel|>")));
|
||||
p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until_one_of({"<channel|>", "<|tool_call>"}) + (p.literal("<channel|>") | p.peek(p.literal("<|tool_call>")))));
|
||||
}
|
||||
|
||||
auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), "");
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
#include "parsers.h"
|
||||
|
||||
bool is_inkling_template(const std::string & src) {
|
||||
return src.find("<|message_user|>") != std::string::npos &&
|
||||
src.find("<|content_image|>") != std::string::npos;
|
||||
}
|
||||
|
||||
// the template makes one message block per content part, but a media marker comes inside the text of a message
|
||||
// <header><|content_text|>{media}text<|end_message|> --> <header>{media}<|end_message|><header><|content_text|>text<|end_message|>
|
||||
static void inkling_split_media(common_chat_prompt & prompt) {
|
||||
const std::string TEXT = "<|content_text|>";
|
||||
const std::string END = "<|end_message|>";
|
||||
const std::string MEDIA_BEG = "<__media_";
|
||||
const std::string MEDIA_END = "__>";
|
||||
|
||||
std::vector<common_chat_prompt_part> out;
|
||||
for (size_t i = 0; i < prompt.parts.size(); i++) {
|
||||
const auto & part = prompt.parts[i];
|
||||
const bool in_text_block = part.is_input && part.text.find(MEDIA_BEG) != std::string::npos &&
|
||||
!out.empty() && !out.back().is_input && string_ends_with(out.back().text, TEXT) &&
|
||||
i + 1 < prompt.parts.size() && !prompt.parts[i + 1].is_input && string_starts_with(prompt.parts[i + 1].text, END);
|
||||
const size_t header_pos = in_text_block ? out.back().text.rfind("<|message_") : std::string::npos;
|
||||
if (header_pos == std::string::npos) {
|
||||
out.push_back(part);
|
||||
continue;
|
||||
}
|
||||
const std::string header = out.back().text.substr(header_pos, out.back().text.size() - TEXT.size() - header_pos);
|
||||
|
||||
std::string rest = part.text;
|
||||
for (bool first = true; !rest.empty(); first = false) {
|
||||
const size_t beg = rest.find(MEDIA_BEG);
|
||||
const size_t end = beg == std::string::npos ? beg : rest.find(MEDIA_END, beg);
|
||||
const bool is_media = beg == 0 && end != std::string::npos;
|
||||
const size_t len = is_media ? end + MEDIA_END.size() : (end == std::string::npos ? rest.size() : beg);
|
||||
if (!first) {
|
||||
out.push_back({ END + header + (is_media ? "" : TEXT), false });
|
||||
} else if (is_media) {
|
||||
// mtmd opens the media block
|
||||
out.back().text.resize(out.back().text.size() - TEXT.size());
|
||||
}
|
||||
out.push_back({ rest.substr(0, len), true });
|
||||
rest.erase(0, len);
|
||||
}
|
||||
}
|
||||
prompt.parts = std::move(out);
|
||||
}
|
||||
|
||||
// Inkling 1 and 2 - the assistant turn is a list of blocks, all closed by <|end_message|>:
|
||||
// <|message_model|><|content_thinking|>...<|end_message|>
|
||||
// <|message_model|><|content_text|>...<|end_message|>
|
||||
// <|message_model|>NAME<|content_invoke_tool_json|>{"name":"NAME","args":{...}}<|end_message|>
|
||||
common_chat_params common_chat_params_init_inkling(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
inkling_split_media(data.prompt);
|
||||
|
||||
const std::string HEADER = "<|message_model|>";
|
||||
const std::string THINK = "<|content_thinking|>";
|
||||
const std::string TEXT = "<|content_text|>";
|
||||
const std::string TOOL_JSON = "<|content_invoke_tool_json|>";
|
||||
const std::string END = "<|end_message|>";
|
||||
|
||||
data.preserved_tokens = { HEADER, THINK, TEXT, TOOL_JSON, END };
|
||||
|
||||
data.thinking_start_tag = THINK;
|
||||
data.thinking_end_tags = { END };
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, HEADER },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|message_user|>" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|message_system|>" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|message_tool|>" },
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt.clear();
|
||||
if (!msg.reasoning_content.empty()) {
|
||||
data.generation_prompt = HEADER + THINK + msg.reasoning_content;
|
||||
}
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
if (!data.generation_prompt.empty()) {
|
||||
data.generation_prompt += END;
|
||||
}
|
||||
data.generation_prompt += HEADER + TEXT + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto use_tools = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto header = p.literal(HEADER);
|
||||
auto end = p.literal(END);
|
||||
|
||||
auto think = p.literal(THINK);
|
||||
auto thinking = p.rule("thinking", extract_reasoning ? think + p.reasoning(p.until(END)) + end :
|
||||
p.atomic(p.content(think)) + p.content(p.until(END) + end));
|
||||
auto text = p.rule("text", p.literal(TEXT) + p.content(p.until(END)) + end);
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_format = p.rule("response-format", p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
|
||||
return p.zero_or_more(header + thinking) + header + p.literal(TEXT) + response_format + end;
|
||||
}
|
||||
|
||||
auto blocks = p.zero_or_more(header + (thinking | text));
|
||||
|
||||
if (!use_tools) {
|
||||
return blocks;
|
||||
}
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
|
||||
auto open = p.tool_name(p.literal(name)) + p.literal(TOOL_JSON) + p.literal("{\"name\":" + json(name).dump() + ",\"args\":");
|
||||
auto args = p.tool_args(p.json());
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), p.tool(p.tool_open(open) + args + p.tool_close(p.literal("}")) + end));
|
||||
});
|
||||
|
||||
// tool calls are the last blocks of the turn
|
||||
auto calls = inputs.parallel_tool_calls ? tool_choice + p.zero_or_more(header + tool_choice) : tool_choice;
|
||||
auto tool_calls = header + p.trigger_rule("tool-calls", calls);
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return blocks + tool_calls;
|
||||
}
|
||||
|
||||
return blocks + p.optional(tool_calls);
|
||||
});
|
||||
|
||||
if (has_response_format || use_tools) {
|
||||
data.grammar_lazy = !has_response_format && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
if (data.grammar_lazy) {
|
||||
// a header followed by text and not by a content token is a tool call, the grammar starts at the name
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|message_model\\|>([^<])" },
|
||||
};
|
||||
// Inkling 1: the first header is in the generation prompt, and a lazy grammar does not get the prompt.
|
||||
// an output that starts with no content token is a tool call. not for a continuation, it starts with text
|
||||
if (!inputs.has_continuation()) {
|
||||
data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL, "[^<]+" });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -83,7 +83,8 @@ common_chat_params common_chat_params_init_lfm2(const common_chat_template &
|
||||
auto tool_calls = p.rule("tool-calls",
|
||||
p.trigger_rule("tool-call",
|
||||
p.literal(TOOL_CALL_START) +
|
||||
p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls, /* allow_json_literals = */ true) +
|
||||
p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls, /* allow_json_literals = */ true,
|
||||
/* allow_undeclared = */ true) +
|
||||
p.literal(TOOL_CALL_END)
|
||||
)
|
||||
);
|
||||
|
||||
@@ -50,7 +50,7 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat
|
||||
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = "[THINK]";
|
||||
data.thinking_end_tags = {"[/THINK]"};
|
||||
data.thinking_end_tags = {"[/THINK]", "[TOOL_CALLS]"};
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
@@ -75,7 +75,7 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.eps();
|
||||
auto reasoning =
|
||||
extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();
|
||||
extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until_one_of({"[/THINK]", "[TOOL_CALLS]"})) + p.choice({p.literal("[/THINK]"), p.peek(p.literal("[TOOL_CALLS]"))})) : p.eps();
|
||||
|
||||
// Response format parser
|
||||
if (has_response_format) {
|
||||
|
||||
@@ -41,6 +41,8 @@ std::string common_chat_template_generation_prompt_impl(
|
||||
|
||||
bool is_lfm2_template(const std::string & src);
|
||||
|
||||
bool is_inkling_template(const std::string & src);
|
||||
|
||||
common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
@@ -55,6 +57,8 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
|
||||
|
||||
common_chat_params common_chat_params_init_harmony_xml(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_inkling(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_k2_horizon(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
@@ -10,6 +10,7 @@ set(LLAMA_CHAT_PARSERS_SOURCES
|
||||
${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/harmony-xml.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/inkling.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/k2-horizon.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp
|
||||
|
||||
@@ -1003,9 +1003,9 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id
|
||||
} else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {
|
||||
return "Ref(" + p.name + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_tag_parser>) {
|
||||
return "Tag(" + p.tag + ", " + dump(p.child) + ")";
|
||||
return "Tag(" + p.tag + ", " + dump_impl(p.child, visited) + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_atomic_parser>) {
|
||||
return "Atomic(" + dump(p.child) + ")";
|
||||
return "Atomic(" + dump_impl(p.child, visited) + ")";
|
||||
} else {
|
||||
return "Unknown";
|
||||
}
|
||||
|
||||
+9
-1
@@ -203,6 +203,13 @@ struct common_sampler * common_sampler_init(
|
||||
throw std::invalid_argument("penalty_present must be finite");
|
||||
}
|
||||
const llama_vocab * vocab = llama_model_get_vocab(model);
|
||||
if (!common_grammar_value(params.grammar).empty()) {
|
||||
for (const auto & b : params.logit_bias) {
|
||||
if (b.bias == -INFINITY && llama_vocab_is_eog(vocab, b.token)) {
|
||||
throw std::invalid_argument("EOG tokens cannot be banned when a grammar is set");
|
||||
}
|
||||
}
|
||||
}
|
||||
llama_sampler_chain_params lparams = llama_sampler_chain_default_params();
|
||||
|
||||
lparams.no_perf = params.no_perf;
|
||||
@@ -255,7 +262,8 @@ struct common_sampler * common_sampler_init(
|
||||
break;
|
||||
}
|
||||
default:
|
||||
GGML_ASSERT(false && "unknown trigger type");
|
||||
throw std::runtime_error(
|
||||
"unknown grammar trigger type: " + std::to_string(trigger.type));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+585
-253
File diff suppressed because it is too large
Load Diff
@@ -73,7 +73,7 @@ struct common_speculative_draft_params {
|
||||
// the generated draft from the last _draft() call
|
||||
llama_tokens * result;
|
||||
|
||||
// candidate distribution per drafted token; set it to make draft-simple, draft-mtp and draft-dflash (DFlash2 only) sample
|
||||
// candidate distribution per drafted token; set it to make draft-simple, draft-mtp and draft-dflash2 sample
|
||||
std::vector<std::vector<llama_token_data>> * result_q = nullptr;
|
||||
|
||||
// the target's temp and seed, read only when the drafter samples probabilistically
|
||||
|
||||
@@ -126,6 +126,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"GlmOcrForConditionalGeneration": "glm",
|
||||
"GptOssForCausalLM": "gpt_oss",
|
||||
"GraniteForCausalLM": "granite",
|
||||
"Granite4VisionForConditionalGeneration": "granite",
|
||||
"GraniteMoeForCausalLM": "granite",
|
||||
"GraniteMoeHybridForCausalLM": "granite",
|
||||
"GraniteMoeSharedForCausalLM": "granite",
|
||||
@@ -137,6 +138,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Grok1ForCausalLM": "grok",
|
||||
"GrokForCausalLM": "grok",
|
||||
"GroveMoeForCausalLM": "grovemoe",
|
||||
"HiggsAudio3Model": "higgs_audio",
|
||||
"HunYuanDenseV1ForCausalLM": "hunyuan",
|
||||
"HunYuanMoEV1ForCausalLM": "hunyuan",
|
||||
"HunYuanVLForConditionalGeneration": "hunyuan",
|
||||
@@ -144,6 +146,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"HYV3ForCausalLM": "hunyuan",
|
||||
"HYV4ForCausalLM": "hy_v4",
|
||||
"IQuestCoderForCausalLM": "llama",
|
||||
"Inkling2ForCausalLM": "inkling2",
|
||||
"Inkling2ForConditionalGeneration": "inkling2",
|
||||
"InternLM2ForCausalLM": "internlm",
|
||||
"InternLM3ForCausalLM": "internlm",
|
||||
"JAISLMHeadModel": "jais",
|
||||
@@ -217,6 +221,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"MiniMaxM2ForCausalLM": "minimax",
|
||||
"MiniMaxM3SparseForCausalLM": "minimax",
|
||||
"MiniMaxM3SparseForConditionalGeneration": "minimax",
|
||||
"DSparkMiniMaxDraftModel": "minimax",
|
||||
"Ministral3ForCausalLM": "mistral3",
|
||||
"Ministral3Model": "mistral3",
|
||||
"Mistral3ForConditionalGeneration": "mistral3",
|
||||
@@ -278,6 +283,11 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Qwen3NextForCausalLM": "qwen",
|
||||
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
|
||||
"PocketTTSModel": "pockettts",
|
||||
"KokoroModel": "kokoro",
|
||||
"TinyVocaModel": "tiny_voca",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
|
||||
@@ -320,6 +330,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"UMT5Model": "t5",
|
||||
"UltravoxModel": "ultravox",
|
||||
"UnlimitedOCRForCausalLM": "deepseek",
|
||||
"VibeVoiceAsrForConditionalGeneration": "vibevoice_asr",
|
||||
"VLlama3ForCausalLM": "llama",
|
||||
"VllmDecision2Model": "vllm_decision2",
|
||||
"VoxtralForConditionalGeneration": "llama",
|
||||
@@ -329,6 +340,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"XverseForCausalLM": "xverse",
|
||||
"YoutuForCausalLM": "deepseek",
|
||||
"YoutuVLForConditionalGeneration": "deepseek",
|
||||
"fish_qwen3_omni": "fish_s2",
|
||||
"modeling_grove_moe.GroveMoeForCausalLM": "grovemoe",
|
||||
"modeling_sarvam_moe.SarvamMoEForCausalLM": "bailingmoe",
|
||||
}
|
||||
@@ -352,6 +364,9 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"EmbeddingGemma2Model": "gemma",
|
||||
"Exaone4_5_ForConditionalGeneration": "exaone",
|
||||
"FishOmniModel": "drama3",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Gemma3ForConditionalGeneration": "gemma",
|
||||
"Gemma3nForConditionalGeneration": "gemma",
|
||||
"Gemma4ForConditionalGeneration": "gemma",
|
||||
@@ -366,8 +381,10 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"Granite4VisionForConditionalGeneration": "granite",
|
||||
"GraniteSpeechForConditionalGeneration": "granite",
|
||||
"GraniteSpeechPlusForConditionalGeneration": "granite",
|
||||
"HiggsAudio3Model": "higgs_audio",
|
||||
"HunYuanVLForConditionalGeneration": "hunyuan",
|
||||
"Idefics3ForConditionalGeneration": "smolvlm",
|
||||
"Inkling2ForConditionalGeneration": "inkling2",
|
||||
"InternVisionModel": "internvl",
|
||||
"JanusForConditionalGeneration": "januspro",
|
||||
"KimiK25ForConditionalGeneration": "kimivl",
|
||||
@@ -408,6 +425,8 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"Qwen3ASRForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
|
||||
"PocketTTSModel": "pockettts",
|
||||
"KokoroModel": "kokoro",
|
||||
"TinyVocaModel": "tiny_voca",
|
||||
"SopranoModel": "soprano",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
@@ -424,9 +443,11 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"Step3p7ForConditionalGeneration": "step3",
|
||||
"UltravoxModel": "ultravox",
|
||||
"UnlimitedOCRForCausalLM": "deepseek",
|
||||
"VibeVoiceAsrForConditionalGeneration": "vibevoice_asr",
|
||||
"VoxtralForConditionalGeneration": "ultravox",
|
||||
"VoxtralRealtimeForConditionalGeneration": "ultravox",
|
||||
"YoutuVLForConditionalGeneration": "youtuvl",
|
||||
"fish_qwen3_omni": "fish_s2",
|
||||
}
|
||||
|
||||
|
||||
|
||||
+45
-4
@@ -398,6 +398,9 @@ class ModelBase:
|
||||
return data / scale.float()
|
||||
|
||||
def dequant_simple(weight: Tensor, scale: Tensor, block_size: Sequence[int] | None = None) -> Tensor:
|
||||
if scale.dtype == torch.uint8:
|
||||
# E8M0 exponent stored as raw bytes (MXFP8)
|
||||
scale = (scale.float() - 127.0).exp2()
|
||||
scale = scale.float()
|
||||
|
||||
# ModelOpt block scales are pre-shaped for broadcasting, e.g. [out/128, 1, in/128, 1]
|
||||
@@ -538,7 +541,7 @@ class ModelBase:
|
||||
s = self.model_tensors[name]
|
||||
self.model_tensors[weight_name] = lambda w=w, s=s: dequant_bitnet(w(), s())
|
||||
tensors_to_remove.append(name)
|
||||
elif quant_method in ("fp8", "bsgemm-moe"): # bsgemm-moe: drama-3, same block layout
|
||||
elif quant_method in ("fp8", "bsgemm-moe", "mxfp8"): # bsgemm-moe: drama-3, same block layout
|
||||
block_size = quant_config.get("weight_block_size")
|
||||
for name in self.model_tensors.keys():
|
||||
if name.endswith("_scale_inv"):
|
||||
@@ -650,6 +653,28 @@ class ModelBase:
|
||||
raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported")
|
||||
elif quant_method in ("modelopt", "modelopt_mixed"):
|
||||
dequant_fp8()
|
||||
# FP8_PB_WO: one scale per block, the block size follows from the two shapes
|
||||
for name in self.model_tensors.keys():
|
||||
if not name.endswith(".weight_scale_inv"):
|
||||
continue
|
||||
weight_name = name.removesuffix("_scale_inv")
|
||||
if weight_name not in self.model_tensors:
|
||||
tensors_to_remove.append(name)
|
||||
continue
|
||||
w = self.model_tensors[weight_name]
|
||||
s = self.model_tensors[name]
|
||||
if w().dtype != torch.float8_e4m3fn:
|
||||
continue
|
||||
w_shape, s_shape = w().shape, s().shape
|
||||
block_size = None
|
||||
if prod(s_shape) != 1:
|
||||
if len(s_shape) != len(w_shape) or any(ws % ss for ws, ss in zip(w_shape, s_shape)):
|
||||
raise ValueError(f"FP8 weight {weight_name!r} {list(w_shape)} is not tiled by its scale {list(s_shape)}")
|
||||
block_size = [ws // ss for ws, ss in zip(w_shape, s_shape)]
|
||||
self.model_tensors[weight_name] = lambda w=w, s=s, bs=block_size: dequant_simple(w(), s(), bs)
|
||||
tensors_to_remove.append(name)
|
||||
if self._fp8_as_q8:
|
||||
self._fp8_dequantized.add(weight_name)
|
||||
elif quant_method is not None:
|
||||
raise NotImplementedError(f"Quant method is not yet supported: {quant_method!r}")
|
||||
|
||||
@@ -1357,10 +1382,10 @@ class ModelBase:
|
||||
consumed.append(input_scale_name)
|
||||
|
||||
# Check if this is a per-expert tensor
|
||||
m = re.search(r'\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight$', name)
|
||||
m = re.search(r'\.experts\.(\d+)\.(gate_proj|up_proj|down_proj|w1|w2|w3)\.weight$', name)
|
||||
if m:
|
||||
expert_id = int(m.group(1))
|
||||
proj_type = m.group(2)
|
||||
proj_type = {"w1": "gate_proj", "w2": "down_proj", "w3": "up_proj"}.get(m.group(2), m.group(2))
|
||||
bid_m = re.search(r'\.layers\.(\d+)\.', name)
|
||||
bid = int(bid_m.group(1)) if bid_m else 0
|
||||
key = (bid, proj_type)
|
||||
@@ -1451,6 +1476,8 @@ class ModelBase:
|
||||
quant_layers = quant_config.get("quantized_layers", quant_layers) or {}
|
||||
|
||||
nvfp4_compressed_tensors = self._is_nvfp4_compressed_tensors(quant_method, quant_format, quant_groups)
|
||||
# e.g. MXFP8 dense weights with compressed-tensors NVFP4 experts
|
||||
nvfp4_compressed_tensors |= quantization_config.get("moe_quant_format") == "nvfp4-pack-quantized"
|
||||
|
||||
self._nvfp4_global_algo = quant_algo
|
||||
|
||||
@@ -1979,6 +2006,14 @@ class TextModel(ModelBase):
|
||||
if (local_rope_theta := self.rope_parameters.get("sliding_attention", {}).get("rope_theta")) is not None:
|
||||
self.gguf_writer.add_rope_freq_base_swa(local_rope_theta)
|
||||
logger.info(f"gguf: rope theta swa = {local_rope_theta}")
|
||||
# Add rope_pattern from no_rope_layers
|
||||
if no_rope_layers := self.hparams.get("no_rope_layers"):
|
||||
if len(no_rope_layers) == self.block_count: # "no_rope_layers": [0, 0, 0, 1]
|
||||
rope_pattern = [bool(x) for x in no_rope_layers]
|
||||
else: # "no_rope_layers": [3]
|
||||
rope_pattern = [i not in no_rope_layers for i in range(self.block_count)]
|
||||
self.gguf_writer.add_rope_pattern(rope_pattern)
|
||||
logger.info(f"gguf: rope_pattern = {sum(rope_pattern)} RoPE layers / {len(rope_pattern)} total")
|
||||
if (f_rms_eps := self.find_hparam(["rms_norm_eps", "norm_eps"], optional=True)) is not None:
|
||||
self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps)
|
||||
logger.info(f"gguf: rms norm epsilon = {f_rms_eps}")
|
||||
@@ -2432,6 +2467,9 @@ class TextModel(ModelBase):
|
||||
if chkhsh == "a9af07a84191f55098b248ae6f3dfe9e32d3190bebe8eafd91c1ddec9bc3449f":
|
||||
# ref: https://huggingface.co/IFM/K2-Horizon-36B
|
||||
res = "k2-horizon"
|
||||
if chkhsh == "5635e1c8d621dff7b4825f41f2f97b6ceb39bcf2d5c541db28577080d6efd5e5":
|
||||
# ref: https://huggingface.co/thinkingmachines/Inkling-2-Small
|
||||
res = "inkling2"
|
||||
|
||||
if res is None:
|
||||
logger.warning("\n")
|
||||
@@ -3461,7 +3499,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
|
||||
# Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older
|
||||
# Kimi-Linear-48B architecture and cannot load K3 (no attention residuals,
|
||||
# latent MoE, situ, ...). Route on the top-level architecture instead.
|
||||
if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration"):
|
||||
if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration", "HiggsAudio3Model"):
|
||||
return arch
|
||||
|
||||
# if "architectures" is found in the sub-config, use that instead
|
||||
@@ -3472,6 +3510,9 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
|
||||
# some models (llava_onevision2) only declare the architecture inside text_config
|
||||
if arch is None and text_config.get("architectures") is not None:
|
||||
arch = text_config["architectures"][0]
|
||||
# some configs only have a model_type (fishaudio/s2-pro)
|
||||
if arch is None:
|
||||
arch = hparams.get("model_type")
|
||||
if arch is None:
|
||||
raise ValueError("Failed to detect model architecture")
|
||||
return arch
|
||||
|
||||
+32
-19
@@ -18,6 +18,30 @@ _TOWER_SHARED = "audio_decoder_mtp_shared.layers."
|
||||
_TOWER_RESIDUAL = "audio_decoder_mtp.layers."
|
||||
|
||||
|
||||
def load_codec(dir_model) -> dict[str, Tensor]:
|
||||
sd = torch.load(dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
|
||||
return sd.get("state_dict", sd)
|
||||
|
||||
|
||||
def fold_conv(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# fold weight norm
|
||||
if prefix + ".conv.weight" in sd:
|
||||
return sd[prefix + ".conv.weight"].float()
|
||||
g = sd[prefix + ".conv.parametrizations.weight.original0"].float()
|
||||
v = sd[prefix + ".conv.parametrizations.weight.original1"].float()
|
||||
return g * v / v.norm(dim=(1, 2), keepdim=True)
|
||||
|
||||
|
||||
def fold_wn(g: Tensor, v: Tensor) -> Tensor:
|
||||
# fold old-style weight norm of a 1x1 conv, returns [out, in]
|
||||
return (g.float() * v.float() / v.float().norm(dim=(1, 2), keepdim=True)).squeeze(-1)
|
||||
|
||||
|
||||
def fold_snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
|
||||
alpha = alpha.float().flatten()
|
||||
return alpha, 1.0 / (alpha + 1e-9)
|
||||
|
||||
|
||||
@ModelBase.register("FishOmniModel")
|
||||
@ModelBase.example("fishaudio/drama-3")
|
||||
class Drama3TextModel(Qwen3_5MoeTextModel):
|
||||
@@ -36,6 +60,7 @@ class Drama3TextModel(Qwen3_5MoeTextModel):
|
||||
class Drama3MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
gen_projector_type = gguf.VisionProjectorType.DRAMA3_GEN
|
||||
|
||||
_LAYER_MAP = {
|
||||
"enorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ENORM, True),
|
||||
@@ -69,7 +94,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
cfg = self.hparams
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.DRAMA3_GEN)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(self.gen_projector_type)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(cfg["dim"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(cfg["intermediate_size"])
|
||||
@@ -153,32 +178,20 @@ class Drama3MmprojModel(MmprojModel):
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# only the DAC decoder, the encoder is for voice cloning (not supported yet)
|
||||
sd = torch.load(self.dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
|
||||
sd = sd.get("state_dict", sd)
|
||||
sd = load_codec(self.dir_model)
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
def get(name: str) -> Tensor:
|
||||
return sd[name].float()
|
||||
|
||||
def conv(prefix: str) -> Tensor:
|
||||
# fold weight norm
|
||||
if prefix + ".conv.weight" in sd:
|
||||
return get(prefix + ".conv.weight")
|
||||
g = get(prefix + ".conv.parametrizations.weight.original0")
|
||||
v = get(prefix + ".conv.parametrizations.weight.original1")
|
||||
return g * v / v.norm(dim=(1, 2), keepdim=True)
|
||||
return fold_conv(sd, prefix)
|
||||
|
||||
def quant_table(prefix: str) -> Tensor:
|
||||
# fold out_proj into the codebook, so decoding is a row lookup
|
||||
g = get(prefix + ".out_proj.weight_g")
|
||||
v = get(prefix + ".out_proj.weight_v")
|
||||
w = (g * v / v.norm(dim=(1, 2), keepdim=True)).squeeze(-1)
|
||||
w = fold_wn(sd[prefix + ".out_proj.weight_g"], sd[prefix + ".out_proj.weight_v"])
|
||||
return get(prefix + ".codebook.weight") @ w.T + get(prefix + ".out_proj.bias")
|
||||
|
||||
def snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
|
||||
alpha = alpha.flatten()
|
||||
return alpha, 1.0 / (alpha + 1e-9)
|
||||
|
||||
q = "quantizer."
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), quant_table(q + "semantic_quantizer.quantizers.0"))
|
||||
n_rest = sum(1 for k in sd if k.startswith(q + "quantizer.quantizers.") and k.endswith(".codebook.weight"))
|
||||
@@ -228,7 +241,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get(d + "0.conv.bias"))
|
||||
for bid in range(4):
|
||||
p = f"{d}{bid + 1}.block."
|
||||
a, b = snake(get(p + "0.alpha"))
|
||||
a, b = fold_snake(get(p + "0.alpha"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), conv(p + "1"))
|
||||
@@ -236,7 +249,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
for xid in range(3):
|
||||
r = f"{p}{xid + 2}.block."
|
||||
for act, tensor in (("0", T.A_GEN_WAV_DAC_RES_ACT1), ("2", T.A_GEN_WAV_DAC_RES_ACT2)):
|
||||
a, b = snake(get(r + act + ".alpha"))
|
||||
a, b = fold_snake(get(r + act + ".alpha"))
|
||||
name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
|
||||
yield (name + ".alpha", a)
|
||||
yield (name + ".beta", b)
|
||||
@@ -244,7 +257,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
|
||||
yield (name + ".weight", conv(r + cv))
|
||||
yield (name + ".bias", get(r + cv + ".conv.bias"))
|
||||
a, b = snake(get(d + "5.alpha"))
|
||||
a, b = fold_snake(get(d + "5.alpha"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6"))
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, gguf
|
||||
from .drama3 import Drama3MmprojModel, fold_conv, fold_snake, fold_wn, load_codec
|
||||
from .qwen import Qwen3Model
|
||||
|
||||
# Fish Audio S2 Pro (dual-AR)
|
||||
# text model: Qwen3 backbone (slow AR), samples the semantic code of each frame
|
||||
# mmproj: fast AR code predictor (other codebooks) and Fish DAC decoder (codec.pth, same as drama3)
|
||||
|
||||
# residual codebook size of codec.pth, the fast AR head is cut to it
|
||||
_N_RESIDUAL_CODES = 1024
|
||||
|
||||
# codec encoder strides (encoder_rates of the codec config), the quantizer then downsamples 2x2
|
||||
_CODEC_ENC_STRIDES = (2, 4, 8, 8)
|
||||
|
||||
# codec encoder transformers, both 1024 wide with 16 heads of 64
|
||||
# generic audio layers a.blk.0..3: last encoder block, a.blk.4..11: quantizer pre_module
|
||||
_N_ENC_TFM_LAYERS = 4
|
||||
_N_PRE_TFM_LAYERS = 8
|
||||
_CODEC_HEAD_DIM = 64
|
||||
|
||||
|
||||
def _load_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
|
||||
cfg = hparams["text_config"]
|
||||
hparams["text_config"] = {
|
||||
"hidden_size": cfg["dim"],
|
||||
"num_hidden_layers": cfg["n_layer"],
|
||||
"num_attention_heads": cfg["n_head"],
|
||||
"num_key_value_heads": cfg["n_local_heads"],
|
||||
"head_dim": cfg["head_dim"],
|
||||
"intermediate_size": cfg["intermediate_size"],
|
||||
"rms_norm_eps": cfg["norm_eps"],
|
||||
"rope_theta": cfg["rope_base"],
|
||||
"max_position_embeddings": cfg["max_seq_len"],
|
||||
"vocab_size": cfg["vocab_size"],
|
||||
"tie_word_embeddings": cfg["tie_word_embeddings"],
|
||||
}
|
||||
return hparams
|
||||
|
||||
|
||||
def _neox_rows(w: Tensor, head_dim: int) -> Tensor:
|
||||
# checkpoint uses interleaved RoPE, reorder the rows of each head so NEOX RoPE gives the same result
|
||||
order = torch.cat([torch.arange(0, head_dim, 2), torch.arange(1, head_dim, 2)])
|
||||
return w.reshape(-1, head_dim, *w.shape[1:])[:, order].reshape(w.shape)
|
||||
|
||||
|
||||
@ModelBase.register("fish_qwen3_omni")
|
||||
@ModelBase.example("fishaudio/s2-pro")
|
||||
class FishS2TextModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# the rest is mapped by tensor_mapping.py
|
||||
name = name.removeprefix("text_model.")
|
||||
T = gguf.MODEL_TENSOR
|
||||
head_dim = self.hparams["head_dim"]
|
||||
if name.endswith(".attention.wqkv.weight"):
|
||||
n_q = self.hparams["num_attention_heads"] * head_dim
|
||||
n_kv = self.hparams["num_key_value_heads"] * head_dim
|
||||
q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
|
||||
yield (self.format_tensor_name(T.ATTN_Q, bid), _neox_rows(q, head_dim))
|
||||
yield (self.format_tensor_name(T.ATTN_K, bid), _neox_rows(k, head_dim))
|
||||
yield (self.format_tensor_name(T.ATTN_V, bid), v)
|
||||
return
|
||||
if name.endswith(".attention.q_norm.weight"):
|
||||
yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), _neox_rows(data_torch, head_dim))
|
||||
return
|
||||
if name.endswith(".attention.k_norm.weight"):
|
||||
yield (self.format_tensor_name(T.ATTN_K_NORM, bid), _neox_rows(data_torch, head_dim))
|
||||
return
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("fish_qwen3_omni")
|
||||
@ModelBase.example("fishaudio/s2-pro")
|
||||
class FishS2MmprojModel(Drama3MmprojModel):
|
||||
has_audio_encoder = True # codec encoder, for voice cloning
|
||||
gen_projector_type = gguf.VisionProjectorType.FISH_S2_GEN
|
||||
|
||||
_FAST_AR_MAP = {
|
||||
"attention_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
|
||||
"attention.wo.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
|
||||
"ffn_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
|
||||
"feed_forward.w1.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
|
||||
"feed_forward.w3.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP,
|
||||
"feed_forward.w2.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
|
||||
}
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
cfg = dict(self.global_config["audio_decoder_config"])
|
||||
cfg["num_hidden_layers"] = cfg["n_layer"]
|
||||
return cfg
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
# codec encoder: raw 44.1 kHz waveform -> backbone input rows, other hparams are hardcoded in tools/mtmd
|
||||
self.gguf_writer.add_clip_has_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.FISH_S2_SPKENC)
|
||||
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_audio_num_mel_bins(1)
|
||||
self.gguf_writer.add_audio_block_count(_N_ENC_TFM_LAYERS + _N_PRE_TFM_LAYERS)
|
||||
self.gguf_writer.add_audio_embedding_length(1024)
|
||||
self.gguf_writer.add_audio_feed_forward_length(3072)
|
||||
self.gguf_writer.add_audio_head_count(1024 // _CODEC_HEAD_DIM)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# the VQ picks the nearest codebook entry, small errors in the encoder change the codes
|
||||
if new_name.startswith("a.codec."):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# codec transformers: F16 gives codes closer to the reference than BF16
|
||||
if new_name.startswith("a.blk.") and n_dims == 2:
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name = item[0]
|
||||
if name.startswith("audio_decoder.") or name == "text_model.model.embeddings.weight":
|
||||
return item
|
||||
return None
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
T = gguf.MODEL_TENSOR
|
||||
cfg = self.hparams
|
||||
|
||||
# already named by generate_extra_tensors()
|
||||
if name.startswith("a."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
if name == "text_model.model.embeddings.weight":
|
||||
# semantic band of the backbone embeddings
|
||||
begin = self.global_config["semantic_start_token_id"]
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch[begin:begin + cfg["vocab_size"]])
|
||||
return
|
||||
|
||||
if name == "audio_decoder.codebook_embeddings.weight":
|
||||
# backbone input of a speech row, one table per codebook
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), data_torch.reshape(cfg["num_codebooks"], -1, data_torch.shape[-1]))
|
||||
return
|
||||
|
||||
if name == "audio_decoder.embeddings.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_TOK_EMBD), data_torch)
|
||||
return
|
||||
|
||||
if name == "audio_decoder.output.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch[:_N_RESIDUAL_CODES])
|
||||
return
|
||||
|
||||
if name == "audio_decoder.norm.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_OUTPUT_NORM), data_torch)
|
||||
return
|
||||
|
||||
assert bid is not None and name.startswith(f"audio_decoder.layers.{bid}.")
|
||||
key = name.removeprefix(f"audio_decoder.layers.{bid}.")
|
||||
if key == "attention.wqkv.weight":
|
||||
head_dim = cfg["head_dim"]
|
||||
n_q = cfg["n_head"] * head_dim
|
||||
n_kv = cfg["n_local_heads"] * head_dim
|
||||
q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, bid), _neox_rows(q, head_dim))
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, bid), _neox_rows(k, head_dim))
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, bid), v)
|
||||
return
|
||||
yield (self.format_tensor_name(self._FAST_AR_MAP[key], bid), data_torch)
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
yield from super().generate_extra_tensors()
|
||||
|
||||
sd = load_codec(self.dir_model)
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
def get(name: str) -> Tensor:
|
||||
return sd[name].float()
|
||||
|
||||
def tn(tensor: gguf.MODEL_TENSOR, bid: int | None = None, xid: int | None = None) -> str:
|
||||
return gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
|
||||
|
||||
def conv_wb(tensor: gguf.MODEL_TENSOR, prefix: str, bid: int | None = None, xid: int | None = None, stride: int = 1):
|
||||
# taps-major [K / stride, OC, stride * IC], see conv() in tools/mtmd/models/fish-s2-spkenc.cpp
|
||||
w = fold_conv(sd, prefix)
|
||||
n_out, n_in, k = w.shape
|
||||
w = w.reshape(n_out, n_in, k // stride, stride).permute(2, 0, 3, 1).reshape(k // stride, n_out, stride * n_in)
|
||||
yield (tn(tensor, bid, xid) + ".weight", w)
|
||||
yield (tn(tensor, bid, xid) + ".bias", get(prefix + ".conv.bias"))
|
||||
|
||||
def snake_ab(tensor: gguf.MODEL_TENSOR, name: str, bid: int | None = None, xid: int | None = None):
|
||||
a, b = fold_snake(get(name))
|
||||
yield (tn(tensor, bid, xid) + ".alpha", a)
|
||||
yield (tn(tensor, bid, xid) + ".beta", b)
|
||||
|
||||
def tfm_layers(prefix: str, n_layer: int, offset: int):
|
||||
for il in range(n_layer):
|
||||
p = f"{prefix}layers.{il}."
|
||||
bid = offset + il
|
||||
wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0)
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_Q, bid), _neox_rows(wq, _CODEC_HEAD_DIM))
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_K, bid), _neox_rows(wk, _CODEC_HEAD_DIM))
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_V, bid), wv)
|
||||
yield (self.format_tensor_name(T.A_ENC_OUTPUT, bid), get(p + "attention.wo.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_INPUT_NORM, bid), get(p + "attention_norm.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_SCALE, bid), get(p + "attention_layer_scale.gamma"))
|
||||
yield (self.format_tensor_name(T.A_ENC_OUTPUT_NORM, bid), get(p + "ffn_norm.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_GATE, bid), get(p + "feed_forward.w1.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_UP, bid), get(p + "feed_forward.w3.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_DOWN, bid), get(p + "feed_forward.w2.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_SCALE_LS, bid), get(p + "ffn_layer_scale.gamma"))
|
||||
|
||||
# encoder: entry conv, 4x (3 residual units, Snake, strided conv), Snake, post conv
|
||||
e = "encoder.block."
|
||||
yield from conv_wb(T.A_CODEC_ENC_ENTRY, e + "0")
|
||||
for bid, stride in enumerate(_CODEC_ENC_STRIDES):
|
||||
p = f"{e}{bid + 1}.block."
|
||||
for xid in range(3):
|
||||
r = f"{p}{xid}.block."
|
||||
yield from snake_ab(T.A_CODEC_ENC_RES_ACT1, r + "0.alpha", bid, xid)
|
||||
yield from conv_wb(T.A_CODEC_ENC_RES_CONV1, r + "1", bid, xid)
|
||||
yield from snake_ab(T.A_CODEC_ENC_RES_ACT2, r + "2.alpha", bid, xid)
|
||||
yield from conv_wb(T.A_CODEC_ENC_RES_CONV2, r + "3", bid, xid)
|
||||
yield from snake_ab(T.A_CODEC_ENC_SNAKE, p + "3.alpha", bid)
|
||||
yield from conv_wb(T.A_CODEC_ENC_CONV, p + "4", bid, stride=stride)
|
||||
# the last block ends with a windowed transformer
|
||||
yield from tfm_layers(f"{e}4.block.5.", _N_ENC_TFM_LAYERS, 0)
|
||||
yield (self.format_tensor_name(T.A_POST_NORM), get(f"{e}4.block.5.norm.weight"))
|
||||
yield from snake_ab(T.A_CODEC_ENC_POST_SNAKE, e + "5.alpha")
|
||||
yield from conv_wb(T.A_CODEC_ENC_POST_CONV, e + "6")
|
||||
|
||||
# quantizer: 2x (strided conv + ConvNeXt), windowed transformer, then semantic + residual VQ
|
||||
q = "quantizer."
|
||||
for bid in range(2):
|
||||
p = f"{q}downsample.{bid}."
|
||||
yield from conv_wb(T.A_CODEC_DOWN_CONV, p + "0", bid, stride=2)
|
||||
# depthwise: one [C] row per tap
|
||||
yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".weight", fold_conv(sd, p + "1.dwconv").squeeze(1).T.contiguous())
|
||||
yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".bias", get(p + "1.dwconv.conv.bias"))
|
||||
for key, tensor in (("norm", T.A_CODEC_DOWN_NORM), ("pwconv1", T.A_CODEC_DOWN_PW1), ("pwconv2", T.A_CODEC_DOWN_PW2)):
|
||||
yield (tn(tensor, bid) + ".weight", get(f"{p}1.{key}.weight"))
|
||||
yield (tn(tensor, bid) + ".bias", get(f"{p}1.{key}.bias"))
|
||||
yield (tn(T.A_CODEC_DOWN_GAMMA, bid), get(p + "1.gamma"))
|
||||
yield from tfm_layers(q + "pre_module.", _N_PRE_TFM_LAYERS, _N_ENC_TFM_LAYERS)
|
||||
yield (self.format_tensor_name(T.A_CODEC_PRE_NORM), get(q + "pre_module.norm.weight"))
|
||||
|
||||
# nearest codebook entry by cosine similarity, see VectorQuantize.decode_latents()
|
||||
def vq(prefix: str) -> tuple[Tensor, Tensor, Tensor]:
|
||||
w = fold_wn(sd[prefix + ".in_proj.weight_g"], sd[prefix + ".in_proj.weight_v"])
|
||||
cb = torch.nn.functional.normalize(get(prefix + ".codebook.weight"), dim=-1)
|
||||
return w, get(prefix + ".in_proj.bias"), cb
|
||||
|
||||
w, b, cb = vq(q + "semantic_quantizer.quantizers.0")
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".weight"), w)
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".bias"), b)
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_CB), cb)
|
||||
n_rest = self.hparams["num_codebooks"] - 1
|
||||
rest = [vq(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)]
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".weight"), torch.stack([r[0] for r in rest]))
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".bias"), torch.stack([r[1] for r in rest]))
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_CB), torch.stack([r[2] for r in rest]))
|
||||
+4
-10
@@ -14,7 +14,7 @@ from .llama import LlamaModel
|
||||
from .mamba import Mamba2Model
|
||||
|
||||
|
||||
@ModelBase.register("GraniteForCausalLM")
|
||||
@ModelBase.register("GraniteForCausalLM", "Granite4VisionForConditionalGeneration")
|
||||
@ModelBase.example("ibm-granite/granite-3.3-2b-instruct")
|
||||
class GraniteModel(LlamaModel):
|
||||
"""Conversion for IBM's GraniteForCausalLM"""
|
||||
@@ -112,14 +112,6 @@ class GraniteSWAModel(GraniteModel):
|
||||
logger.info("gguf: (granite_swa) sliding_window_pattern (inferred) = %d SWA layers / %d total",
|
||||
sum(is_swa), n_layers)
|
||||
|
||||
# Add rope_pattern from no_rope_layers
|
||||
if no_rope_layers := self.hparams.get("no_rope_layers"):
|
||||
# Convert 1/0 to bool (1 = use RoPE, 0 = NoPE)
|
||||
rope_pattern = [bool(x) for x in no_rope_layers]
|
||||
self.gguf_writer.add_rope_pattern(rope_pattern)
|
||||
logger.info("gguf: (granite_swa) rope_pattern = %d RoPE layers / %d total",
|
||||
sum(rope_pattern), len(rope_pattern))
|
||||
|
||||
|
||||
@ModelBase.register("GraniteMoeSWAForCausalLM")
|
||||
class GraniteMoeSWAModel(GraniteSWAModel):
|
||||
@@ -560,7 +552,9 @@ class GraniteHybridModel(Mamba2Model, GraniteMoeModel):
|
||||
or not self._ssm_layers
|
||||
)
|
||||
self.gguf_writer.add_rope_scaling_finetuned(use_rope)
|
||||
if not use_rope:
|
||||
# Set context length to an arbitrary large value if rope is not used AND no previous value is present.
|
||||
# https://huggingface.co/ibm-granite/granite-4.0-h-micro-GGUF/discussions/1
|
||||
if not use_rope and "max_position_embeddings" not in self.hparams:
|
||||
self.gguf_writer.add_context_length(2**20)
|
||||
|
||||
## Validation ##
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, gguf
|
||||
from .qwen import Qwen3Model
|
||||
from .ultravox import WhisperEncoderModel
|
||||
|
||||
|
||||
# plain ChatML as built by transcribe.py upstream, the Qwen3 template renders past assistant turns differently from the last one
|
||||
HIGGS_AUDIO_CHAT_TEMPLATE = (
|
||||
"{%- for message in messages -%}"
|
||||
"{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}"
|
||||
"{%- endfor -%}"
|
||||
"{%- if add_generation_prompt -%}"
|
||||
"{{- '<|im_start|>assistant\\n' -}}"
|
||||
"{%- if enable_thinking is defined and enable_thinking is false -%}"
|
||||
"{{- '<think>\\n\\n</think>\\n\\n' -}}"
|
||||
"{%- endif -%}"
|
||||
"{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
@ModelBase.register("HiggsAudio3Model")
|
||||
@ModelBase.example("bosonai/higgs-audio-v3-stt")
|
||||
class HiggsAudioModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
def set_vocab(self):
|
||||
super().set_vocab()
|
||||
self.gguf_writer.add_chat_template(HIGGS_AUDIO_CHAT_TEMPLATE)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
# the audio_* tensors are dropped by TextModel, rename the text head before that
|
||||
if name == "audio_decoder_proj.text_lm_head.weight":
|
||||
name = "lm_head.weight"
|
||||
else:
|
||||
name = "model." + name
|
||||
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
|
||||
@ModelBase.register("HiggsAudio3Model")
|
||||
class HiggsAudioWhisperEncoderModel(WhisperEncoderModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
return self.global_config["audio_encoder_config"]
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# depthwise conv kernel, same as the whisper convs
|
||||
if name == "audio_encoder_proj.temporal.weight":
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
assert self.global_config["projector_type"] == "mlp" and self.global_config["projector_temporal_downsample"] == 2
|
||||
assert self.global_config["audio_adapter_type"] == "stack" and not self.global_config.get("duplicate_audio_triplet", False)
|
||||
super().set_gguf_parameters()
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.HIGGS_AUDIO)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not name.startswith(("audio_tower.", "audio_encoder_proj.")):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
@@ -0,0 +1,331 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from functools import partial
|
||||
from typing import Callable, Iterable, TYPE_CHECKING, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import LazyTorchTensor, ModelBase, MmprojModel, TextModel, gguf, logger
|
||||
|
||||
|
||||
def _e4m3_table() -> np.ndarray:
|
||||
codes = np.arange(256, dtype=np.uint8)
|
||||
exponent = ((codes & 0x7F) >> 3).astype(np.int32)
|
||||
mantissa = (codes & 7).astype(np.float32)
|
||||
values = np.where(exponent == 0, np.ldexp(mantissa, -9), np.ldexp(1 + mantissa / 8, exponent - 7))
|
||||
values = np.where(codes & 0x80, -values, values)
|
||||
values[(codes & 0x7F) == 0x7F] = np.nan
|
||||
return values.astype(np.float32)
|
||||
|
||||
|
||||
_E4M3 = _e4m3_table()
|
||||
_E2M1 = np.array([0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6], dtype=np.float32)
|
||||
_EXPERT_CHUNK_ROWS = 128
|
||||
|
||||
|
||||
def _dequant_expert_chunk(weight: Callable[[], Tensor], scale: Callable[[], Tensor], amax: Callable[[], Tensor] | None, denominator: Callable[[], Tensor] | None, name: str, expert: int, rows: slice) -> np.ndarray:
|
||||
# Fresh lazy tensors release their mappings after each chunk instead of caching the layer.
|
||||
packed = LazyTorchTensor.to_eager(weight().view(torch.uint8)[expert, rows]).numpy()
|
||||
scales = LazyTorchTensor.to_eager(scale().view(torch.uint8)[expert, rows]).numpy()
|
||||
if amax is not None:
|
||||
block_scale = _E4M3[scales]
|
||||
if np.isnan(block_scale).any():
|
||||
raise ValueError(f"Inkling2: {name}_scale expert {expert}: E4M3FN NaN")
|
||||
values = np.empty((packed.shape[0], packed.shape[1] * 2), dtype=np.float32)
|
||||
values[:, 0::2] = _E2M1[packed & 15]
|
||||
values[:, 1::2] = _E2M1[packed >> 4]
|
||||
values.reshape(values.shape[0], -1, 16)[:] *= block_scale[:, :, None]
|
||||
assert denominator is not None
|
||||
a = LazyTorchTensor.to_eager(amax()[expert]).item()
|
||||
d = LazyTorchTensor.to_eager(denominator()).item()
|
||||
if not np.isfinite(a) or a < 0:
|
||||
raise ValueError(f"Inkling2: {name}_amax expert {expert}: expected finite nonnegative value")
|
||||
if not np.isfinite(d) or d <= 0:
|
||||
raise ValueError(f"Inkling2: {name}_outer_scale_denominator expert {expert}: expected finite positive value")
|
||||
with np.errstate(over="ignore", invalid="ignore"):
|
||||
values *= np.float32(a) / np.float32(d)
|
||||
else:
|
||||
if np.any(scales == 255):
|
||||
raise ValueError(f"Inkling2: {name}_scale expert {expert}: UE8M0 byte 255 is NaN")
|
||||
values = _E4M3[packed]
|
||||
if np.isnan(values).any():
|
||||
raise ValueError(f"Inkling2: {name} expert {expert}: E4M3FN NaN")
|
||||
block_scale = np.ldexp(np.ones(scales.shape, dtype=np.float32), scales.astype(np.int32) - 127)
|
||||
with np.errstate(over="ignore", invalid="ignore"):
|
||||
values.reshape(values.shape[0], -1, 32)[:] *= block_scale[:, :, None]
|
||||
if not np.isfinite(values).all():
|
||||
raise ValueError(f"Inkling2: {name} expert {expert}: non-finite dequantized weight")
|
||||
return values
|
||||
|
||||
|
||||
def _is_inkling2(dir_model: Path) -> bool:
|
||||
if not (dir_model / "config.json").is_file():
|
||||
return False
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
return json.load(f).get("model_type") in ("inkling2", "inkling2_text")
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_inkling2)
|
||||
def _load_inkling2_hparams(dir_model: Path):
|
||||
# AutoConfig injects MXFP8 defaults and can replace the original layer schedule.
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@ModelBase.register("Inkling2ForConditionalGeneration", "Inkling2ForCausalLM")
|
||||
@ModelBase.example("thinkingmachines/Inkling-2-Small")
|
||||
class Inkling2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.INKLING2
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
self._expert_quant: dict[str, dict] = {}
|
||||
self._packed_experts: dict[str, gguf.LazyChunkedTensor] = {}
|
||||
quant_path = dir_model / "hf_quant_config.json"
|
||||
if quant_path.is_file():
|
||||
with open(quant_path, encoding="utf-8") as f:
|
||||
groups = json.load(f).get("model", {}).get("parameter_groups", [])
|
||||
for group in groups:
|
||||
for name in group.get("tensors", []):
|
||||
item = self.filter_tensors((name, lambda: torch.empty(0)))
|
||||
if item is not None and item[0].endswith((".w13_weight", ".w2_weight")):
|
||||
name = item[0]
|
||||
if name in self._expert_quant:
|
||||
raise ValueError(f"Inkling2: duplicate hf_quant_config.json scheme for {name}")
|
||||
self._expert_quant[name] = group
|
||||
super().__init__(dir_model, *args, **kwargs)
|
||||
logger.warning("Inkling2: converting the text backbone only; vision and MTP tensors are skipped")
|
||||
hp = self.hparams
|
||||
for name, default in (("q_bias", False), ("o_bias", False), ("n_shared_experts", 0), ("use_value_norm_no_gamma", True), ("attn_res_normalize_values", True), ("scoring_func", "sigmoid")):
|
||||
if hp.get(name, default) != default:
|
||||
raise ValueError(f"Inkling2: unsupported {name}={hp[name]!r}")
|
||||
types = hp.get("layer_types")
|
||||
if types is None:
|
||||
local = hp["local_layer_ids"]
|
||||
types = ["hybrid_sliding" if i in local else "hybrid" for i in range(self.block_count)]
|
||||
if len(types) != self.block_count or set(types) - {"hybrid", "hybrid_sliding"}:
|
||||
raise ValueError("Inkling2: invalid layer_types")
|
||||
self.is_swa = [t == "hybrid_sliding" for t in types]
|
||||
|
||||
def dequant_model(self):
|
||||
for name in list(self.model_tensors):
|
||||
if not name.endswith((".w13_weight", ".w2_weight")):
|
||||
continue
|
||||
weight = self.model_tensors[name]
|
||||
tensor = weight()
|
||||
companions = [name + suffix for suffix in ("_scale", "_amax", "_outer_scale_denominator")]
|
||||
if tensor.dtype.is_floating_point and tensor.element_size() > 1:
|
||||
if any(n in self.model_tensors for n in companions):
|
||||
raise ValueError(f"Inkling2: floating-point {name} has packed quantization companions")
|
||||
continue
|
||||
config = self._expert_quant.get(name, {})
|
||||
scheme = config.get("weight_transform")
|
||||
if scheme not in ("nvfp4_nearest", "mxfp8_nearest"):
|
||||
raise ValueError(f"Inkling2: {name}: missing or unsupported hf_quant_config.json scheme {scheme!r}")
|
||||
nvfp4 = scheme == "nvfp4_nearest"
|
||||
if nvfp4 and config.get("weight_scale_format") != "amax_denominator":
|
||||
raise ValueError(f"Inkling2: {name}: expected NVFP4 amax_denominator scale format")
|
||||
required = companions if nvfp4 else companions[:1]
|
||||
for companion in required:
|
||||
if companion not in self.model_tensors:
|
||||
raise ValueError(f"Inkling2: {name}: missing companion {companion}")
|
||||
if not nvfp4 and any(n in self.model_tensors for n in companions[1:]):
|
||||
raise ValueError(f"Inkling2: {name}: MXFP8 must not have amax or outer_scale_denominator")
|
||||
scale = self.model_tensors[companions[0]]
|
||||
scale_tensor = scale()
|
||||
weight_dtype, scale_dtype = (torch.uint8, torch.float8_e4m3fn) if nvfp4 else (torch.float8_e4m3fn, torch.uint8)
|
||||
if tensor.dtype != weight_dtype or scale_tensor.dtype != scale_dtype:
|
||||
raise ValueError(f"Inkling2: {name}: expected {weight_dtype} weights and {scale_dtype} scales, got {tensor.dtype} and {scale_tensor.dtype}")
|
||||
experts = self.hparams["n_routed_experts"]
|
||||
hidden = self.hparams["hidden_size"]
|
||||
inter = self.find_hparam(["moe_intermediate_size", "intermediate_size"])
|
||||
w13 = name.endswith(".w13_weight")
|
||||
rows, cols = (2 * inter, hidden) if w13 else (hidden, inter)
|
||||
shape = (experts, rows, cols)
|
||||
if min(shape) <= 0 or cols % (16 if nvfp4 else 32) or tuple(tensor.shape) != (experts, rows, cols // (2 if nvfp4 else 1)):
|
||||
raise ValueError(f"Inkling2: {name}: invalid packed shape {tuple(tensor.shape)} for logical shape {shape}")
|
||||
ss = tuple(scale_tensor.shape)
|
||||
if len(ss) != 3 or ss[0] != experts or min(ss) <= 0 or rows % ss[1] or cols % ss[2]:
|
||||
raise ValueError(f"Inkling2: {name}_scale: shape {ss} does not divide logical shape {shape}")
|
||||
block = (rows // ss[1], cols // ss[2])
|
||||
if block != (1, 16 if nvfp4 else 32):
|
||||
raise ValueError(f"Inkling2: {name}_scale: unsupported stored block shape {block}")
|
||||
if list(block) != config.get("weight_block_shape"):
|
||||
logger.warning("Inkling2: %s: hf_quant_config.json weight_block_shape %s conflicts with stored weight/scale shapes %s/%s; using %s", name, config.get("weight_block_shape"), tuple(tensor.shape), ss, list(block))
|
||||
amax = denominator = None
|
||||
if nvfp4:
|
||||
amax, denominator = (self.model_tensors[n] for n in companions[1:])
|
||||
for companion, gen, expected in ((companions[1], amax, (experts,)), (companions[2], denominator, (1,))):
|
||||
t = gen()
|
||||
if t.dtype != torch.float32 or tuple(t.shape) != expected:
|
||||
raise ValueError(f"Inkling2: {companion}: expected F32 {expected}, got {t.dtype} {tuple(t.shape)}")
|
||||
chunks: list[Callable[[], np.ndarray]] = []
|
||||
for expert in range(experts):
|
||||
for half in range(2 if w13 else 1):
|
||||
for start in range(0, rows // 2 if w13 else rows, _EXPERT_CHUNK_ROWS):
|
||||
stop = min(start + _EXPERT_CHUNK_ROWS, rows // 2 if w13 else rows)
|
||||
# The same packed row slice selects the weight and its scale groups.
|
||||
row_slice = slice(2 * start + half, 2 * stop + half, 2) if w13 else slice(start, stop)
|
||||
chunks.append(partial(_dequant_expert_chunk, weight, scale, amax, denominator, name, expert, row_slice))
|
||||
self._packed_experts[name] = gguf.LazyChunkedTensor(chunks, shape, np.float32)
|
||||
self.model_tensors[name] = lambda shape=shape: torch.empty(shape, dtype=torch.float32, device="meta")
|
||||
for companion in required:
|
||||
del self.model_tensors[companion]
|
||||
for name, gen in self.model_tensors.items():
|
||||
t = gen()
|
||||
if not t.dtype.is_floating_point or t.element_size() == 1 or name.endswith(("_weight_scale", "_weight_amax", "_weight_outer_scale_denominator")):
|
||||
raise ValueError(f"Inkling2: unexpected packed tensor or orphan companion {name}")
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
if any(p in name for p in ("visual.", "vision_tower.", "mtp.", "markov_head", "nextn_predict")):
|
||||
return None
|
||||
name = name.replace("model.language_model.", "model.").replace("model.llm.", "model.")
|
||||
# the mapping table cannot shift a block index: attn_queries.{i} feeds block i + 1
|
||||
name = re.sub(r"attn_res\.attn_queries\.(\d+)", lambda m: f"layers.{int(m[1]) + 1}.attention_input.query", name)
|
||||
# the table maps a module, not its bias: gate.weight is the router, gate.bias the score correction
|
||||
name = name.replace("mlp.gate.bias", "mlp.gate.e_score_correction_bias")
|
||||
return name, gen
|
||||
|
||||
def get_vocab_base(self):
|
||||
tokens, types, pre = super().get_vocab_base()
|
||||
n = self.hparams["unpadded_vocab_size"]
|
||||
return tokens[:n], types[:n], pre
|
||||
|
||||
def set_vocab(self):
|
||||
tokens, types, pre = self.get_vocab_base()
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre(pre)
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(types)
|
||||
special = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
eos = self.hparams["eos_token_id"]
|
||||
special.special_token_ids["eos"] = eos
|
||||
special.special_token_ids["eot"] = eos
|
||||
special.add_special_token["bos"] = False
|
||||
special.add_to_gguf(self.gguf_writer)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
hp["num_attention_heads"] = [hp.get("swa_num_attention_heads", hp["num_attention_heads"]) if s else hp["num_attention_heads"] for s in self.is_swa]
|
||||
hp["num_key_value_heads"] = [hp.get("swa_num_key_value_heads", hp["num_key_value_heads"]) if s else hp["num_key_value_heads"] for s in self.is_swa]
|
||||
for name in ("n_routed_experts", "num_experts_per_tok"):
|
||||
if name not in hp:
|
||||
raise KeyError(f"Inkling2: missing {name}")
|
||||
super().set_gguf_parameters()
|
||||
w = self.gguf_writer
|
||||
w.add_vocab_size(hp["vocab_size"])
|
||||
w.add_vocab_size_unpadded(hp["unpadded_vocab_size"])
|
||||
w.add_expert_feed_forward_length(self.find_hparam(["moe_intermediate_size", "intermediate_size"]))
|
||||
w.add_key_length_swa(hp.get("swa_head_dim", hp["head_dim"]))
|
||||
w.add_value_length_swa(hp.get("swa_head_dim", hp["head_dim"]))
|
||||
w.add_rope_dimension_count(0)
|
||||
w.add_sliding_window(hp["sliding_window_size"])
|
||||
w.add_sliding_window_pattern(self.is_swa)
|
||||
w.add_shortconv_l_cache(self.find_hparam(["conv_kernel_size", "sconv_kernel_size"]))
|
||||
w.add_expert_weights_scale(hp["route_scale"])
|
||||
w.add_activation_situ_beta(hp["mlp_gate_softcap"])
|
||||
w.add_activation_situ_linear_beta(hp["mlp_up_softcap"])
|
||||
w.add_attn_res_block_size(hp["n_layers_per_attn_res_block"])
|
||||
w.add_final_logit_softcapping(hp["final_logit_softcapping"])
|
||||
w.add_relative_length(hp["d_rel"])
|
||||
w.add_relative_extent(hp["rel_extent"])
|
||||
w.add_qk_clip(hp["qk_l2_clip_threshold"])
|
||||
w.add_relative_clip(hp["rel_l2_clip_threshold"])
|
||||
w.add_log_scale_floor(hp["log_scaling_n_floor"])
|
||||
w.add_log_scale_alpha(hp["log_scaling_alpha"])
|
||||
|
||||
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
|
||||
if new_name.endswith(("_conv.weight", "_clip.weight", "_res_query.weight", "attn_rel_bias.weight")):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name in self._packed_experts:
|
||||
target = name.replace(".w13_weight", ".gate_up_proj").replace(".w2_weight", ".down_proj")
|
||||
yield self.map_tensor_name(target, try_suffixes=()) + ".weight", cast("Tensor", self._packed_experts[name])
|
||||
return
|
||||
if name.endswith(".w13_weight"):
|
||||
shape = data_torch.shape
|
||||
data_torch = data_torch.reshape(shape[0], shape[1] // 2, 2, shape[2]).transpose(1, 2).reshape(shape)
|
||||
name = name.replace(".w13_weight", ".gate_up_proj")
|
||||
name = name.replace(".w2_weight", ".down_proj")
|
||||
for static, conv in ((".static_weight", ".conv1d.weight"), (".static_w", ".weight")):
|
||||
if name.endswith(static):
|
||||
data_torch = data_torch.flip(0).T.contiguous()
|
||||
name = name.removesuffix(static) + conv
|
||||
if name.endswith((".conv1d.weight", "sconv.weight")):
|
||||
data_torch = data_torch.squeeze(1) if data_torch.ndim == 3 else data_torch
|
||||
elif name.endswith(".rel_logits_proj.proj"):
|
||||
data_torch = data_torch.T.contiguous()
|
||||
# a name the table maps without a suffix is a bare parameter
|
||||
new_name = self.tensor_map.get_name(name)
|
||||
if new_name is not None:
|
||||
suffix = ".bias" if name.endswith(".e_score_correction_bias") else ".weight"
|
||||
yield new_name + suffix, data_torch
|
||||
else:
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("Inkling2ForConditionalGeneration")
|
||||
class Inkling2VisionModel(MmprojModel):
|
||||
def get_vision_config(self):
|
||||
config = self.global_config.get("vision_config")
|
||||
if not config:
|
||||
return None
|
||||
config = dict(config)
|
||||
config["patch_size"] = config.get("native_patch_size", config.get("patch_size", 20))
|
||||
config["spatial_merge_size"] = config.get("adapter_merge_size", config.get("spatial_merge_size", 2))
|
||||
config["intermediate_size"] = config.get("intermediate_size") or int(config["hidden_size"] * config.get("mlp_ratio", 4.0))
|
||||
config["image_size"] = config["posemb_grid_size"] * config["patch_size"]
|
||||
if config.get("num_channels", 3) != 3 or config.get("hidden_act", "gelu") != "gelu":
|
||||
raise ValueError("Inkling2 vision requires RGB patches and exact GELU")
|
||||
return config
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
self.preprocessor_config.setdefault("image_mean", [0.5, 0.5, 0.5])
|
||||
self.preprocessor_config.setdefault("image_std", [0.5, 0.5, 0.5])
|
||||
super().set_gguf_parameters()
|
||||
config = self.hparams
|
||||
rope = config.get("rope_parameters") or {"rope_type": "axial", "rope_theta": 10000.0}
|
||||
if rope["rope_type"] != "axial" or rope["rope_theta"] != 10000.0:
|
||||
raise ValueError("Inkling2 vision requires axial RoPE with theta 10000")
|
||||
w = self.gguf_writer
|
||||
w.add_clip_vision_projector_type(gguf.VisionProjectorType.INKLING2V)
|
||||
w.add_vision_attention_layernorm_eps(config["layer_norm_eps"])
|
||||
w.add_vision_spatial_merge_size(config["spatial_merge_size"])
|
||||
|
||||
def dequant_model(self):
|
||||
# vision tensors are BF16, only the text experts are quantized
|
||||
return
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
name, gen = item
|
||||
if not name.startswith(("model.visual.", "model.vision_tower.")):
|
||||
return None
|
||||
return name, gen
|
||||
|
||||
# mm.{bid} takes the projector layer index, the mapping table cannot derive it from these names
|
||||
_projector_layers = {"model.visual.proj_in": 1, "model.visual.proj_out": 2, "model.vision_tower.merger.linear_1": 1, "model.vision_tower.merger.linear_2": 2}
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
module, _, suffix = name.rpartition(".")
|
||||
if module in self._projector_layers:
|
||||
yield self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, self._projector_layers[module], "." + suffix), data_torch
|
||||
return
|
||||
# lay out the linear patch embedding (HWC patches) as a conv2d for build_inp()
|
||||
if name.endswith(".patch_embedding.weight"):
|
||||
ps = self.hparams["patch_size"]
|
||||
data_torch = data_torch.reshape(data_torch.shape[0], ps, ps, 3).permute(0, 3, 1, 2)
|
||||
# these are bare parameters, they have no .weight
|
||||
if name.endswith(("position_embedding", ".lambda_1", ".lambda_2", ".gamma")):
|
||||
name = name.removesuffix(".gamma") + ".weight"
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
@@ -0,0 +1,419 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable
|
||||
|
||||
import torch
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, MmprojModel, gguf, logger
|
||||
|
||||
# Kokoro (StyleTTS2 with an iSTFTNet decoder), https://huggingface.co/hexgrad/Kokoro-82M
|
||||
# text model: the PL-BERT phoneme encoder (ALBERT), one token per phoneme character
|
||||
# mmproj: bert_encoder, prosody predictor, text encoder, decoder and the voices
|
||||
# the text to phonemes step is the "tts" chat template, its dictionaries come from the misaki and nltk python packages
|
||||
|
||||
DEFAULT_VOICE = "af_heart"
|
||||
|
||||
MODULE_PREFIX = {
|
||||
"bert_encoder": "a.gen.bert_enc.",
|
||||
"predictor": "a.gen.pred.",
|
||||
"text_encoder": "a.gen.text.",
|
||||
"decoder": "a.gen.dec.",
|
||||
}
|
||||
|
||||
|
||||
def _checkpoint(dir_model: Path) -> Path | None:
|
||||
paths = sorted(dir_model.glob("kokoro-*.pth"))
|
||||
return paths[0] if paths else None
|
||||
|
||||
|
||||
def _is_kokoro_checkpoint(dir_model: Path) -> bool:
|
||||
if _checkpoint(dir_model) is None or not (dir_model / "config.json").is_file():
|
||||
return False
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
return "plbert" in config and "istftnet" in config and "vocab" in config
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_kokoro_checkpoint)
|
||||
def _load_kokoro_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected Kokoro checkpoint")
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
plbert = config["plbert"]
|
||||
return {
|
||||
"architectures": ["KokoroModel"],
|
||||
"model_type": "kokoro",
|
||||
"num_hidden_layers": plbert["num_hidden_layers"],
|
||||
"hidden_size": plbert["hidden_size"],
|
||||
"intermediate_size": plbert["intermediate_size"],
|
||||
"num_attention_heads": plbert["num_attention_heads"],
|
||||
"max_position_embeddings": plbert["max_position_embeddings"],
|
||||
"vocab_size": config["n_token"],
|
||||
"layer_norm_eps": 1e-12, # AlbertConfig default
|
||||
"kokoro": config,
|
||||
# the mmproj tensors are named explicitly, they do not go through the block tensor map
|
||||
"audio_config": {"num_hidden_layers": 0},
|
||||
}
|
||||
|
||||
|
||||
def _load_modules(dir_model: Path, modules: Iterable[str]) -> dict[str, dict[str, Tensor]]:
|
||||
path = _checkpoint(dir_model)
|
||||
assert path is not None
|
||||
state = torch.load(path, map_location="cpu", mmap=True, weights_only=True)
|
||||
# every module was saved from a DataParallel wrapper
|
||||
return {m: {k.removeprefix("module."): v for k, v in state[m].items()} for m in modules}
|
||||
|
||||
|
||||
def _g2p_fallback(lexicons: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
# model for the words missing from the lexicon: n-grams over units "letter:sounds", learned from the lexicon
|
||||
# key "<0 to 4 units before> <letter>" -> {sounds: probability}; "<s>" pads the start of a word, the letter "$" is its end
|
||||
# the "tts" template decodes it with a beam search
|
||||
order, min_count, stress = 5, 3, "\u02c8\u02cc"
|
||||
vowels = set("AIOQWYaiu\u00e6\u0251\u0252\u0254\u0259\u025b\u025c\u026a\u028a\u028c\u1d7b\u1d4a")
|
||||
|
||||
def sounds(ps: str) -> list[str]:
|
||||
# a stress mark goes with the sound after it, a length mark with the one before
|
||||
out: list[str] = []
|
||||
pre = ""
|
||||
for c in ps:
|
||||
if c in stress:
|
||||
pre += c
|
||||
elif c == "\u02d0" and out:
|
||||
out[-1] += c
|
||||
else:
|
||||
out.append(pre + c)
|
||||
pre = ""
|
||||
return out
|
||||
|
||||
def penalty(letter: str, chunk: str) -> float:
|
||||
# a consonant letter says consonants only; a vowel letter says a vowel, with at most a glide
|
||||
flat = [c for c in chunk if c not in stress and c != "\u02d0"]
|
||||
n_vowel = sum(c in vowels for c in flat)
|
||||
if letter in "bcdfgjkmnpqstvxz" and n_vowel > 0:
|
||||
return -8.0
|
||||
if letter in "aeiou" and flat and (n_vowel == 0 or any(c not in vowels and c not in "jw" for c in flat)):
|
||||
return -8.0
|
||||
return 0.0
|
||||
|
||||
words: dict[str, list[str]] = {}
|
||||
for lexicon in lexicons:
|
||||
for word, v in lexicon.items():
|
||||
ps = v["DEFAULT"] if isinstance(v, dict) else v
|
||||
# same spelling normalization as the template: doubled consonants and "ck" say one sound
|
||||
w = re.sub(r"([b-df-hj-np-tv-z])\1", r"\1", word.lower()).replace("ck", "k")
|
||||
if ps and " " not in ps and len(w) >= 2 and re.fullmatch(r"[a-z']+", w) and w not in words:
|
||||
t = sounds(ps)
|
||||
if len(t) <= 2 * len(w):
|
||||
words[w] = t
|
||||
|
||||
def align(w: str, t: list[str], cost: dict[str, dict[str, float]]) -> list[str] | None:
|
||||
# best split of the sounds over the letters, each letter takes 0, 1 or 2 of them
|
||||
n_l, n_t = len(w), len(t)
|
||||
two = [t[j] + t[j + 1] for j in range(n_t - 1)]
|
||||
best = [[-1e18] * (n_t + 1) for _ in range(n_l + 1)]
|
||||
back = [[0] * (n_t + 1) for _ in range(n_l + 1)]
|
||||
best[0][0] = 0.0
|
||||
for i in range(n_l):
|
||||
letter = w[i]
|
||||
row = cost.setdefault(letter, {})
|
||||
cur, nxt, bk = best[i], best[i + 1], back[i + 1]
|
||||
# the sounds left must fit in the letters left
|
||||
for j in range(max(0, n_t - 2 * (n_l - i)), n_t + 1):
|
||||
base = cur[j]
|
||||
if base < -1e17:
|
||||
continue
|
||||
for k, chunk in ((0, ""), (1, t[j] if j < n_t else None), (2, two[j] if j + 1 < n_t else None)):
|
||||
if chunk is None:
|
||||
break
|
||||
c = row.get(chunk)
|
||||
if c is None:
|
||||
# not seen yet for this letter
|
||||
c = row[chunk] = -12.0 + penalty(letter, chunk)
|
||||
if base + c > nxt[j + k]:
|
||||
nxt[j + k] = base + c
|
||||
bk[j + k] = k
|
||||
if best[n_l][n_t] < -1e17:
|
||||
return None
|
||||
out: list[str] = []
|
||||
j = n_t
|
||||
for i in range(n_l, 0, -1):
|
||||
k = back[i][j]
|
||||
out.append("".join(t[j - k:j]))
|
||||
j -= k
|
||||
return out[::-1]
|
||||
|
||||
# hard EM: align with the current letter -> sounds costs, count, repeat
|
||||
# the costs are learned on a sample of the words, then every word is aligned once
|
||||
def step(items: Iterable[tuple[str, list[str]]], cost: dict[str, dict[str, float]]) -> list[tuple[str, list[str]]]:
|
||||
return [(w, a) for w, a in ((w, align(w, t, cost)) for w, t in items) if a is not None]
|
||||
|
||||
sample = list(words.items())[::max(1, len(words) // 20000)]
|
||||
cost: dict[str, dict[str, float]] = {}
|
||||
for _ in range(5):
|
||||
counts: dict[str, Counter[str]] = defaultdict(Counter)
|
||||
for w, a in step(sample, cost):
|
||||
for c, o in zip(w, a):
|
||||
counts[c][o] += 1
|
||||
# small cost for a silent letter, so that a sound stays on its own letter
|
||||
cost = {c: {o: math.log(n / sum(cnt.values())) - (0.5 if o == "" else 0.0) + penalty(c, o) for o, n in cnt.items()}
|
||||
for c, cnt in counts.items()}
|
||||
aligned = step(words.items(), cost)
|
||||
|
||||
# counts of a unit after each of its contexts, from 4 units down to none
|
||||
ngram: dict[str, Counter[str]] = defaultdict(Counter)
|
||||
n_ctx: Counter[str] = Counter()
|
||||
for w, a in aligned:
|
||||
units = ["<s>"] * (order - 1) + [c + ":" + o for c, o in zip(w, a)] + ["$:"]
|
||||
for i in range(order - 1, len(units)):
|
||||
letter, _, o = units[i].partition(":")
|
||||
for n in range(order):
|
||||
ctx = " ".join(units[i - n:i])
|
||||
ngram[(ctx + " " if ctx else "") + letter][o] += 1
|
||||
n_ctx[ctx] += 1
|
||||
|
||||
# the probability is over all the units after the context, not only the ones of this letter
|
||||
# this tells the search how likely the letter is after the sounds it chose, so it can drop a bad choice
|
||||
model: dict[str, Any] = {}
|
||||
for key, cnt in ngram.items():
|
||||
ctx, _, letter = key.rpartition(" ")
|
||||
# the two shortest contexts keep all the readings that the letter has min_count times or more
|
||||
kept = {o: float(f"{n / n_ctx[ctx]:.4g}") for o, n in cnt.most_common()
|
||||
if n >= min_count or (key.count(" ") < 2 and ngram[letter][o] >= min_count)}
|
||||
if kept:
|
||||
model[key] = kept
|
||||
return model
|
||||
|
||||
|
||||
def _g2p_dicts() -> dict[str, dict[str, Any]] | None:
|
||||
# dictionaries of the "tts" template: the misaki lexicon and the nltk averaged perceptron tagger
|
||||
spec = importlib.util.find_spec("misaki")
|
||||
if spec is None or spec.origin is None:
|
||||
logger.warning("misaki is not installed, the model will take phonemes as prompt (pip install misaki nltk)")
|
||||
return None
|
||||
data = Path(spec.origin).parent / "data"
|
||||
try:
|
||||
nltk = importlib.import_module("nltk")
|
||||
base = Path(str(nltk.data.find("taggers/averaged_perceptron_tagger_eng"))) / "averaged_perceptron_tagger_eng"
|
||||
except (ImportError, LookupError):
|
||||
logger.warning("the nltk tagger is not available, the model will take phonemes as prompt "
|
||||
"(pip install nltk && python -m nltk.downloader averaged_perceptron_tagger_eng)")
|
||||
return None
|
||||
|
||||
def load(path: Path) -> Any:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
# tagger: classes sorted by label, feature -> {"c<class index>": weight}
|
||||
# (index keys: jinja2's map(attribute=...) splits on "." and some labels are punctuation)
|
||||
classes = sorted(load(Path(f"{base}.classes.json")))
|
||||
cid = {c: f"c{i}" for i, c in enumerate(classes)}
|
||||
weights = load(Path(f"{base}.weights.json"))
|
||||
lex = {
|
||||
"gold": load(data / "us_gold.json"),
|
||||
"silver": load(data / "us_silver.json"),
|
||||
"gold_gb": load(data / "gb_gold.json"),
|
||||
"silver_gb": load(data / "gb_silver.json"),
|
||||
}
|
||||
logger.info("gguf: learning the pronunciation of unknown words from the lexicon, this takes a few minutes")
|
||||
# one process per variant
|
||||
with ProcessPoolExecutor(max_workers=2) as pool:
|
||||
lts, lts_gb = pool.map(_g2p_fallback, [[lex["gold"], lex["silver"]], [lex["gold_gb"], lex["silver_gb"]]])
|
||||
return {
|
||||
**lex,
|
||||
"lts": lts,
|
||||
"lts_gb": lts_gb,
|
||||
"pos_weights": {f: {cid[t]: w for t, w in tw.items()} for f, tw in weights.items()},
|
||||
"pos_tagdict": load(Path(f"{base}.tagdict.json")),
|
||||
"pos_classes": {"": classes},
|
||||
}
|
||||
|
||||
|
||||
@ModelBase.register("KokoroModel")
|
||||
@ModelBase.example("hexgrad/Kokoro-82M")
|
||||
class KokoroTextModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.KOKORO
|
||||
|
||||
_LAYER = "encoder.albert_layer_groups.0.albert_layers.0."
|
||||
_TENSOR_MAP = {
|
||||
"embeddings.LayerNorm": gguf.MODEL_TENSOR.TOKEN_EMBD_NORM,
|
||||
"encoder.embedding_hidden_mapping_in": gguf.MODEL_TENSOR.TOKEN_EMBD_PROJ,
|
||||
}
|
||||
# the layers share their weights, stored as block 0
|
||||
_LAYER_TENSOR_MAP = {
|
||||
"attention.query": gguf.MODEL_TENSOR.ATTN_Q,
|
||||
"attention.key": gguf.MODEL_TENSOR.ATTN_K,
|
||||
"attention.value": gguf.MODEL_TENSOR.ATTN_V,
|
||||
"attention.dense": gguf.MODEL_TENSOR.ATTN_OUT,
|
||||
"attention.LayerNorm": gguf.MODEL_TENSOR.ATTN_OUT_NORM,
|
||||
"ffn": gguf.MODEL_TENSOR.FFN_UP,
|
||||
"ffn_output": gguf.MODEL_TENSOR.FFN_DOWN,
|
||||
"full_layer_layer_norm": gguf.MODEL_TENSOR.LAYER_OUT_NORM,
|
||||
}
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
del remote_hf_model_id
|
||||
bert = _load_modules(self.dir_model, ["bert"])["bert"]
|
||||
return {name: (lambda t=t: t) for name, t in bert.items()}
|
||||
|
||||
def set_vocab(self):
|
||||
config = self.hparams["kokoro"]
|
||||
n_vocab = config["n_token"]
|
||||
tokens: list[str] = [f"[UNUSED_{i}]" for i in range(n_vocab)]
|
||||
toktypes = [gguf.TokenType.UNUSED] * n_vocab
|
||||
for text, i in config["vocab"].items():
|
||||
tokens[i] = text
|
||||
toktypes[i] = gguf.TokenType.NORMAL
|
||||
# id 0 pads both ends of the phoneme sequence
|
||||
tokens[0] = "<pad>"
|
||||
toktypes[0] = gguf.TokenType.CONTROL
|
||||
|
||||
# the phonemes are mapped to tokens one character at a time by the audio pipeline
|
||||
self.gguf_writer.add_tokenizer_model("llama")
|
||||
self.gguf_writer.add_tokenizer_pre("default")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores([0.0] * n_vocab)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
self.gguf_writer.add_bos_token_id(0)
|
||||
self.gguf_writer.add_eos_token_id(0)
|
||||
self.gguf_writer.add_pad_token_id(0)
|
||||
self.gguf_writer.add_add_bos_token(False)
|
||||
self.gguf_writer.add_add_eos_token(False)
|
||||
|
||||
dicts = _g2p_dicts()
|
||||
if dicts is not None:
|
||||
template_path = Path(__file__).parent.parent / "models" / "templates" / "hexgrad-Kokoro-82M-tts.jinja"
|
||||
with open(template_path, encoding="utf-8") as f:
|
||||
self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}])
|
||||
self.gguf_writer.add_tts_dicts(dicts)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
self.gguf_writer.add_causal_attention(False)
|
||||
self.gguf_writer.add_pooling_type(gguf.PoolingType.NONE)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
del bid
|
||||
n_embd = self.hparams["hidden_size"]
|
||||
|
||||
if name == "embeddings.word_embeddings.weight":
|
||||
# zero-pad the rows to n_embd, llama.cpp wants token and embedding inputs of the same width
|
||||
data_torch = torch.nn.functional.pad(data_torch, (0, n_embd - data_torch.shape[1]))
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), data_torch)
|
||||
return
|
||||
if name == "embeddings.position_embeddings.weight":
|
||||
# every token has type 0, fold its embedding into the positions
|
||||
type0 = self.model_tensors["embeddings.token_type_embeddings.weight"]()[0]
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.POS_EMBD), data_torch + type0)
|
||||
return
|
||||
if name.startswith(("embeddings.token_type_embeddings.", "pooler.")):
|
||||
return
|
||||
|
||||
key, suffix = name.rsplit(".", 1)
|
||||
if key in self._TENSOR_MAP:
|
||||
yield (self.format_tensor_name(self._TENSOR_MAP[key], suffix="." + suffix), data_torch)
|
||||
return
|
||||
if key.startswith(self._LAYER):
|
||||
tensor = self._LAYER_TENSOR_MAP[key.removeprefix(self._LAYER)]
|
||||
yield (self.format_tensor_name(tensor, 0, suffix="." + suffix), data_torch)
|
||||
return
|
||||
raise ValueError(f"unexpected tensor {name}")
|
||||
|
||||
|
||||
@ModelBase.register("KokoroModel")
|
||||
@ModelBase.example("hexgrad/Kokoro-82M")
|
||||
class KokoroMmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._wnorm_g: dict[str, Tensor] = {}
|
||||
self._wnorm_v: dict[str, Tensor] = {}
|
||||
self._lstm_bias: dict[str, Tensor] = {}
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
return self.global_config.get("audio_config")
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
del remote_hf_model_id
|
||||
tensors: dict[str, Callable[[], Tensor]] = {}
|
||||
for module, state in _load_modules(self.dir_model, MODULE_PREFIX.keys()).items():
|
||||
for name, t in state.items():
|
||||
tensors[MODULE_PREFIX[module] + name] = (lambda t=t: t)
|
||||
return tensors
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
# the names are final already
|
||||
return item
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
config = self.global_config["kokoro"]
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.KOKORO)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(config["hidden_dim"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(config["hidden_dim"])
|
||||
self.gguf_writer.add_gen_audio_block_count(config["n_layer"])
|
||||
self.gguf_writer.add_gen_audio_head_count(1)
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5)
|
||||
if (self.dir_model / "voices" / f"{DEFAULT_VOICE}.pt").is_file():
|
||||
self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
del bid
|
||||
# torch weight_norm: weight = g * v / |v| over dims 1..n
|
||||
if name.endswith((".weight_g", ".weight_v")):
|
||||
base = name[:-len(".weight_g")]
|
||||
(self._wnorm_g if name.endswith("_g") else self._wnorm_v)[base] = data_torch
|
||||
if base in self._wnorm_g and base in self._wnorm_v:
|
||||
g = self._wnorm_g.pop(base).float()
|
||||
v = self._wnorm_v.pop(base).float()
|
||||
w = g * v / v.norm(dim=tuple(range(1, v.dim())), keepdim=True)
|
||||
if base.endswith(".pool"):
|
||||
# depthwise transposed conv [C, 1, 3], stored as one row per tap
|
||||
w = w.squeeze(1).T.contiguous()
|
||||
yield (base + ".weight", w)
|
||||
return
|
||||
# the two LSTM biases are always added
|
||||
if ".bias_ih_l0" in name or ".bias_hh_l0" in name:
|
||||
base = name.replace(".bias_ih_l0", ".bias_l0").replace(".bias_hh_l0", ".bias_l0")
|
||||
if base in self._lstm_bias:
|
||||
yield (base, self._lstm_bias.pop(base).float() + data_torch.float())
|
||||
else:
|
||||
self._lstm_bias[base] = data_torch
|
||||
return
|
||||
# snake alphas are per channel
|
||||
if ".alpha1." in name or ".alpha2." in name:
|
||||
data_torch = data_torch.reshape(-1)
|
||||
yield (name, data_torch)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
del name, bid
|
||||
# tensors read on the host stay F32
|
||||
if "lstm" in new_name or ".shared." in new_name or new_name.startswith(("a.gen.bert_enc.", "voice.")) \
|
||||
or ".duration_proj." in new_name or ".m_source." in new_name or ".pool." in new_name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# CONV_TRANSPOSE_1D takes only F32 weights on some backends
|
||||
if ".ups." in new_name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# the conv ops have no BF16 kernels
|
||||
if n_dims >= 2:
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# a voice is a table of style vectors, the row is picked by the number of phonemes
|
||||
for path in sorted((self.dir_model / "voices").glob("*.pt")):
|
||||
pack = torch.load(path, map_location="cpu", weights_only=True)
|
||||
yield (f"voice.{path.stem}", pack.squeeze(1).float())
|
||||
+54
-1
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from typing import Iterable, Sequence, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
@@ -8,6 +10,7 @@ if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, MmprojModel, gguf, logger
|
||||
from .qwen import DSparkModel
|
||||
|
||||
|
||||
@ModelBase.register("MiniMaxText01ForCausalLM")
|
||||
@@ -170,6 +173,15 @@ class MiniMaxM2Model(TextModel):
|
||||
class MiniMaxM3Model(MiniMaxM2Model):
|
||||
model_arch = gguf.MODEL_ARCH.MINIMAXM3
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
# transformers 5.x folds sparse_attention_config and moe_layer_freq into other keys
|
||||
with open(self.dir_model / "config.json", "r", encoding="utf-8") as f:
|
||||
raw_hparams = json.load(f)
|
||||
for key, value in raw_hparams.get("text_config", raw_hparams).items():
|
||||
self.hparams.setdefault(key, value)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
if ".indexer." in new_name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
@@ -188,6 +200,7 @@ class MiniMaxM3Model(MiniMaxM2Model):
|
||||
self.gguf_writer.add_indexer_top_k(sac["sparse_topk_blocks"])
|
||||
self.gguf_writer.add_indexer_block_size(sac["sparse_block_size"])
|
||||
self.gguf_writer.add_indexer_local_blocks(sac["sparse_local_block"])
|
||||
self.gguf_writer.add_indexer_types([bool(v) for v in sac["sparse_attention_freq"]])
|
||||
|
||||
moe_layer_freq = self.find_hparam(["moe_layer_freq"])
|
||||
n_dense = 0
|
||||
@@ -201,7 +214,47 @@ class MiniMaxM3Model(MiniMaxM2Model):
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
|
||||
# Gemma-style (1 + w) RMSNorm: bake the +1 in so llama.cpp can use plain RMSNorm
|
||||
if name.endswith("norm.weight"):
|
||||
data_torch = data_torch + 1.0
|
||||
data_torch = data_torch.float() + 1.0
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("DSparkMiniMaxDraftModel")
|
||||
class MiniMaxM3DSparkModel(DSparkModel):
|
||||
model_arch = gguf.MODEL_ARCH.DFLASH
|
||||
|
||||
def __init__(self, dir_model, *args, **kwargs):
|
||||
# transformers drops the draft keys when it parses the minimax_m3_vl config
|
||||
with open(dir_model / "config.json", "r", encoding="utf-8") as f:
|
||||
kwargs["hparams"] = json.load(f)
|
||||
super().__init__(dir_model, *args, **kwargs)
|
||||
|
||||
# draft layers use a dense FFN, the MoE keys are copied from the target config
|
||||
self.hparams["intermediate_size"] = self.hparams["dspark_ffn_hidden_size"]
|
||||
for key in ("num_local_experts", "num_experts_per_tok", "scoring_func"):
|
||||
self.hparams.pop(key, None)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
|
||||
self.gguf_writer.add_hidden_act(self.hparams["hidden_act"])
|
||||
|
||||
layer_types = self.hparams["layer_types"]
|
||||
self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
|
||||
self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in layer_types])
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
name, gen = item
|
||||
name = name.removeprefix("language_model.model.dspark.").replace(".decoder_layer.", ".")
|
||||
if name == "final_norm.weight":
|
||||
name = "norm.weight"
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
|
||||
# Gemma-style (1 + w) RMSNorm, same as the target
|
||||
if name.endswith("norm.weight"):
|
||||
data_torch = data_torch.float() + 1.0
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, MmprojModel, gguf
|
||||
from .drama3 import fold_snake
|
||||
from .qwen import Qwen3Model
|
||||
|
||||
# MiniMax Music 3, diffusers layout: one checkpoint per component
|
||||
# text model: the global LLM (language_model/), samples the semantic code of each frame
|
||||
# mmproj: RVQ depth decoder, condition encoder, flow-matching DiT and DAC decoder (vocoder)
|
||||
|
||||
LM_DIR = "language_model"
|
||||
TOK_DIR = "tokenizer"
|
||||
MMPROJ_DIRS = ("rvq_depth_decoder", "condition_encoder", "transformer", "vocoder")
|
||||
|
||||
# token ids of the checkpoint, see encoders.py of the diffusers pipeline
|
||||
AUDIO_START = 151669
|
||||
AUDIO_END = 151670
|
||||
AUDIO_CODE_OFFSET = 151675
|
||||
N_SEMANTIC = 16384
|
||||
|
||||
|
||||
def _load_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
return ModelBase.load_hparams(dir_model / LM_DIR, is_mistral_format=False)
|
||||
|
||||
|
||||
def _index_dirs(model: ModelBase, dirs: Iterable[str], keep: Callable[[str], bool]) -> dict[str, Callable[[], Tensor]]:
|
||||
# the names get the component dir as prefix
|
||||
root = model.dir_model
|
||||
tensors: dict[str, Callable[[], Tensor]] = {}
|
||||
try:
|
||||
for d in dirs:
|
||||
model.dir_model = root / d
|
||||
prefix = "model" if d == LM_DIR else "diffusion_pytorch_model"
|
||||
for name, gen in model.index_local_tensors(prefix).items():
|
||||
if keep(f"{d}.{name}"):
|
||||
tensors[f"{d}.{name}"] = gen
|
||||
finally:
|
||||
model.dir_model = root
|
||||
return tensors
|
||||
|
||||
|
||||
def _fold_wn(g: Tensor, v: Tensor) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = g.float(), v.float()
|
||||
return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
|
||||
|
||||
|
||||
@ModelBase.register("MiniMaxMusic3ForConditionalGeneration")
|
||||
class MiniMaxMusic3TextModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
tensors = _index_dirs(self, [LM_DIR], lambda _: True)
|
||||
return {name.removeprefix(LM_DIR + "."): gen for name, gen in tensors.items()}
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
# the codes are sampled with classifier-free guidance, the uncond prompt runs as a second sequence
|
||||
self.gguf_writer.add_classifier_free_guidance(True)
|
||||
|
||||
def set_vocab(self):
|
||||
root = self.dir_model
|
||||
self.dir_model = root / TOK_DIR
|
||||
try:
|
||||
tokens, toktypes, tokpre = self.get_vocab_base()
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
finally:
|
||||
self.dir_model = root
|
||||
|
||||
# the semantic codes are not in the tokenizer, name them for debugging
|
||||
for i in range(N_SEMANTIC):
|
||||
tokens[AUDIO_CODE_OFFSET + i] = f"<|audio_code_{i}|>"
|
||||
toktypes[AUDIO_CODE_OFFSET + i] = gguf.TokenType.CONTROL
|
||||
|
||||
# the generation ends with <|audio_end|>, the chat template of the tokenizer is not used
|
||||
template_path = Path(__file__).parent.parent / "models" / "templates" / "MiniMaxAI-MiniMax-Music3-tts.jinja"
|
||||
with open(template_path, encoding="utf-8") as f:
|
||||
special_vocab.chat_template = [{"name": "tts", "template": f.read()}]
|
||||
special_vocab.special_token_ids["eos"] = AUDIO_END
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre(tokpre)
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
|
||||
@ModelBase.register("MiniMaxMusic3ForConditionalGeneration")
|
||||
class MiniMaxMusic3MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
|
||||
hparams["text_config"] = _load_hparams(dir_model)
|
||||
for d in MMPROJ_DIRS:
|
||||
hparams[d] = ModelBase.load_hparams(dir_model / d, is_mistral_format=False)
|
||||
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
# the generic gen layers are the depth decoder
|
||||
return self.global_config["rvq_depth_decoder"]
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
# from the backbone, only the embeddings of the semantic codes
|
||||
return _index_dirs(self, (LM_DIR, *MMPROJ_DIRS),
|
||||
lambda name: not name.startswith(LM_DIR + ".") or name == f"{LM_DIR}.model.embed_tokens.weight")
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
return item
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.MINIMAX_MUSIC3_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(hp["hidden_size"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(hp["intermediate_size"])
|
||||
self.gguf_writer.add_gen_audio_block_count(hp["num_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(hp["num_attention_heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-6)
|
||||
# the DiT, the condition encoder and the vocoder hparams come from the tensor shapes or are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# only the transformer layers follow --outtype, the convs and the small projections stay F32
|
||||
if n_dims >= 2 and new_name.startswith(("a.gen.code.blk.", "a.gen.flow.blk.")):
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
if new_name.startswith(("a.gen.code.head", "a.gen.code.embd", "a.gen.code.out_embd", "a.gen.code.proj_in")):
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name.startswith("a."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
comp, key = name.split(".", 1)
|
||||
if comp == LM_DIR:
|
||||
yield ("a.gen.code.out_embd.weight", data_torch[AUDIO_CODE_OFFSET:AUDIO_CODE_OFFSET + N_SEMANTIC])
|
||||
elif comp == "rvq_depth_decoder":
|
||||
yield from self._depth(data_torch, key)
|
||||
elif comp == "condition_encoder":
|
||||
yield from self._cond(data_torch, key)
|
||||
elif comp == "transformer":
|
||||
yield from self._dit(data_torch, key)
|
||||
# the vocoder is folded in generate_extra_tensors()
|
||||
|
||||
def _depth(self, t: Tensor, key: str) -> Iterable[tuple[str, Tensor]]:
|
||||
p = "a.gen.code"
|
||||
n_cb = self.hparams["num_codebooks"] - 1
|
||||
if key == "audio_embeddings.weight":
|
||||
# one table per residual codebook
|
||||
yield (f"{p}.embd.weight", t.reshape(n_cb, -1, t.shape[-1]))
|
||||
elif key.startswith("audio_heads."):
|
||||
if key == "audio_heads.0.weight":
|
||||
heads = [self._tensor(f"rvq_depth_decoder.audio_heads.{i}.weight") for i in range(n_cb)]
|
||||
yield (f"{p}.head.weight", torch.stack(heads))
|
||||
elif key == "projection.weight":
|
||||
yield (f"{p}.proj_in.weight", t)
|
||||
elif key == "pos_embedding.weight":
|
||||
yield (f"{p}.position_embd.weight", t)
|
||||
elif key == "norm.weight":
|
||||
yield (f"{p}.post_ln.weight", t)
|
||||
else:
|
||||
_, il, rest = key.split(".", 2)
|
||||
layer = {
|
||||
"input_layernorm.weight": "ln1.weight",
|
||||
"attn.to_q.weight": "attn_q.weight",
|
||||
"attn.to_k.weight": "attn_k.weight",
|
||||
"attn.to_v.weight": "attn_v.weight",
|
||||
"attn.to_out.weight": "attn_out.weight",
|
||||
"post_attention_layernorm.weight": "ln2.weight",
|
||||
"gate_proj.weight": "ffn_gate.weight",
|
||||
"up_proj.weight": "ffn_up.weight",
|
||||
"down_proj.weight": "ffn_down.weight",
|
||||
}
|
||||
yield (f"{p}.blk.{il}.{layer[rest]}", t)
|
||||
|
||||
def _cond(self, t: Tensor, key: str) -> Iterable[tuple[str, Tensor]]:
|
||||
p = "a.gen.cond"
|
||||
if key == "layer_weight_logits":
|
||||
# softmax mix of the 8 hidden states of a frame, with the scale folded in
|
||||
scale = self._tensor("condition_encoder.layer_scale").float()
|
||||
yield (f"{p}.layer_w", torch.softmax(t.float(), dim=0) * scale)
|
||||
elif key == "proj.weight":
|
||||
yield (f"{p}.proj.weight", t)
|
||||
elif key == "proj.bias":
|
||||
yield (f"{p}.proj.bias", t)
|
||||
|
||||
def _dit(self, t: Tensor, key: str) -> Iterable[tuple[str, Tensor]]:
|
||||
p = "a.gen.flow"
|
||||
if key.startswith("transformer_blocks."):
|
||||
_, il, rest = key.split(".", 2)
|
||||
q = f"{p}.blk.{il}"
|
||||
if rest.startswith("ff_in."):
|
||||
# value half first, then the silu gate half
|
||||
suffix = rest.removeprefix("ff_in.")
|
||||
up, gate = t.chunk(2, dim=0)
|
||||
yield (f"{q}.ffn_up.{suffix}", up)
|
||||
yield (f"{q}.ffn_gate.{suffix}", gate)
|
||||
return
|
||||
layer = {
|
||||
"norm1.weight": "ln1.weight",
|
||||
"norm1.bias": "ln1.bias",
|
||||
"attn.to_q.weight": "attn_q.weight",
|
||||
"attn.to_k.weight": "attn_k.weight",
|
||||
"attn.to_v.weight": "attn_v.weight",
|
||||
"attn.to_out.0.weight": "attn_out.weight",
|
||||
"norm2.weight": "ln2.weight",
|
||||
"norm2.bias": "ln2.bias",
|
||||
"ff_out.weight": "ffn_down.weight",
|
||||
"ff_out.bias": "ffn_down.bias",
|
||||
}
|
||||
yield (f"{q}.{layer[rest]}", t)
|
||||
return
|
||||
if key == "time_proj.weight":
|
||||
# random Fourier features, 2 * pi folded in
|
||||
yield (f"{p}.time_freq", t.float().flatten() * 2.0 * torch.pi)
|
||||
return
|
||||
out = {
|
||||
"time_embed.linear_1.weight": "time_in.weight",
|
||||
"time_embed.linear_1.bias": "time_in.bias",
|
||||
"time_embed.linear_2.weight": "time_out.weight",
|
||||
"time_embed.linear_2.bias": "time_out.bias",
|
||||
"preprocess_conv.weight": "pre.weight",
|
||||
"proj_in.weight": "proj_in.weight",
|
||||
"proj_out.weight": "proj_out.weight",
|
||||
"postprocess_conv.weight": "post.weight",
|
||||
}[key]
|
||||
yield (f"{p}.{out}", t.squeeze(-1) if key.endswith("conv.weight") else t)
|
||||
|
||||
def _tensor(self, name: str) -> Tensor:
|
||||
# read directly, without the dtype handling of prepare_tensors()
|
||||
return self.model_tensors[name]().float()
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# DAC decoder, same layout as the yue2 VAE (a.gen.vae.*), with the snake and the weight norm folded
|
||||
def get(key: str) -> Tensor:
|
||||
return self._tensor(f"vocoder.{key}")
|
||||
|
||||
def wn(key: str) -> Tensor:
|
||||
return _fold_wn(get(key + ".weight_g"), get(key + ".weight_v"))
|
||||
|
||||
def snake(dst: str, key: str) -> Iterable[tuple[str, Tensor]]:
|
||||
a, b = fold_snake(get(key + ".alpha"))
|
||||
yield (f"{dst}.alpha", a)
|
||||
yield (f"{dst}.beta", b)
|
||||
|
||||
p = "a.gen.vae"
|
||||
yield (f"{p}.in_proj.weight", get("dec_in_proj.weight"))
|
||||
yield (f"{p}.in_proj.bias", get("dec_in_proj.bias"))
|
||||
yield (f"{p}.conv_in.weight", wn("conv_in"))
|
||||
yield (f"{p}.conv_in.bias", get("conv_in.bias"))
|
||||
n_blk = len(self.global_config["vocoder"]["upsampling_ratios"])
|
||||
for b in range(n_blk):
|
||||
s = f"blocks.{b}"
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.snake1")
|
||||
yield (f"{p}.blk.{b}.up.weight", wn(f"{s}.conv_t1"))
|
||||
yield (f"{p}.blk.{b}.up.bias", get(f"{s}.conv_t1.bias"))
|
||||
for r in range(3):
|
||||
rs = f"{s}.res_unit{r + 1}"
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake1", f"{rs}.snake1")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.weight", wn(f"{rs}.conv1"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.bias", get(f"{rs}.conv1.bias"))
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake2", f"{rs}.snake2")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.weight", wn(f"{rs}.conv2"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.bias", get(f"{rs}.conv2.bias"))
|
||||
yield from snake(f"{p}.post_snake", "snake_out")
|
||||
yield (f"{p}.conv_out.weight", wn("conv_out"))
|
||||
yield (f"{p}.conv_out.bias", get("conv_out.bias"))
|
||||
@@ -18,9 +18,12 @@ from .qwen3vl import Qwen3VLVisionModel
|
||||
def _is_pplx_decider_checkpoint(dir_model: Path) -> bool:
|
||||
if not all((dir_model / name).is_file() for name in ("decision_config.json", "readout.safetensors", "config.json")):
|
||||
return False
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
with open(dir_model / "decision_config.json", encoding="utf-8") as f:
|
||||
# the same layout with a format_id is a vllm-d3 checkpoint
|
||||
return "format_id" not in json.load(f)
|
||||
decision = json.load(f)
|
||||
# pplx-decider is a Qwen3.5 backbone, and a format_id means vllm-d3
|
||||
return config.get("model_type") in ("qwen3_5", "qwen3_5_text") and "format_id" not in decision
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_pplx_decider_checkpoint)
|
||||
|
||||
+72
-2
@@ -275,6 +275,14 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
|
||||
or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight")))
|
||||
):
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
# the reference encoder's convs run with an F32 im2col, see f32_im2col in pockettts-seanet.cpp
|
||||
if new_name.startswith("a.seanet.") or new_name == "a.downsample.conv.weight":
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# codes are picked by the nearest codebook entry: Q8_0 changes about 8% of them, F16 none
|
||||
if new_name.startswith("a.gen.enc.tfm.blk.") and new_name.endswith(".weight") and n_dims == 2:
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
if new_name.startswith("a.gen.enc.quant."):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# ConvTranspose1d kernels: only F16/F32 are implemented, no BF16
|
||||
if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
@@ -298,8 +306,8 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# code2wav tensors are already named by generate_extra_tensors(), pass them through
|
||||
if name.startswith("a.gen.wav."):
|
||||
# code2wav and reference encoder tensors are already named by generate_extra_tensors(), pass them through
|
||||
if name.startswith(("a.gen.wav.", "a.gen.enc.", "a.seanet.", "a.downsample.")):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
@@ -356,6 +364,68 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
yield from self._generate_code2wav_tensors()
|
||||
yield from self._generate_ref_encoder_tensors()
|
||||
|
||||
# mimi encoder of the speech tokenizer: reference clip -> codes, for voice cloning with a transcript (ICL)
|
||||
def _generate_ref_encoder_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
from safetensors.torch import load_file
|
||||
|
||||
state_dict = load_file(self.dir_model / "speech_tokenizer" / "model.safetensors")
|
||||
if "encoder.downsample.conv.weight" not in state_dict:
|
||||
return
|
||||
|
||||
with open(self.dir_model / "speech_tokenizer" / "config.json", "r", encoding="utf-8") as f:
|
||||
enc_config = json.load(f)["encoder_config"]
|
||||
n_stage = len(enc_config["upsampling_ratios"])
|
||||
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
# encoder.encoder.layers: conv_in, then per stage a residual unit, an ELU and a strided conv, then an ELU and conv_out
|
||||
def seanet(idx: int, suffix: str) -> Tensor:
|
||||
return state_dict[f"encoder.encoder.layers.{idx}.{suffix}"]
|
||||
|
||||
for suffix in ("weight", "bias"):
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_CONV_IN, suffix=f".{suffix}"), seanet(0, f"conv.{suffix}"))
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_CONV_OUT, suffix=f".{suffix}"), seanet(3 * n_stage + 2, f"conv.{suffix}"))
|
||||
for stage in range(n_stage):
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_RES_CONV1, stage, suffix=f".{suffix}"), seanet(3 * stage + 1, f"block.1.conv.{suffix}"))
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_RES_CONV2, stage, suffix=f".{suffix}"), seanet(3 * stage + 1, f"block.3.conv.{suffix}"))
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_SCALE_CONV, stage, suffix=f".{suffix}"), seanet(3 * stage + 3, f"conv.{suffix}"))
|
||||
|
||||
yield (self.format_tensor_name(T.A_ENC_DOWNSAMPLE_CONV), state_dict["encoder.downsample.conv.weight"])
|
||||
|
||||
tfm_layer_map = {
|
||||
"input_layernorm.weight": (T.A_GEN_ENC_TFM_ATTN_NORM, ".weight"),
|
||||
"input_layernorm.bias": (T.A_GEN_ENC_TFM_ATTN_NORM, ".bias"),
|
||||
"self_attn.q_proj.weight": (T.A_GEN_ENC_TFM_ATTN_Q, ".weight"),
|
||||
"self_attn.k_proj.weight": (T.A_GEN_ENC_TFM_ATTN_K, ".weight"),
|
||||
"self_attn.v_proj.weight": (T.A_GEN_ENC_TFM_ATTN_V, ".weight"),
|
||||
"self_attn.o_proj.weight": (T.A_GEN_ENC_TFM_ATTN_OUT, ".weight"),
|
||||
"self_attn_layer_scale.scale": (T.A_GEN_ENC_TFM_ATTN_SCALE, ".weight"),
|
||||
"post_attention_layernorm.weight": (T.A_GEN_ENC_TFM_FFN_NORM, ".weight"),
|
||||
"post_attention_layernorm.bias": (T.A_GEN_ENC_TFM_FFN_NORM, ".bias"),
|
||||
"mlp.fc1.weight": (T.A_GEN_ENC_TFM_FFN_UP, ".weight"),
|
||||
"mlp.fc2.weight": (T.A_GEN_ENC_TFM_FFN_DOWN, ".weight"),
|
||||
"mlp_layer_scale.scale": (T.A_GEN_ENC_TFM_FFN_SCALE, ".weight"),
|
||||
}
|
||||
for bid in range(enc_config["num_hidden_layers"]):
|
||||
for key, (tensor_id, suffix) in tfm_layer_map.items():
|
||||
yield (self.format_tensor_name(tensor_id, bid, suffix=suffix), state_dict[f"encoder.encoder_transformer.layers.{bid}.{key}"])
|
||||
|
||||
def rvq_codebook(prefix: str, idx: int) -> Tensor:
|
||||
# checkpoint has EMA accumulators, so codebook = embed_sum / cluster_usage
|
||||
embed_sum = state_dict[f"{prefix}.layers.{idx}.codebook.embed_sum"]
|
||||
cluster_usage = state_dict[f"{prefix}.layers.{idx}.codebook.cluster_usage"]
|
||||
return embed_sum / cluster_usage.clamp_min(1e-5).unsqueeze(-1)
|
||||
|
||||
# the tokenizer keeps the semantic codebook and the first acoustic ones, see encoder_valid_num_quantizers
|
||||
semantic = "encoder.quantizer.semantic_residual_vector_quantizer"
|
||||
acoustic = "encoder.quantizer.acoustic_residual_vector_quantizer"
|
||||
yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_FIRST_IN), state_dict[f"{semantic}.input_proj.weight"].squeeze(-1))
|
||||
yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_FIRST_CB), rvq_codebook(semantic, 0))
|
||||
yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_REST_IN), state_dict[f"{acoustic}.input_proj.weight"].squeeze(-1))
|
||||
yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_REST_CB),
|
||||
torch.stack([rvq_codebook(acoustic, i) for i in range(self._CODE_GEN_N_CODEBOOKS)], dim=0))
|
||||
|
||||
def _generate_code2wav_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# code2wav weights live in speech_tokenizer/model.safetensors, not the main safetensors
|
||||
|
||||
+30
-8
@@ -8,7 +8,7 @@ from torch import Tensor
|
||||
import gguf
|
||||
import numpy as np
|
||||
|
||||
from .base import ModelBase
|
||||
from .base import LazyTorchTensor, ModelBase
|
||||
from .qwen import _LinearAttentionVReorderBase, _Qwen35MRopeMixin
|
||||
from .qwen3vl import Qwen3VLVisionModel
|
||||
|
||||
@@ -41,6 +41,18 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
self._ple_shards: dict[int, str] = {}
|
||||
self._ple_row_dim: int | None = None
|
||||
self._mtp_fc: dict[str, Tensor] = {}
|
||||
self._ple_scale: float | None = None
|
||||
|
||||
def dequant_model(self):
|
||||
# FP8 checkpoints: all n-gram shards share one scale, stored under the parent module name
|
||||
# there is no "ngram_embedding.weight", so the generic FP8 pass drops it or cannot map it
|
||||
for name in list(self.model_tensors):
|
||||
if name.endswith(".ngram_embedding.weight_scale"):
|
||||
scale = LazyTorchTensor.to_eager(self.model_tensors.pop(name)()).float().flatten()
|
||||
if scale.numel() != 1:
|
||||
raise ValueError(f"PLE scale {name!r} must be a scalar, got {scale.numel()} values")
|
||||
self._ple_scale = float(scale[0])
|
||||
super().dequant_model()
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
@@ -203,21 +215,31 @@ class Qwen4ExpTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
f"PLE shard {shard} has row dim {int(shape[-1])}, expected {self._ple_row_dim}")
|
||||
rows += int(shape[0])
|
||||
|
||||
# an FP8 table keeps its bytes and gets a scale tensor, any other type has the scale folded in
|
||||
keep_fp8 = name in self._fp8_e4m3_preserved
|
||||
table = gguf.LazyChunkedTensor(
|
||||
[self._load_ple_shard(shard) for shard in shards],
|
||||
[self._load_ple_shard(shard, keep_fp8) for shard in shards],
|
||||
shape=(rows, self._ple_row_dim),
|
||||
dtype=np.float32,
|
||||
dtype=np.uint8 if keep_fp8 else np.float32,
|
||||
)
|
||||
gguf_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD]
|
||||
return [(gguf_name + ".weight", cast(Tensor, table))]
|
||||
out = [(gguf_name + ".weight", cast(Tensor, table))]
|
||||
if keep_fp8 and self._ple_scale is not None:
|
||||
out.append((gguf_name + ".scale", torch.tensor([self._ple_scale], dtype=torch.float32)))
|
||||
return out
|
||||
|
||||
def _load_ple_shard(self, name: str):
|
||||
def _load_ple_shard(self, name: str, keep_fp8: bool):
|
||||
def load() -> np.ndarray:
|
||||
from .base import LazyTorchTensor
|
||||
|
||||
# a fresh lazy tensor every call, or to_eager() memoizes every shard
|
||||
eager = LazyTorchTensor.to_eager(self.model_tensors[name]())
|
||||
return eager.to(torch.float32).contiguous().numpy()
|
||||
if keep_fp8:
|
||||
if eager.dtype != torch.float8_e4m3fn:
|
||||
raise ValueError(f"PLE shard {name} is {eager.dtype}, expected FP8 like the other shards")
|
||||
return eager.contiguous().view(torch.uint8).numpy()
|
||||
data = eager.to(torch.float32)
|
||||
if self._ple_scale is not None:
|
||||
data = data * self._ple_scale
|
||||
return data.contiguous().numpy()
|
||||
return load
|
||||
|
||||
def prepare_tensors(self):
|
||||
|
||||
@@ -0,0 +1,443 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, MmprojModel, gguf, logger
|
||||
|
||||
# tiny-voca, https://huggingface.co/n4ze3m/tiny-voca
|
||||
#
|
||||
# text model: phoneme encoder (4 blocks), the input rows are built by the audio pipeline (token + stress + voice)
|
||||
# mmproj: input tables, voices, duration predictor, prosody and acoustic stacks, source-filter heads
|
||||
#
|
||||
# the text to phonemes step (dizione) is the "tts" chat template, see models/templates/n4ze3m-tiny-voca-tts.jinja
|
||||
# its lexicon comes from the phonemizer/ directory of the model, the other tables from the dizione python package
|
||||
#
|
||||
# the prosody and acoustic stacks have the same blocks as the encoder, they are stored with the libllama names
|
||||
# of the tiny-voca arch under a.gen.prosody. and a.gen.acoustic., mtmd runs them on llama contexts of its own
|
||||
|
||||
CHECKPOINT = "tiny-voca.pt"
|
||||
DEFAULT_VOICE = "M2"
|
||||
|
||||
# phone classes of the reference (text.py): the phones get at least one frame, the vowels are always voiced
|
||||
VOWELS = ("ə", "ɐ", "ɚ", "ᵻ", "ɪ", "i", "iː", "ɛ", "æ", "ʌ", "ɑː", "ɔ", "ɔː", "ʊ", "uː", "ɜː",
|
||||
"eɪ", "aɪ", "ɔɪ", "oʊ", "aʊ", "iə", "oː",
|
||||
"ɑːɹ", "ɔːɹ", "oːɹ", "ɪɹ", "ɛɹ", "ʊɹ", "aɪɚ", "aɪə", "əl", "n̩")
|
||||
CONSONANTS = ("p", "b", "t", "d", "k", "ɡ", "f", "v", "θ", "ð", "s", "z", "ʃ", "ʒ", "h",
|
||||
"m", "n", "ŋ", "l", "ɹ", "j", "w", "tʃ", "dʒ", "ɾ", "ʔ")
|
||||
|
||||
# input letters and output phonemes of the unknown-word model, fixed in mtmd (see mtmd_graph_tiny_voca)
|
||||
G2P_LETTERS = "abcdefghijklmnopqrstuvwxyz'"
|
||||
G2P_SYMBOLS = [v + d for v in ("AA", "AE", "AH", "AO", "AW", "AY") for d in "012"] + ["B", "CH", "D", "DH"] + \
|
||||
[v + d for v in ("EH", "ER", "EY") for d in "012"] + ["F", "G", "HH"] + [v + d for v in ("IH", "IY") for d in "012"] + \
|
||||
["JH", "K", "L", "M", "N", "NG"] + [v + d for v in ("OW", "OY") for d in "012"] + ["P", "R", "S", "SH", "T", "TH"] + \
|
||||
[v + d for v in ("UH", "UW") for d in "012"] + ["V", "W", "Y", "Z", "ZH"]
|
||||
|
||||
# block tensors: reference name -> libllama name
|
||||
BLOCK_MAP = {
|
||||
"norm1": "attn_norm",
|
||||
"attention.qkv": "attn_qkv",
|
||||
"attention.out": "attn_output",
|
||||
"norm2": "ffn_norm",
|
||||
"depthwise": "ssm_conv1d",
|
||||
"expand": "ffn_up",
|
||||
"project": "ffn_down",
|
||||
}
|
||||
|
||||
# mmproj stacks: reference module -> prefix
|
||||
STACK_PREFIX = {
|
||||
"prosody_stack": "a.gen.prosody.",
|
||||
"acoustic_stack": "a.gen.acoustic.",
|
||||
}
|
||||
|
||||
# other mmproj tensors: reference module -> name
|
||||
MMPROJ_MAP = {
|
||||
"tokens": "a.gen.tok_embd",
|
||||
"stress": "a.gen.stress_embd",
|
||||
"duration.conv1": "a.gen.dur.conv1",
|
||||
"duration.conv2": "a.gen.dur.conv2",
|
||||
"duration.norm1": "a.gen.dur.norm1",
|
||||
"duration.norm2": "a.gen.dur.norm2",
|
||||
"duration.out": "a.gen.dur.out",
|
||||
"frame_position": "a.gen.frame_pos",
|
||||
"prosody_head": "a.gen.prosody_head",
|
||||
"prosody_projection": "a.gen.prosody_proj",
|
||||
"vocoder.harmonic": "a.gen.voc.harmonic",
|
||||
"vocoder.noise": "a.gen.voc.noise",
|
||||
}
|
||||
|
||||
|
||||
def _is_tiny_voca_checkpoint(dir_model: Path) -> bool:
|
||||
if not (dir_model / CHECKPOINT).is_file() or not (dir_model / "config.json").is_file():
|
||||
return False
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
return config.get("phonemizer", {}).get("name") == "dizione" and "vocab" in config and "model" in config
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_tiny_voca_checkpoint)
|
||||
def _load_tiny_voca_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected tiny-voca checkpoint")
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
c = config["model"]
|
||||
return {
|
||||
"architectures": ["TinyVocaModel"],
|
||||
"model_type": "tiny-voca",
|
||||
"num_hidden_layers": c["encoder_layers"],
|
||||
"hidden_size": c["channels"],
|
||||
"intermediate_size": c["feedforward"],
|
||||
"num_attention_heads": c["heads"],
|
||||
"num_key_value_heads": c["heads"],
|
||||
"max_position_embeddings": c["max_tokens"],
|
||||
"vocab_size": c["vocab_size"],
|
||||
"layer_norm_eps": 1e-5, # torch LayerNorm default
|
||||
"rope_theta": 10000.0,
|
||||
"tiny_voca": config,
|
||||
# the mmproj tensors are named explicitly, they do not go through the block tensor map
|
||||
"audio_config": {"num_hidden_layers": 0},
|
||||
}
|
||||
|
||||
|
||||
def _table_rows(path: Path) -> Iterable[list[str]]:
|
||||
# dizione tables: columns separated by tabs or two or more spaces, "#" comments
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.split("#", 1)[0].rstrip() if line.lstrip().startswith("#") else line.rstrip()
|
||||
if line.strip():
|
||||
yield [c.strip() for c in re.split(r"\t+| {2,}", line.strip())]
|
||||
|
||||
|
||||
def _g2p_dicts(dir_model: Path) -> dict[str, dict[str, Any]] | None:
|
||||
lexicon_path = dir_model / "phonemizer" / "lexicon.tsv"
|
||||
spec = importlib.util.find_spec("dizione")
|
||||
if not lexicon_path.is_file() or spec is None or spec.origin is None:
|
||||
logger.warning("dizione or phonemizer/lexicon.tsv not found, the model will take phonemes as prompt "
|
||||
"(pip install git+https://github.com/n4ze3m/dizione)")
|
||||
return None
|
||||
data = Path(spec.origin).parent / "data"
|
||||
lexicon: dict[str, str] = {}
|
||||
for line in lexicon_path.read_text(encoding="utf-8").splitlines():
|
||||
if "\t" in line:
|
||||
word, phones = line.split("\t", 1)
|
||||
lexicon[word] = phones.strip()
|
||||
|
||||
# word -> [stress class, form, final form or ""], see function_words.txt
|
||||
functions = {r[0]: [r[1], r[2], r[3] if len(r) > 3 else ""] for r in _table_rows(data / "function_words.txt")}
|
||||
# words -> [class, forms (one, or one per word), final form or ""], see phrases.txt
|
||||
phrases = {r[0]: [r[1], [f.strip() for f in r[2].split("|")], r[3] if len(r) > 3 else ""] for r in _table_rows(data / "phrases.txt")}
|
||||
letters = {r[0]: r[1] for r in _table_rows(data / "letters.txt")}
|
||||
# a value is phones, or CMUdict notation (converted by the template, like the output of the unknown-word model)
|
||||
exceptions = {r[0].lower(): r[1] for r in _table_rows(data / "exceptions.txt") if " " not in r[0]}
|
||||
# a name of several words: "w1 w2 ..." -> one pronunciation per word; first word -> the longest name it starts
|
||||
names: dict[str, Any] = {}
|
||||
name_starts: dict[str, int] = {}
|
||||
for r in _table_rows(data / "exceptions.txt"):
|
||||
words = r[0].lower().split()
|
||||
if len(words) > 1:
|
||||
names[" ".join(words)] = [part.strip() for part in r[1].split("|")]
|
||||
name_starts[words[0]] = max(name_starts.get(words[0], 0), len(words))
|
||||
# phoneme (CMUdict, no stress) -> {letters: log probability}: aligns the letters of a word to its phonemes
|
||||
with open(dir_model / "phonemizer" / "align.json", encoding="utf-8") as f:
|
||||
align = json.load(f)
|
||||
# abbreviation -> [when, words]
|
||||
abbreviations = {r[0]: [r[1], r[2]] for r in _table_rows(data / "abbreviations.txt")}
|
||||
# chat spelling -> [words to say ("*LOL": letter by letter, "-": nothing), only as a reply by itself], see chat.txt
|
||||
chat = {r[0].lower(): [r[1] if len(r) > 1 else "-", len(r) > 2] for r in _table_rows(data / "chat.txt")}
|
||||
# accented latin letter -> plain letter(s): "café" is in the dictionary as "cafe" (dizione: _plain_letter)
|
||||
accents = {"ß": "ss", "æ": "ae", "Æ": "Ae", "œ": "oe", "Œ": "Oe", "ø": "o", "Ø": "O", "đ": "d", "ł": "l", "Ł": "L"}
|
||||
for cp in range(0x80, 0x2000):
|
||||
char = chr(cp)
|
||||
if char.isalpha() and char not in accents:
|
||||
plain = "".join(c for c in unicodedata.normalize("NFKD", char) if not unicodedata.combining(c))
|
||||
if plain.isascii() and plain.isalpha():
|
||||
accents[char] = plain
|
||||
# compatibility forms of the text (NFKC), a character at a time: "…" is "...", "fi" is "fi", "№" is "No", full
|
||||
# width letters. Sequences that NFKC would compose are left as they are
|
||||
nfkc: dict[str, str] = {}
|
||||
for cp in list(range(0x80, 0x3000)) + list(range(0xFB00, 0xFB07)) + list(range(0xFF01, 0xFF5F)):
|
||||
if unicodedata.normalize("NFKC", chr(cp)) != chr(cp):
|
||||
nfkc[chr(cp)] = unicodedata.normalize("NFKC", chr(cp))
|
||||
# units after a number: name -> [one, many]; "exact" ones are written in one letter case only ("m", not "M")
|
||||
units: dict[str, Any] = {}
|
||||
exact_units: dict[str, Any] = {}
|
||||
for r in _table_rows(data / "units.txt"):
|
||||
if len(r) > 3 and r[3] == "exact":
|
||||
exact_units[r[0]] = [r[1], r[2]]
|
||||
else:
|
||||
units[r[0].lower()] = [r[1], r[2]]
|
||||
# the unit names as a pattern of tts_norm_scan(), the longest first; a unit does not stop inside a word
|
||||
unit_names = sorted(set(units) | set(exact_units), key=lambda name: (-len(name), name))
|
||||
# (the pattern ignores the case of ASCII letters only, the greek letter is written in both)
|
||||
unit_pattern = "|".join("".join("[μΜ]" if c == "μ" else c if c.isalnum() else "%" + c for c in name) + "(?![A-Za-z])" for name in unit_names)
|
||||
# endings of file names and addresses: "word" (dot com) or "letters" (dot P D F)
|
||||
endings = {r[0]: (r[1] if len(r) > 1 else "") for r in _table_rows(data / "endings.txt")}
|
||||
out = {
|
||||
"lexicon": lexicon,
|
||||
"functions": functions,
|
||||
"phrases": phrases,
|
||||
"letters": letters,
|
||||
"exceptions": exceptions,
|
||||
"abbreviations": abbreviations,
|
||||
"align": align,
|
||||
"names": names,
|
||||
"chat": chat,
|
||||
"accents": accents,
|
||||
"nfkc": nfkc,
|
||||
"units": units,
|
||||
"exact_units": exact_units,
|
||||
"endings": endings,
|
||||
"patterns": {"units": unit_pattern},
|
||||
"name_starts": name_starts,
|
||||
}
|
||||
out.update(_homograph_dicts(dir_model, data))
|
||||
return out
|
||||
|
||||
|
||||
def _homograph_dicts(dir_model: Path, data: Path) -> dict[str, dict[str, Any]]:
|
||||
# words spelled alike and pronounced differently (dizione homographs.py): a linear classifier on the neighbouring
|
||||
# words, and hand-written rules that are tried first. Weights are stored x10000 (they have four decimals), so the
|
||||
# template adds integers.
|
||||
path = dir_model / "phonemizer" / "homographs.json"
|
||||
if not path.is_file():
|
||||
return {"homographs": {}, "homograph_shared": {}, "homograph_plurals": {}, "homograph_rules": {}}
|
||||
with open(path, encoding="utf-8") as f:
|
||||
source = json.load(f)
|
||||
|
||||
def fixed(x: float) -> int:
|
||||
value = round(x * 10000)
|
||||
if abs(value - x * 10000) > 1e-6:
|
||||
raise ValueError(f"homographs.json: weight {x} has more than four decimals")
|
||||
return value
|
||||
|
||||
words: dict[str, Any] = {}
|
||||
for word, entry in source["words"].items():
|
||||
model = entry["model"]
|
||||
noun_verb = model["kind"] == "noun-verb"
|
||||
if not noun_verb and model["kind"] != "senses":
|
||||
raise ValueError(f"homographs.json: unknown model kind {model['kind']!r}")
|
||||
words[word] = {
|
||||
# sense: [id, role, phones]
|
||||
"senses": [[s["id"], s["role"], s["phones"]] for s in entry["senses"]],
|
||||
# noun-verb: one score for the verb reading; else one score per sense
|
||||
"noun_verb": noun_verb,
|
||||
"bias": [fixed(model["bias"])] if noun_verb else [fixed(b) for b in model["bias"]],
|
||||
"weights": [{k: fixed(v) for k, v in w.items()} for w in ([model["weights"]] if noun_verb else model["weights"])],
|
||||
}
|
||||
# the weights all noun-or-verb homographs share; "" is the bias
|
||||
shared = {k: fixed(v) for k, v in source["pooled"]["weights"].items()}
|
||||
shared[""] = fixed(source["pooled"]["bias"])
|
||||
# word -> [[sense id, [[condition, [words]], ...]], ...], see homograph_rules.txt
|
||||
rules: dict[str, Any] = {}
|
||||
for r in _table_rows(data / "homograph_rules.txt"):
|
||||
conditions = []
|
||||
for part in r[2].split(";"):
|
||||
name, _, listed = part.strip().partition(":")
|
||||
conditions.append([name, listed.split()])
|
||||
rules.setdefault(r[0], []).append([r[1], conditions])
|
||||
return {
|
||||
"homographs": words,
|
||||
"homograph_shared": shared,
|
||||
"homograph_plurals": {base: 1 for base in source.get("plural_bases", ())},
|
||||
"homograph_rules": rules,
|
||||
}
|
||||
|
||||
|
||||
def _load_state(dir_model: Path) -> dict[str, Tensor]:
|
||||
ckpt = torch.load(dir_model / CHECKPOINT, map_location="cpu", weights_only=True)
|
||||
if ckpt.get("format_version") != 4:
|
||||
raise ValueError(f"unsupported tiny-voca checkpoint format {ckpt.get('format_version')}")
|
||||
return ckpt["model"]
|
||||
|
||||
|
||||
def _block_name(name: str) -> tuple[int, str] | None:
|
||||
# "blocks.<i>.<module>.<weight|bias>" -> (i, "blk.<i>.<libllama name>.<weight|bias>")
|
||||
parts = name.split(".")
|
||||
if len(parts) < 4 or parts[0] != "blocks":
|
||||
return None
|
||||
bid = int(parts[1])
|
||||
module, suffix = ".".join(parts[2:-1]), parts[-1]
|
||||
return bid, f"blk.{bid}.{BLOCK_MAP[module]}.{suffix}"
|
||||
|
||||
|
||||
def _block_tensors(new_name: str, t: Tensor) -> Iterable[tuple[str, Tensor]]:
|
||||
# depthwise conv [C, 1, K] -> [C, K], the layout of ggml_ssm_conv
|
||||
if ".ssm_conv1d.weight" in new_name:
|
||||
yield (new_name, t.squeeze(1))
|
||||
return
|
||||
# separate q, k, v: a model that libllama builds from metadata (the mmproj stacks) has no fused qkv
|
||||
if ".attn_qkv." in new_name:
|
||||
for part, chunk in zip(("attn_q", "attn_k", "attn_v"), t.chunk(3, dim=0)):
|
||||
yield (new_name.replace("attn_qkv", part), chunk.contiguous())
|
||||
return
|
||||
yield (new_name, t)
|
||||
|
||||
|
||||
@ModelBase.register("TinyVocaModel")
|
||||
@ModelBase.example("n4ze3m/tiny-voca")
|
||||
class TinyVocaTextModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.TINY_VOCA
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
del remote_hf_model_id
|
||||
state = _load_state(self.dir_model)
|
||||
out: dict[str, Callable[[], Tensor]] = {}
|
||||
for name, t in state.items():
|
||||
if name == "tokens.weight" or name.startswith("encoder."):
|
||||
out[name] = (lambda t=t: t)
|
||||
return out
|
||||
|
||||
def set_vocab(self):
|
||||
config = self.hparams["tiny_voca"]
|
||||
n_vocab = config["model"]["vocab_size"]
|
||||
tokens: list[str] = [""] * n_vocab
|
||||
toktypes = [gguf.TokenType.NORMAL] * n_vocab
|
||||
for text, i in config["vocab"].items():
|
||||
tokens[i] = text
|
||||
if text.startswith("<") and text.endswith(">"):
|
||||
toktypes[i] = gguf.TokenType.CONTROL
|
||||
assert all(tokens)
|
||||
|
||||
# the prompt is split into phonemes by the audio pipeline, see mtmd-helper-gen.cpp
|
||||
self.gguf_writer.add_tokenizer_model("llama")
|
||||
self.gguf_writer.add_tokenizer_pre("default")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores([0.0] * n_vocab)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
self.gguf_writer.add_bos_token_id(config["vocab"]["<bos>"])
|
||||
self.gguf_writer.add_eos_token_id(config["vocab"]["<eos>"])
|
||||
self.gguf_writer.add_pad_token_id(config["vocab"]["<pad>"])
|
||||
self.gguf_writer.add_unk_token_id(config["vocab"]["<unk>"])
|
||||
self.gguf_writer.add_add_bos_token(False)
|
||||
self.gguf_writer.add_add_eos_token(False)
|
||||
|
||||
dicts = _g2p_dicts(self.dir_model)
|
||||
if dicts is not None:
|
||||
template_path = Path(__file__).parent.parent / "models" / "templates" / "n4ze3m-tiny-voca-tts.jinja"
|
||||
with open(template_path, encoding="utf-8") as f:
|
||||
self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}])
|
||||
self.gguf_writer.add_tts_dicts(dicts)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
self.gguf_writer.add_causal_attention(False)
|
||||
self.gguf_writer.add_pooling_type(gguf.PoolingType.NONE)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
del bid
|
||||
if name == "tokens.weight":
|
||||
yield ("token_embd.weight", data_torch)
|
||||
return
|
||||
if name.startswith("encoder.norm."):
|
||||
yield ("output_norm." + name.rsplit(".", 1)[1], data_torch)
|
||||
return
|
||||
res = _block_name(name.removeprefix("encoder."))
|
||||
if res is None:
|
||||
raise ValueError(f"unexpected tensor {name}")
|
||||
yield from _block_tensors(res[1], data_torch)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
del name, bid, n_dims
|
||||
# ggml_ssm_conv takes F32 weights; token_embd is also read row by row on the host side
|
||||
if ".ssm_conv1d." in new_name or new_name == "token_embd.weight":
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return False
|
||||
|
||||
|
||||
@ModelBase.register("TinyVocaModel")
|
||||
@ModelBase.example("n4ze3m/tiny-voca")
|
||||
class TinyVocaMmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
return self.global_config.get("audio_config")
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
del remote_hf_model_id
|
||||
state = _load_state(self.dir_model)
|
||||
out: dict[str, Callable[[], Tensor]] = {}
|
||||
for name, t in state.items():
|
||||
if name.startswith("encoder.") or name in ("voices.weight", "frame_voices.weight"):
|
||||
continue
|
||||
out[name] = (lambda t=t: t)
|
||||
return out
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
return item
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
c = self.global_config["tiny_voca"]["model"]
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.TINY_VOCA)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(c["channels"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(c["feedforward"])
|
||||
self.gguf_writer.add_gen_audio_block_count(c["acoustic_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(c["heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5)
|
||||
self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
del bid
|
||||
# from generate_extra_tensors(), named already
|
||||
if name.startswith(("voice.", "a.gen.")):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
stack, _, rest = name.partition(".")
|
||||
if stack in STACK_PREFIX:
|
||||
prefix = STACK_PREFIX[stack]
|
||||
if rest.startswith("norm."):
|
||||
yield (prefix + "output_norm." + rest.rsplit(".", 1)[1], data_torch)
|
||||
return
|
||||
res = _block_name(rest)
|
||||
if res is None:
|
||||
raise ValueError(f"unexpected tensor {name}")
|
||||
yield from _block_tensors(prefix + res[1], data_torch)
|
||||
return
|
||||
module, suffix = name.rsplit(".", 1)
|
||||
if module not in MMPROJ_MAP:
|
||||
raise ValueError(f"unexpected tensor {name}")
|
||||
yield (MMPROJ_MAP[module] + "." + suffix, data_torch)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
del name, bid
|
||||
# the blocks of the stacks are loaded by libllama, like the text model
|
||||
if new_name.startswith(tuple(STACK_PREFIX.values())) and ".ssm_conv1d." not in new_name and n_dims >= 2:
|
||||
return False
|
||||
# the rest is small and partly read on the host
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# the unknown-word model of dizione (letters -> CMUdict phonemes), run on the host by mtmd_gen_audio_g2p()
|
||||
g2p_dir = self.dir_model / "phonemizer"
|
||||
if (g2p_dir / "g2p.npz").is_file():
|
||||
with open(g2p_dir / "g2p.json", encoding="utf-8") as f:
|
||||
settings = json.load(f)
|
||||
if settings["letters"] != G2P_LETTERS or settings["symbols"] != G2P_SYMBOLS or settings["convs"] != 0 or settings["slots"] != 2:
|
||||
raise ValueError("unsupported g2p model in phonemizer/g2p.json")
|
||||
with np.load(g2p_dir / "g2p.npz") as arrays:
|
||||
for name in arrays.files:
|
||||
yield ("a.gen.g2p." + name.replace("__", "."), torch.from_numpy(arrays[name].astype(np.float32)))
|
||||
config = self.global_config["tiny_voca"]
|
||||
state = _load_state(self.dir_model)
|
||||
# a voice row is the embedding added to the encoder input, then the one added to the frames
|
||||
for i, voice in enumerate(config["voices"]):
|
||||
yield (f"voice.{voice}", torch.cat([state["voices.weight"][i], state["frame_voices.weight"][i]]).float())
|
||||
# class of each token: 0 = pause (special or punctuation), 1 = consonant or <unk>, 2 = vowel
|
||||
vocab = config["vocab"]
|
||||
cls = torch.zeros(len(vocab), dtype=torch.float32)
|
||||
for p in CONSONANTS + ("<unk>",):
|
||||
cls[vocab[p]] = 1.0
|
||||
for p in VOWELS:
|
||||
cls[vocab[p]] = 2.0
|
||||
yield ("a.gen.token_class", cls)
|
||||
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import MmprojModel, ModelBase, gguf
|
||||
from .qwen import Qwen2Model
|
||||
|
||||
|
||||
# the upstream chat_template.jinja is a processor template (list content, audio parts), use plain ChatML with its default system prompt
|
||||
VIBEVOICE_ASR_CHAT_TEMPLATE = (
|
||||
"{%- if messages[0]['role'] == 'system' -%}"
|
||||
"{{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' -}}"
|
||||
"{%- set loop_messages = messages[1:] -%}"
|
||||
"{%- else -%}"
|
||||
"{{- '<|im_start|>system\\nYou are a helpful assistant that transcribes audio input into text output in JSON format.<|im_end|>\\n' -}}"
|
||||
"{%- set loop_messages = messages -%}"
|
||||
"{%- endif -%}"
|
||||
"{%- for message in loop_messages -%}"
|
||||
"{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}"
|
||||
"{%- endfor -%}"
|
||||
"{%- if add_generation_prompt -%}"
|
||||
"{{- '<|im_start|>assistant\\n' -}}"
|
||||
"{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
@ModelBase.register("VibeVoiceAsrForConditionalGeneration")
|
||||
@ModelBase.example("microsoft/VibeVoice-ASR-HF")
|
||||
class VibeVoiceAsrModel(Qwen2Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN2
|
||||
|
||||
def set_vocab(self):
|
||||
super().set_vocab()
|
||||
self.gguf_writer.add_chat_template(VIBEVOICE_ASR_CHAT_TEMPLATE)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not name.startswith("language_model."):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
|
||||
@ModelBase.register("VibeVoiceAsrForConditionalGeneration")
|
||||
class VibeVoiceAsrAudioModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
|
||||
# values hard-coded in tools/mtmd/models/vibevoice-asr.cpp
|
||||
_encoder_consts = {"channels": 1, "kernel_size": 7, "num_filters": 32, "ffn_expansion": 4, "hidden_act": "gelu", "rms_norm_eps": 1e-5}
|
||||
_encoders = ("acoustic_tokenizer_encoder", "semantic_tokenizer_encoder")
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
c = self.global_config["acoustic_tokenizer_encoder_config"]
|
||||
return {
|
||||
**c,
|
||||
"intermediate_size": c["ffn_expansion"] * c["hidden_size"],
|
||||
"num_attention_heads": 1,
|
||||
"num_hidden_layers": sum(c["depths"]),
|
||||
}
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
for enc in self._encoders:
|
||||
c = self.global_config[f"{enc}_config"]
|
||||
for k, v in self._encoder_consts.items():
|
||||
assert c[k] == v, f"{enc}.{k} = {c[k]}, expected {v}"
|
||||
assert c["depths"] == self.hparams["depths"] and c["downsampling_ratios"] == self.hparams["downsampling_ratios"]
|
||||
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.VIBEVOICE_ASR)
|
||||
# the input is the raw waveform, one channel takes the place of the mel bins
|
||||
self.gguf_writer.add_audio_num_mel_bins(1)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams["rms_norm_eps"])
|
||||
self.gguf_writer.add_audio_conv_kernel_size(self.hparams["kernel_size"])
|
||||
# stride of each stage, the stem has none
|
||||
self.gguf_writer.add_audio_patch_sizes([1] + [int(r) for r in self.hparams["downsampling_ratios"]])
|
||||
self.gguf_writer.add_audio_depths([int(d) for d in self.hparams["depths"]])
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
if ".dw." in new_name:
|
||||
return gguf.GGMLQuantizationType.F32 # ggml_ssm_conv
|
||||
if ".conv." in new_name and new_name.endswith(".weight"):
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not name.startswith(cls._encoders + ("multi_modal_projector.",)):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
_block_tensors = {
|
||||
"norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_NORM,
|
||||
"mixer.conv.weight": gguf.MODEL_TENSOR.A_ENC_CNX_DW,
|
||||
"mixer.conv.bias": gguf.MODEL_TENSOR.A_ENC_CNX_DW,
|
||||
"gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS1,
|
||||
"ffn_norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_NORM,
|
||||
"ffn.linear1.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP,
|
||||
"ffn.linear1.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP,
|
||||
"ffn.linear2.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
|
||||
"ffn.linear2.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
|
||||
"ffn_gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS2,
|
||||
}
|
||||
|
||||
# mm.a.mlp.{0,1} + mm.a.mlp_norm.0 for the acoustic connector, {2,3} + 2 for the semantic one
|
||||
_proj_tensors = {
|
||||
"acoustic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 0),
|
||||
"acoustic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 0),
|
||||
"acoustic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 1),
|
||||
"semantic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 2),
|
||||
"semantic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 2),
|
||||
"semantic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 3),
|
||||
}
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# the layer scales (gamma) have no suffix upstream
|
||||
suffix = "." + name.rsplit(".", 1)[-1] if name.endswith((".weight", ".bias")) else ".weight"
|
||||
|
||||
if name.startswith("multi_modal_projector."):
|
||||
key, idx = self._proj_tensors[name.split(".")[1]]
|
||||
yield (self.format_tensor_name(key, idx, suffix=suffix), data_torch)
|
||||
return
|
||||
|
||||
enc, _, rest = name.partition(".")
|
||||
enc_id = self._encoders.index(enc)
|
||||
parts = rest.split(".")
|
||||
depths = [int(d) for d in self.hparams["depths"]]
|
||||
|
||||
# causal convs of the stages: 0 = stem, 1..6 = downsample, 7 = head
|
||||
if rest.startswith("stem.conv."):
|
||||
conv_id = 0
|
||||
elif parts[0] == "conv_layers" and parts[2] == "conv":
|
||||
conv_id = int(parts[1]) + 1
|
||||
elif parts[0] == "head":
|
||||
conv_id = len(depths)
|
||||
else:
|
||||
conv_id = None
|
||||
if conv_id is not None:
|
||||
yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_CNX_CONV].format(bid=enc_id, xid=conv_id) + suffix, data_torch)
|
||||
return
|
||||
|
||||
# ConvNeXt blocks are numbered per stage upstream and flat here
|
||||
if parts[0] == "stem":
|
||||
stage, blk, tail = 0, int(parts[2]), ".".join(parts[3:])
|
||||
else:
|
||||
assert parts[0] == "conv_layers" and parts[2] == "stage", name
|
||||
stage, blk, tail = int(parts[1]) + 1, int(parts[3]), ".".join(parts[4:])
|
||||
xid = sum(depths[:stage]) + blk
|
||||
|
||||
key = self._block_tensors[tail]
|
||||
if key == gguf.MODEL_TENSOR.A_ENC_CNX_DW and tail.endswith(".weight"):
|
||||
data_torch = data_torch.squeeze(1) # [C, 1, K] -> [C, K] for ggml_ssm_conv
|
||||
yield (gguf.TENSOR_NAMES[key].format(bid=enc_id, xid=xid) + suffix, data_torch)
|
||||
@@ -0,0 +1,234 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, MmprojModel, gguf, logger
|
||||
from .qwen import QwenModel, Qwen3Model
|
||||
|
||||
# token ids of the yue2_infer protocol (protocol.py)
|
||||
EOD = 151643
|
||||
ABC_START, ABC_END = 151847, 151848
|
||||
MUSIC_START, MUSIC_END = 151851, 151852
|
||||
CODEC_OFFSET, CODEC_SIZE = 151853, 32768
|
||||
LATENT_START = 184621
|
||||
|
||||
# the VAE is a separate repo (m-a-p/YuE2-Vae), downloaded into this subdir of the model dir
|
||||
VAE_DIR = "vae"
|
||||
|
||||
INSTRUCTIONS = {
|
||||
"off": "Generate music with codec tokens from the given conditions.",
|
||||
"melody": "Generate a melody-only ABC transcription without chord symbols, then generate music with codec tokens from the given conditions.",
|
||||
"full": "Generate a chord-annotated ABC transcription, then generate music with codec tokens from the given conditions.",
|
||||
}
|
||||
|
||||
# the text is "[Tags]\n<style>\n[Lyrics]\n<lyrics>", optionally followed by "\n[ABC]\n<score>" (cover mode)
|
||||
# voice picks the planning mode: full (default), melody or off
|
||||
TTS_TEMPLATE = (
|
||||
"{%- set text = messages[0]['content'] -%}"
|
||||
"{%- set mode = (voice | lower) if voice is defined and voice and (voice | lower) in ['full', 'melody', 'off'] else 'full' -%}"
|
||||
"{%- set parts = text.split('\\n[ABC]\\n') -%}"
|
||||
"{%- set instructions = {"
|
||||
+ ", ".join(f"'{k}': '{v}'" for k, v in INSTRUCTIONS.items())
|
||||
+ "} -%}"
|
||||
"{{- '<|endoftext|>' + instructions[mode] + '\\n' + parts[0].rstrip() + '\\n' -}}"
|
||||
"{%- if mode == 'off' -%}"
|
||||
"{{- '<abc></abc><|music_start|>' -}}"
|
||||
"{%- elif parts | length > 1 -%}"
|
||||
"{{- '<abc>' + parts[1] + '</abc><|music_start|>' -}}"
|
||||
"{%- else -%}"
|
||||
"{{- '<abc>' -}}"
|
||||
"{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
def _is_nar_tensor(name: str) -> bool:
|
||||
return ".nar_" in name or name.startswith(("llm2vae.", "vae2llm.", "time_embedder.", "latent_pos_embed."))
|
||||
|
||||
|
||||
@ModelBase.register("YuE2ForCausalLM")
|
||||
class YuE2TextModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
# the NAR path goes to the mmproj
|
||||
if _is_nar_tensor(item[0]):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
# the codes are sampled with classifier-free guidance, the uncond prompt runs as a second sequence
|
||||
self.gguf_writer.add_classifier_free_guidance(True)
|
||||
|
||||
def set_vocab(self):
|
||||
# checkpoint-native tiktoken file, see tokenization_yue2.py
|
||||
ranks: dict[bytes, int] = {}
|
||||
with open(self.dir_model / "qwen.tiktoken", "rb") as f:
|
||||
for line in f.read().splitlines():
|
||||
if line:
|
||||
tok, rank = line.split()
|
||||
ranks[base64.b64decode(tok)] = int(rank)
|
||||
if len(ranks) != EOD:
|
||||
raise ValueError(f"YuE2: expected {EOD} BPE tokens, got {len(ranks)}")
|
||||
|
||||
vocab_size = self.hparams["vocab_size"]
|
||||
tokens: list[str] = [""] * vocab_size
|
||||
toktypes: list[int] = [gguf.TokenType.UNUSED] * vocab_size
|
||||
merges: list[str] = []
|
||||
for tok, rank in ranks.items():
|
||||
tokens[rank] = QwenModel.token_bytes_to_string(tok)
|
||||
toktypes[rank] = gguf.TokenType.NORMAL
|
||||
if len(tok) > 1:
|
||||
merged = QwenModel.bpe(ranks, tok, max_rank=rank)
|
||||
assert len(merged) == 2
|
||||
merges.append(" ".join(map(QwenModel.token_bytes_to_string, merged)))
|
||||
|
||||
specials = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<R>", "<S>", "<X>", "<mask>", "<sep>"]
|
||||
specials += [f"<extra_{i}>" for i in range(200)]
|
||||
specials[204:206] = ["<abc>", "</abc>"]
|
||||
named = {EOD + i: s for i, s in enumerate(specials)}
|
||||
named[MUSIC_START] = "<|music_start|>"
|
||||
named[MUSIC_END] = "<|music_end|>"
|
||||
for i in range(CODEC_SIZE):
|
||||
named[CODEC_OFFSET + i] = f"<|codec_{i}|>"
|
||||
for i, s in enumerate(["<|latent_start|>", "<|latent_end|>", "<|latent_pad|>"]):
|
||||
named[LATENT_START + i] = s
|
||||
for i in range(EOD, vocab_size):
|
||||
tokens[i] = named.get(i, f"[PAD{i}]")
|
||||
toktypes[i] = gguf.TokenType.CONTROL if i in named else gguf.TokenType.UNUSED
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre("qwen2")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
self.gguf_writer.add_token_merges(merges)
|
||||
self.gguf_writer.add_bos_token_id(EOD)
|
||||
self.gguf_writer.add_eos_token_id(MUSIC_END)
|
||||
self.gguf_writer.add_pad_token_id(EOD)
|
||||
self.gguf_writer.add_add_bos_token(False)
|
||||
self.gguf_writer.add_chat_template([{"name": "tts", "template": TTS_TEMPLATE}])
|
||||
|
||||
|
||||
@ModelBase.register("YuE2ForCausalLM")
|
||||
class YuE2MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
# the NAR path: same shapes as the backbone layers
|
||||
return {k: self.global_config[k] for k in ("num_hidden_layers", "hidden_size", "intermediate_size",
|
||||
"num_attention_heads", "num_key_value_heads", "rms_norm_eps")}
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name = item[0]
|
||||
if name == "latent_pos_embed.pe":
|
||||
return None # sinusoidal, computed at runtime
|
||||
if _is_nar_tensor(name) or name == "model.norm.weight":
|
||||
return item
|
||||
return None
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.YUE2_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(hp["hidden_size"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(hp["intermediate_size"])
|
||||
self.gguf_writer.add_gen_audio_block_count(hp["num_hidden_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(hp["num_attention_heads"])
|
||||
self.gguf_writer.add_gen_audio_head_count_kv(hp["num_key_value_heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(hp["rms_norm_eps"])
|
||||
# rope theta, the ODE solver and the VAE hparams are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# read on the host, or small convs and projections of the flow / VAE graphs
|
||||
if not new_name.startswith("a.gen.nar.blk.") or n_dims < 2:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@staticmethod
|
||||
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
|
||||
return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
from safetensors.torch import load_file
|
||||
path = self.dir_model / VAE_DIR / "model.safetensors"
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"YuE2: VAE not found at {path}, download m-a-p/YuE2-Vae into it")
|
||||
logger.info(f"gguf: loading VAE decoder from '{path}'")
|
||||
sd = {k[len("decoder.layers."):]: v for k, v in load_file(str(path)).items() if k.startswith("decoder.layers.")}
|
||||
p = "a.gen.vae"
|
||||
|
||||
# SnakeBeta with log-scale params: x + 1/(exp(beta) + 1e-9) * sin(exp(alpha) * x)^2
|
||||
def snake(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
yield (f"{dst}.alpha", torch.exp(sd[src + ".alpha"].float()))
|
||||
yield (f"{dst}.beta", 1.0 / (torch.exp(sd[src + ".beta"].float()) + 1e-9))
|
||||
|
||||
yield (f"{p}.conv_in.weight", self._wn(sd, "0"))
|
||||
yield (f"{p}.conv_in.bias", sd["0.bias"])
|
||||
n_blk = sum(1 for k in sd if k.endswith(".layers.1.weight_g") and k.count(".") == 3)
|
||||
for b in range(n_blk):
|
||||
s = f"{b + 1}.layers"
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.0")
|
||||
yield (f"{p}.blk.{b}.up.weight", self._wn(sd, f"{s}.1"))
|
||||
yield (f"{p}.blk.{b}.up.bias", sd[f"{s}.1.bias"])
|
||||
for r in range(3):
|
||||
rs = f"{s}.{r + 2}.layers"
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake1", f"{rs}.0")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.weight", self._wn(sd, f"{rs}.1"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.bias", sd[f"{rs}.1.bias"])
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake2", f"{rs}.2")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.weight", self._wn(sd, f"{rs}.3"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.bias", sd[f"{rs}.3.bias"])
|
||||
yield from snake(f"{p}.post_snake", f"{n_blk + 1}")
|
||||
yield (f"{p}.conv_out.weight", self._wn(sd, f"{n_blk + 2}"))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name.startswith("a.gen.vae."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
p = "a.gen.nar"
|
||||
aux = {
|
||||
"model.norm.weight": f"{p}.output_norm.weight",
|
||||
"vae2llm.weight": f"{p}.inp.weight",
|
||||
"vae2llm.bias": f"{p}.inp.bias",
|
||||
"llm2vae.weight": f"{p}.out.weight",
|
||||
"llm2vae.bias": f"{p}.out.bias",
|
||||
"time_embedder.mlp.0.weight": f"{p}.time.0.weight",
|
||||
"time_embedder.mlp.0.bias": f"{p}.time.0.bias",
|
||||
"time_embedder.mlp.2.weight": f"{p}.time.2.weight",
|
||||
"time_embedder.mlp.2.bias": f"{p}.time.2.bias",
|
||||
}
|
||||
if name in aux:
|
||||
yield (aux[name], data_torch)
|
||||
return
|
||||
|
||||
# the NAR layers keep the libllama names, mtmd loads them as a text model
|
||||
assert bid is not None, name
|
||||
layer = {
|
||||
"nar_input_layernorm.weight": "attn_norm.weight",
|
||||
"nar_self_attn.q_proj.weight": "attn_q.weight",
|
||||
"nar_self_attn.k_proj.weight": "attn_k.weight",
|
||||
"nar_self_attn.v_proj.weight": "attn_v.weight",
|
||||
"nar_self_attn.o_proj.weight": "attn_output.weight",
|
||||
"nar_self_attn.q_norm.weight": "attn_q_norm.weight",
|
||||
"nar_self_attn.k_norm.weight": "attn_k_norm.weight",
|
||||
"nar_pre_mlp_layernorm.weight": "ffn_norm.weight",
|
||||
"nar_mlp.gate_proj.weight": "ffn_gate.weight",
|
||||
"nar_mlp.up_proj.weight": "ffn_up.weight",
|
||||
"nar_mlp.down_proj.weight": "ffn_down.weight",
|
||||
}
|
||||
key = name.split(".", 3)[3] # strip "model.layers.{bid}."
|
||||
yield (f"{p}.blk.{bid}.{layer[key]}", data_torch)
|
||||
@@ -0,0 +1,575 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, MmprojModel, LazyTorchTensor, gguf, logger
|
||||
|
||||
LM_DIR = Path("lm") / "zonos3_mini"
|
||||
|
||||
# codec decoder: quantizers and semantic head from the generator, the rest from the head checkpoint
|
||||
CODEC_GENERATOR = Path("autoencoder") / "generator.pt"
|
||||
CODEC_VARIANT = "keep2_plain"
|
||||
CODEC_HEAD = Path("autoencoder") / "heads" / CODEC_VARIANT / "step_0050000.pt"
|
||||
CODEC_STAGES = [0, 2, 4, 6, 8] # transformer stages of the decoder
|
||||
|
||||
DEFAULT_VOICE = "Savannah_en-American_F_1"
|
||||
|
||||
# speaker encoder (ownmect_s2), MECT_B2 blocks: (cnn depth, out channels, (stride f, stride t), expansion)
|
||||
SPK_CKPT = Path("speaker_encoders") / "ownmect_s2" / "ckpt_0037.pth"
|
||||
SPK_STAGES = [
|
||||
[(3, 32, (1, 1), 4)] * 2,
|
||||
[(3, 64, (2, 1), 2)] + [(3, 64, (1, 2), 2)] * 3,
|
||||
[(3, 128, (2, 1), 2)] + [(3, 128, (1, 2), 2)] * 4,
|
||||
[(3, 256, (2, 1), 1)] + [(2, 256, (1, 2), 1)] * 2,
|
||||
]
|
||||
VOICE_MAX_FRAMES = 900 # reference_max_frames of the reference
|
||||
|
||||
# GDN geometry, hard-coded in the reference (zonos3.reference.lm)
|
||||
GDN_HEADS, GDN_HEAD_DIM, GDN_KERNEL = 4, 128, 4
|
||||
|
||||
# the text table keeps the tokenizer ids shifted by this offset, ids 2 and 3 are BOS and EOS
|
||||
TEXT_OFFSET = 4
|
||||
TEXT_BOS, TEXT_EOS = 2, 3
|
||||
|
||||
|
||||
def _round_up(value: float, multiple: int) -> int:
|
||||
return multiple * ((int(value) + multiple - 1) // multiple)
|
||||
|
||||
|
||||
def _load_params(dir_model: Path) -> dict[str, Any] | None:
|
||||
path = dir_model / LM_DIR / "params.json"
|
||||
if not path.is_file():
|
||||
return None
|
||||
with open(path, encoding="utf-8") as f:
|
||||
params = json.load(f).get("model", {})
|
||||
return params if params.get("model_type") == "zonos2" else None
|
||||
|
||||
|
||||
def _is_zonos3_checkpoint(dir_model: Path) -> bool:
|
||||
return _load_params(dir_model) is not None
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_zonos3_checkpoint)
|
||||
def _load_zonos3_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected ZONOS3 checkpoint")
|
||||
params = _load_params(dir_model)
|
||||
assert params is not None
|
||||
for key, want in (("attention_pattern", "gdn3_global1"), ("residual_type", "full_attnres"),
|
||||
("moe_router_type", "linear"), ("moe_router_use_eda", False), ("use_xsa", False),
|
||||
("moe_end_from_layer", 0), ("speaker_enabled", True)):
|
||||
if params.get(key) != want:
|
||||
raise NotImplementedError(f"ZONOS3: unsupported {key} = {params.get(key)!r}, expected {want!r}")
|
||||
text_config = dict(params, hidden_size=params["dim"])
|
||||
return {"architectures": ["Zonos3Model"], "text_config": text_config, "audio_config": {"num_hidden_layers": 0}}
|
||||
|
||||
|
||||
def _index_lm_tensors(model: ModelBase) -> dict[str, Callable[[], Tensor]]:
|
||||
path = model.dir_model / LM_DIR / "model.pth"
|
||||
logger.info(f"gguf: indexing model part '{LM_DIR / 'model.pth'}'")
|
||||
state = torch.load(str(path), map_location="cpu", mmap=True, weights_only=True)
|
||||
tensors: dict[str, Callable[[], Tensor]] = {}
|
||||
for name, data in state.items():
|
||||
gen = (lambda data=data: LazyTorchTensor.from_eager(data)) if model.lazy else (lambda data=data: data)
|
||||
titem = model.filter_tensors((name, gen))
|
||||
model._source_tensor_names[name] = titem[0] if titem else None
|
||||
if titem:
|
||||
tensors[titem[0]] = titem[1]
|
||||
return tensors
|
||||
|
||||
|
||||
@ModelBase.register("Zonos3Model")
|
||||
class Zonos3TextModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.ZONOS3
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
hp = self.hparams
|
||||
self.n_embd = hp["dim"]
|
||||
self.n_codebooks = hp["n_codebooks"]
|
||||
self.audio_vocab = hp["codebook_size"] + 2 # codes, EOA, pad
|
||||
self.audio_pad_id = hp["audio_pad_id"]
|
||||
self.text_rows = hp["text_vocab"] + 1 # + the padding row, used by the generated rows
|
||||
self.cond_counts = [n + 1 for n in hp["condition_bucket_counts"]] # + the unknown bucket
|
||||
self.gdn_period = int(hp["attention_pattern"][len("gdn"):].split("_")[0]) + 1
|
||||
self.n_ff = _round_up(hp["ffn_dim_multiplier"] * hp["dim"], hp["multiple_of"])
|
||||
self.n_ff_exp = _round_up(hp["moe_ffn_dim_multiplier"] * hp["dim"], hp["moe_multiple_of"])
|
||||
self.n_latent = int(hp["dim"] * hp["moe_alpha"])
|
||||
self._embd: dict[str, Tensor] = {}
|
||||
self._router: dict[int, dict[str, Tensor]] = {}
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
return _index_lm_tensors(self)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
# audio heads and speaker projection go to the mmproj
|
||||
if name.startswith(("multi_output.", "speaker_projection.")):
|
||||
return None
|
||||
return item
|
||||
|
||||
def set_vocab(self):
|
||||
with open(self.dir_model / LM_DIR / "tokenizer" / "superbpe8192.json", encoding="utf-8") as f:
|
||||
tok = json.load(f)
|
||||
model = tok["model"]
|
||||
if model["type"] != "BPE" or model.get("byte_fallback") or tok.get("normalizer") is not None:
|
||||
raise NotImplementedError("ZONOS3: unexpected tokenizer type")
|
||||
|
||||
vocab: dict[str, int] = model["vocab"]
|
||||
n_bpe = len(vocab)
|
||||
if n_bpe + TEXT_OFFSET != self.hparams["text_vocab"]:
|
||||
raise ValueError(f"ZONOS3: tokenizer has {n_bpe} tokens, text_vocab is {self.hparams['text_vocab']}")
|
||||
added = {t["id"] for t in tok.get("added_tokens", [])}
|
||||
|
||||
tokens: list[str] = ["<|unused_0|>", "<|unused_1|>", "<|bos|>", "<|eos|>"]
|
||||
toktypes: list[int] = [gguf.TokenType.CONTROL] * TEXT_OFFSET
|
||||
inv = {i: t for t, i in vocab.items()}
|
||||
for i in range(n_bpe):
|
||||
tokens.append(inv[i])
|
||||
toktypes.append(gguf.TokenType.CONTROL if i in added else gguf.TokenType.NORMAL)
|
||||
|
||||
# padding row of the text table (the text column of every generated row)
|
||||
tokens.append("<|text_pad|>")
|
||||
toktypes.append(gguf.TokenType.CONTROL)
|
||||
|
||||
# condition rows, the last bucket of each table is "unknown"
|
||||
for cond, n in zip(self.hparams["condition_names"], self.cond_counts):
|
||||
tokens += [f"<|{cond}_{i}|>" for i in range(n - 1)] + [f"<|{cond}_unk|>"]
|
||||
toktypes += [gguf.TokenType.CONTROL] * n
|
||||
|
||||
merges = [" ".join(m) if isinstance(m, list) else m for m in model["merges"]]
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre("zonos3")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
self.gguf_writer.add_token_merges(merges)
|
||||
self.gguf_writer.add_bos_token_id(TEXT_BOS)
|
||||
self.gguf_writer.add_eos_token_id(TEXT_EOS)
|
||||
self.gguf_writer.add_pad_token_id(TEXT_OFFSET + 1) # <|pad|> of the tokenizer
|
||||
# the reference wraps the text as [BOS] + ids + [EOS]
|
||||
self.gguf_writer.add_add_bos_token(True)
|
||||
self.gguf_writer.add_add_eos_token(True)
|
||||
# the reference normalizes the text with NeMo, the template covers the English numbers
|
||||
template_path = Path(__file__).parent.parent / "models" / "templates" / "Zyphra-ZONOS3-Mini-tts.jinja"
|
||||
with open(template_path, "r", encoding="utf-8") as f:
|
||||
self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}])
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
n_layer = self.block_count
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_block_count(n_layer)
|
||||
self.gguf_writer.add_context_length(hp["max_seqlen"])
|
||||
self.gguf_writer.add_embedding_length(self.n_embd)
|
||||
self.gguf_writer.add_feed_forward_length(self.n_ff)
|
||||
self.gguf_writer.add_head_count(hp["n_heads"])
|
||||
self.gguf_writer.add_head_count_kv(hp["n_kv_heads"])
|
||||
self.gguf_writer.add_key_length(hp["head_dim"])
|
||||
self.gguf_writer.add_value_length(hp["head_dim"])
|
||||
self.gguf_writer.add_rope_dimension_count(hp["head_dim"])
|
||||
self.gguf_writer.add_rope_freq_base(hp["rope_theta"])
|
||||
self.gguf_writer.add_layer_norm_rms_eps(hp["norm_eps"])
|
||||
self.gguf_writer.add_full_attention_interval(self.gdn_period)
|
||||
|
||||
# gated delta net
|
||||
self.gguf_writer.add_ssm_conv_kernel(GDN_KERNEL)
|
||||
self.gguf_writer.add_ssm_inner_size(GDN_HEADS * GDN_HEAD_DIM)
|
||||
self.gguf_writer.add_ssm_state_size(GDN_HEAD_DIM)
|
||||
self.gguf_writer.add_ssm_time_step_rank(GDN_HEADS)
|
||||
self.gguf_writer.add_ssm_group_count(GDN_HEADS)
|
||||
|
||||
# latent MoE, the gating (softmax over the selected logits) is hard-coded in libllama
|
||||
self.gguf_writer.add_leading_dense_block_count(hp["moe_start_from_layer"])
|
||||
self.gguf_writer.add_expert_count(hp["moe_n_experts"])
|
||||
self.gguf_writer.add_expert_used_count(hp["moe_router_topk"])
|
||||
self.gguf_writer.add_expert_feed_forward_length(self.n_ff_exp)
|
||||
self.gguf_writer.add_moe_latent_size(self.n_latent)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# router logits pick the experts, keep them exact
|
||||
if new_name.endswith(("ffn_gate_inp.weight", "exp_probs_b.bias")):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def _token_embd(self) -> Tensor:
|
||||
n_cb = self.n_codebooks
|
||||
pad = sum(self._embd[f"multi_embedder.embedders.{cb}.weight"][self.audio_pad_id].float() for cb in range(n_cb))
|
||||
text = self._embd[f"multi_embedder.embedders.{n_cb}.weight"].float()
|
||||
if text.shape[0] != self.text_rows:
|
||||
raise ValueError(f"ZONOS3: text table has {text.shape[0]} rows, expected {self.text_rows}")
|
||||
conds = [self._embd[f"condition_embeddings.{i}.weight"].float() for i in range(len(self.cond_counts))]
|
||||
for t, n in zip(conds, self.cond_counts):
|
||||
assert t.shape[0] == n
|
||||
# a text row is [pad x n_codebooks, text_id], condition rows replace the summed row
|
||||
return torch.cat([text + pad] + conds, dim=0)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
T = gguf.MODEL_TENSOR
|
||||
hp = self.hparams
|
||||
n_layer = self.block_count
|
||||
|
||||
if name.startswith(("multi_embedder.", "condition_embeddings.")):
|
||||
self._embd[name] = data_torch
|
||||
if len(self._embd) == self.n_codebooks + 1 + len(self.cond_counts):
|
||||
yield (self.format_tensor_name(T.TOKEN_EMBD), self._token_embd())
|
||||
self._embd.clear()
|
||||
return
|
||||
|
||||
if name == "block_attnres_queries":
|
||||
# one query per routing point: 2*il before the mixer, 2*il+1 before the FFN, the last one before out_norm
|
||||
assert data_torch.shape[0] == 2 * n_layer + 1
|
||||
for il in range(n_layer):
|
||||
yield (self.format_tensor_name(T.ATTN_RES_SCORE, il), data_torch[2 * il])
|
||||
yield (self.format_tensor_name(T.FFN_RES_SCORE, il), data_torch[2 * il + 1])
|
||||
yield (self.format_tensor_name(T.OUTPUT_RES_SCORE), data_torch[2 * n_layer])
|
||||
return
|
||||
|
||||
if name == "out_norm.weight":
|
||||
yield (self.format_tensor_name(T.OUTPUT_NORM), data_torch)
|
||||
return
|
||||
|
||||
assert bid is not None, name
|
||||
key = name.split(".", 2)[2] # strip "layers.{bid}."
|
||||
|
||||
if key == "attention_norm.weight":
|
||||
yield (self.format_tensor_name(T.ATTN_NORM, bid), data_torch)
|
||||
return
|
||||
if key == "ffn_norm.weight":
|
||||
yield (self.format_tensor_name(T.FFN_NORM, bid), data_torch)
|
||||
return
|
||||
|
||||
# gated delta net
|
||||
if key == "attention.in_proj.weight":
|
||||
# split the fused [qkv | z | a | b] weight into the qwen35 GDN tensors
|
||||
# attn_gate holds z, per-head output gate in the attention layers
|
||||
inner = GDN_HEADS * GDN_HEAD_DIM
|
||||
qkv, z, a, b = data_torch.split([3 * inner, inner, GDN_HEADS, GDN_HEADS], dim=0)
|
||||
yield (self.format_tensor_name(T.ATTN_QKV, bid), qkv)
|
||||
yield (self.format_tensor_name(T.ATTN_GATE, bid), z)
|
||||
yield (self.format_tensor_name(T.SSM_ALPHA, bid), a)
|
||||
yield (self.format_tensor_name(T.SSM_BETA, bid), b)
|
||||
return
|
||||
if key == "attention.conv.weight":
|
||||
yield (self.format_tensor_name(T.SSM_CONV1D, bid), data_torch.squeeze(1))
|
||||
return
|
||||
if key == "attention.A_log":
|
||||
yield (self.format_tensor_name(T.SSM_A, bid, suffix=""), -torch.exp(data_torch.float()))
|
||||
return
|
||||
if key == "attention.dt_bias":
|
||||
yield (self.format_tensor_name(T.SSM_DT, bid, suffix=".bias"), data_torch)
|
||||
return
|
||||
if key == "attention.out_weight":
|
||||
yield (self.format_tensor_name(T.SSM_NORM, bid), data_torch)
|
||||
return
|
||||
if key == "attention.wo.weight" and (bid + 1) % self.gdn_period != 0:
|
||||
yield (self.format_tensor_name(T.SSM_OUT, bid), data_torch)
|
||||
return
|
||||
|
||||
# global attention
|
||||
if key == "attention.wq.weight":
|
||||
yield (self.format_tensor_name(T.ATTN_Q, bid), data_torch)
|
||||
return
|
||||
if key == "attention.wkv.weight":
|
||||
yield (self.format_tensor_name(T.ATTN_K, bid), data_torch[0])
|
||||
yield (self.format_tensor_name(T.ATTN_V, bid), data_torch[1])
|
||||
return
|
||||
if key == "attention.wo.weight":
|
||||
yield (self.format_tensor_name(T.ATTN_OUT, bid), data_torch)
|
||||
return
|
||||
if key == "attention.gater.weight":
|
||||
# one sigmoid gate per head, same tensor name as the GDN z gate
|
||||
yield (self.format_tensor_name(T.ATTN_GATE, bid), data_torch)
|
||||
return
|
||||
if key == "attention.temp":
|
||||
# Q = rms_norm(q) * |temp|, the per-head scale is stored as the Q norm weight
|
||||
temp = data_torch.float().abs().reshape(hp["n_heads"], 1).expand(hp["n_heads"], hp["head_dim"])
|
||||
yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), temp.contiguous())
|
||||
return
|
||||
|
||||
# dense FFN (layer 0): out = w_out(w_in[0](x) * silu(w_in[1](x)))
|
||||
if key == "feed_forward.w_in.weight":
|
||||
yield (self.format_tensor_name(T.FFN_UP, bid), data_torch[0])
|
||||
yield (self.format_tensor_name(T.FFN_GATE, bid), data_torch[1])
|
||||
return
|
||||
if key == "feed_forward.w_out.weight":
|
||||
yield (self.format_tensor_name(T.FFN_DOWN, bid), data_torch)
|
||||
return
|
||||
|
||||
# latent MoE
|
||||
if key in ("feed_forward.router.router_linear.weight", "feed_forward.router.rmsnorm_eda.weight"):
|
||||
# logits = router_linear(rms_norm(x) * w), the norm weight is folded into the router columns
|
||||
r = self._router.setdefault(bid, {})
|
||||
r[key] = data_torch
|
||||
if len(r) == 2:
|
||||
w = r["feed_forward.router.router_linear.weight"].float() * r["feed_forward.router.rmsnorm_eda.weight"].float()[None, :]
|
||||
yield (self.format_tensor_name(T.FFN_GATE_INP, bid), w)
|
||||
del self._router[bid]
|
||||
return
|
||||
if key == "feed_forward.router.balancing_biases":
|
||||
# the reference selects on logits - bias
|
||||
yield (self.format_tensor_name(T.FFN_EXP_PROBS_B, bid, suffix=".bias"), -data_torch.float())
|
||||
return
|
||||
if key == "feed_forward.latent_down.weight":
|
||||
yield (self.format_tensor_name(T.MOE_LATENT_DOWN, bid), data_torch)
|
||||
return
|
||||
if key == "feed_forward.latent_up.weight":
|
||||
yield (self.format_tensor_name(T.MOE_LATENT_UP, bid), data_torch)
|
||||
return
|
||||
if key == "feed_forward.experts.gate_up_proj":
|
||||
gate, up = data_torch.chunk(2, dim=1)
|
||||
yield (self.format_tensor_name(T.FFN_GATE_EXP, bid), gate)
|
||||
yield (self.format_tensor_name(T.FFN_UP_EXP, bid), up)
|
||||
return
|
||||
if key == "feed_forward.experts.down_proj":
|
||||
yield (self.format_tensor_name(T.FFN_DOWN_EXP, bid), data_torch)
|
||||
return
|
||||
|
||||
raise ValueError(f"ZONOS3: unexpected tensor {name}")
|
||||
|
||||
|
||||
@ModelBase.register("Zonos3Model")
|
||||
class Zonos3MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
hp = self.global_config["text_config"]
|
||||
self.n_codebooks = hp["n_codebooks"]
|
||||
self.text_vocab = hp["text_vocab"]
|
||||
self._embd: dict[str, Tensor] = {}
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
return _index_lm_tensors(self)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if name.startswith(("multi_output.", "multi_embedder.", "speaker_projection.")):
|
||||
return item
|
||||
return None
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
|
||||
# speaker encoder for voice cloning, the front-end hparams are hard-coded in tools/mtmd
|
||||
self.gguf_writer.add_clip_has_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.ZONOS3_SPKENC)
|
||||
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_audio_num_mel_bins(80)
|
||||
self.gguf_writer.add_audio_block_count(0)
|
||||
self.gguf_writer.add_audio_embedding_length(192)
|
||||
self.gguf_writer.add_audio_head_count(1)
|
||||
self.gguf_writer.add_audio_feed_forward_length(192)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
|
||||
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.ZONOS3_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(0)
|
||||
self.gguf_writer.add_gen_audio_block_count(0)
|
||||
self.gguf_writer.add_gen_audio_head_count(0)
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(self.global_config["text_config"]["norm_eps"])
|
||||
self.gguf_writer.add_gen_audio_model_variant(CODEC_VARIANT)
|
||||
if any(self.dir_model.glob(f"gguf/voices/*/{DEFAULT_VOICE}.codes.npy")):
|
||||
self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE)
|
||||
# softcap and the codec hparams are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# read on the host, or used by get_rows / ggml_ssm_conv / the conv ops of the speaker encoder
|
||||
if new_name.startswith(("voice.", "a.spk.")) or new_name.endswith((".table", ".dw.weight", ".mixer.weight", ".resample.weight")):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@staticmethod
|
||||
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
|
||||
return (g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))).squeeze(-1)
|
||||
|
||||
def _speaker_encoder(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# same layout as the reference ggml runtime (tools/convert_gguf.py speaker-encoder)
|
||||
sd = torch.load(self.dir_model / SPK_CKPT, map_location="cpu", weights_only=True)["modules"]["spk_model"]
|
||||
sd = {k[len("model."):]: v.double() for k, v in sd.items() if not k.endswith("num_batches_tracked")}
|
||||
p = "a.spk"
|
||||
|
||||
def bn(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
scale = sd[src + ".weight"] / torch.sqrt(sd[src + ".running_var"] + 1e-5)
|
||||
yield (f"{p}.{dst}.scale", scale)
|
||||
yield (f"{p}.{dst}.shift", sd[src + ".bias"] - sd[src + ".running_mean"] * scale)
|
||||
|
||||
def put(dst: str, t: Tensor) -> tuple[str, Tensor]:
|
||||
return (f"{p}.{dst}", t)
|
||||
|
||||
fe = "feature_extractor."
|
||||
yield put("fe.head.conv.weight", sd[fe + "head.0.weight"])
|
||||
yield put("fe.head.conv.bias", sd[fe + "head.0.bias"])
|
||||
yield from bn("fe.head.bn", fe + "head.1")
|
||||
c_in, f_in = 32, 80
|
||||
for si, stage in enumerate(SPK_STAGES):
|
||||
for k, (depth, out_ch, (s_f, s_t), expansion) in enumerate(stage):
|
||||
src, dst = f"{fe}mect_block{si + 1}.{k}.", f"fe.b{si + 1}.{k}."
|
||||
for r in range(depth):
|
||||
cs, cd = f"{src}cnn_block.{r}.", f"{dst}cnn.{r}."
|
||||
# grouped 3x3 convs, one depthwise kernel per input slot of a group
|
||||
for conv in ("conv1", "conv2"):
|
||||
w = sd[cs + conv + ".weight"]
|
||||
for j in range(w.shape[1]):
|
||||
yield put(f"{cd}{conv}.w{j}", w[:, j:j + 1])
|
||||
for i in (1, 2):
|
||||
yield put(f"{cd}pw{i}.weight", sd[f"{cs}conv_pw{i}.weight"])
|
||||
yield put(f"{cd}pw{i}.bias", sd[f"{cs}conv_pw{i}.bias"])
|
||||
yield from bn(f"{cd}bn{i}", f"{cs}bn{i}")
|
||||
if cs + "shortcut.0.weight" in sd:
|
||||
yield put(cd + "sc.weight", sd[cs + "shortcut.0.weight"])
|
||||
yield from bn(cd + "sc.bn", cs + "shortcut.1")
|
||||
if expansion > 1:
|
||||
yield put(dst + "reduce.weight", sd[src + "reduce_conv.0.weight"])
|
||||
yield put(dst + "reduce.bias", sd[src + "reduce_conv.0.bias"])
|
||||
yield from bn(dst + "reduce.bn", src + "reduce_conv.1")
|
||||
yield put(dst + "rproj.weight", sd[src + "reduce_project.0.weight"].squeeze(-1))
|
||||
yield put(dst + "rproj.bias", sd[src + "reduce_project.0.bias"])
|
||||
yield put(dst + "rproj.ln.weight", sd[src + "reduce_project.1.weight"])
|
||||
yield put(dst + "rproj.ln.bias", sd[src + "reduce_project.1.bias"])
|
||||
yield put(dst + "pos.dw.weight", sd[src + "pos_conv.pos_conv.0.weight"].squeeze(1))
|
||||
yield put(dst + "pos.dw.bias", sd[src + "pos_conv.pos_conv.0.bias"])
|
||||
yield from bn(dst + "pos.bn", src + "pos_conv.pos_conv.1")
|
||||
yield put(dst + "pos.pw.weight", sd[src + "pos_conv.pos_conv.3.weight"].squeeze(-1))
|
||||
yield put(dst + "pos.pw.bias", sd[src + "pos_conv.pos_conv.3.bias"])
|
||||
tr = src + "transformer."
|
||||
for a, b in (("q", "q_proj"), ("k", "k_proj"), ("v", "v_proj"), ("o", "out_proj")):
|
||||
yield put(f"{dst}attn.{a}.weight", sd[f"{tr}attention.{b}.weight"])
|
||||
yield put(f"{dst}attn.{a}.bias", sd[f"{tr}attention.{b}.bias"])
|
||||
yield put(dst + "ln1.weight", sd[tr + "layer_norm.weight"])
|
||||
yield put(dst + "ln1.bias", sd[tr + "layer_norm.bias"])
|
||||
moe = tr + "feed_forward.intermediate_dense.token_moe."
|
||||
yield put(dst + "moe.gate.weight", sd[moe + "gate.weight"])
|
||||
yield put(dst + "moe.experts.weight", sd[moe + "experts.weight"])
|
||||
yield put(dst + "ffn_out.weight", sd[tr + "feed_forward.output_dense.weight"])
|
||||
yield put(dst + "ffn_out.bias", sd[tr + "feed_forward.output_dense.bias"])
|
||||
yield put(dst + "ln2.weight", sd[tr + "final_layer_norm.weight"])
|
||||
yield put(dst + "ln2.bias", sd[tr + "final_layer_norm.bias"])
|
||||
yield put(dst + "eproj.weight", sd[src + "expansion_project.0.weight"].squeeze(-1))
|
||||
yield put(dst + "eproj.bias", sd[src + "expansion_project.0.bias"])
|
||||
c_in, f_in = c_in * s_f, f_in // s_f
|
||||
# the stage outputs are mixed by softmax weights
|
||||
yield put("fe.dense_w", torch.softmax(sd[fe + "dense3.inputs_weights"], dim=1)[0, :, :, 0])
|
||||
yield put("pool.l1.weight", sd["encoding_layers.linear1.weight"].squeeze(-1))
|
||||
yield put("pool.l1.bias", sd["encoding_layers.linear1.bias"])
|
||||
yield put("pool.l2.weight", sd["encoding_layers.linear2.weight"].squeeze(-1))
|
||||
yield put("pool.l2.bias", sd["encoding_layers.linear2.bias"])
|
||||
yield from bn("bn", "bn")
|
||||
yield put("linear.weight", sd["linear.weight"])
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
yield from self._speaker_encoder()
|
||||
logger.info(f"gguf: loading codec decoder from '{CODEC_GENERATOR}' and '{CODEC_HEAD}'")
|
||||
gen = torch.load(self.dir_model / CODEC_GENERATOR, map_location="cpu", mmap=True, weights_only=True)["generator"]
|
||||
head = torch.load(self.dir_model / CODEC_HEAD, map_location="cpu", mmap=True, weights_only=True)["trainable"]
|
||||
p = "a.gen.wav"
|
||||
|
||||
def convnext(dst: str, src: dict[str, Tensor], prefix: str) -> Iterable[tuple[str, Tensor]]:
|
||||
yield (f"{dst}.dw.weight", src[f"{prefix}.dwconv.weight"].squeeze(1))
|
||||
yield (f"{dst}.dw.bias", src[f"{prefix}.dwconv.bias"])
|
||||
yield (f"{dst}.norm.weight", src[f"{prefix}.norm.weight"])
|
||||
yield (f"{dst}.norm.bias", src[f"{prefix}.norm.bias"])
|
||||
yield (f"{dst}.pw1.weight", src[f"{prefix}.pwconv1.weight"])
|
||||
yield (f"{dst}.pw1.bias", src[f"{prefix}.pwconv1.bias"])
|
||||
yield (f"{dst}.pw2.weight", src[f"{prefix}.pwconv2.weight"])
|
||||
yield (f"{dst}.pw2.bias", src[f"{prefix}.pwconv2.bias"])
|
||||
|
||||
# RVQ codebooks folded through their out_proj, the biases are added once
|
||||
def rvq(prefix: str) -> tuple[Tensor, Tensor]:
|
||||
return gen[prefix + ".codebook.weight"].float() @ self._wn(gen, prefix + ".out_proj").T, gen[prefix + ".out_proj.bias"].float()
|
||||
|
||||
t, b = rvq("quantizer.semantic_quantizer.quantizers.0")
|
||||
yield (f"{p}.sem.table", t)
|
||||
yield (f"{p}.sem.bias", b)
|
||||
res = [rvq(f"quantizer.quantizer.quantizers.{i}") for i in range(self.n_codebooks - 1)]
|
||||
yield (f"{p}.res.table", torch.cat([t for t, _ in res], dim=0))
|
||||
# one bias per residual codebook, a frame decoded before all its codes are known leaves the missing ones out
|
||||
yield (f"{p}.res.bias", torch.stack([b for _, b in res]))
|
||||
|
||||
n_sem = sum(1 for k in gen if k.startswith("semantic_head.") and k.endswith(".dwconv.weight"))
|
||||
for i in range(n_sem):
|
||||
yield from convnext(f"{p}.sem.{i}", gen, f"semantic_head.{i}")
|
||||
yield (f"{p}.sem.out.weight", self._wn(gen, f"semantic_head.{n_sem}"))
|
||||
yield (f"{p}.sem.out.bias", gen[f"semantic_head.{n_sem}.bias"])
|
||||
|
||||
names = {"norm1.weight": "norm1.weight", "norm1.bias": "norm1.bias", "qkv.weight": "qkv.weight",
|
||||
"attention_out.weight": "wo.weight", "attention_scale": "attn_scale", "norm2.weight": "norm2.weight",
|
||||
"norm2.bias": "norm2.bias", "ffn.0.weight": "ffn1.weight", "ffn.3.weight": "ffn2.weight", "ffn_scale": "ffn_scale"}
|
||||
for si, s in enumerate(CODEC_STAGES):
|
||||
yield (f"{p}.stage.{si}.in.weight", head[f"stages.{s}.input_proj.weight"])
|
||||
if f"stages.{s}.output_proj.weight" in head:
|
||||
yield (f"{p}.stage.{si}.out.weight", head[f"stages.{s}.output_proj.weight"])
|
||||
n_layer = sum(1 for k in head if k.startswith(f"stages.{s}.layers.") and k.endswith(".qkv.weight"))
|
||||
for il in range(n_layer):
|
||||
for src, dst in names.items():
|
||||
yield (f"{p}.stage.{si}.blk.{il}.{dst}", head[f"stages.{s}.layers.{il}.{src}"])
|
||||
|
||||
n_head = sum(1 for k in head if k.startswith("head.blocks.") and k.endswith(".dwconv.weight"))
|
||||
for i in range(n_head):
|
||||
yield from convnext(f"{p}.head.{i}", head, f"head.blocks.{i}")
|
||||
yield (f"{p}.head.norm.weight", head["head.norm.weight"])
|
||||
yield (f"{p}.head.norm.bias", head["head.norm.bias"])
|
||||
# both iSTFT branches in one projection
|
||||
yield (f"{p}.head.out.weight", torch.cat([head["head.outs.0.weight"], head["head.outs.1.weight"]], dim=0))
|
||||
yield (f"{p}.head.out.bias", torch.cat([head["head.outs.0.bias"], head["head.outs.1.bias"]], dim=0))
|
||||
# read on the host
|
||||
yield (f"{p}.mixer.weight", head["head.mixer.weight"].squeeze(0))
|
||||
yield (f"{p}.resample.weight", gen["to_external.kernel"].squeeze(1))
|
||||
|
||||
# "reference" speaker mode, the default of the reference: the voice is a prefix of delayed code rows
|
||||
def _voices(self, tables: list[Tensor]) -> Iterable[tuple[str, Tensor]]:
|
||||
n_cb, pad = self.n_codebooks, self.global_config["text_config"]["audio_pad_id"]
|
||||
for path in sorted((self.dir_model / "gguf" / "voices").glob("*/*.codes.npy")):
|
||||
codes = torch.from_numpy(np.load(path, allow_pickle=False).astype(np.int64))
|
||||
if codes.ndim != 2 or codes.shape[1] != n_cb or codes.min() < 0 or codes.max() >= pad - 1:
|
||||
logger.warning(f"voice {path.name}: skipped, expected [frames, {n_cb}] codes")
|
||||
continue
|
||||
codes = codes[:VOICE_MAX_FRAMES]
|
||||
n_frames = codes.shape[0]
|
||||
# codebook cb of frame t is in row t + cb, the other cells are pad
|
||||
rows = torch.full((n_frames + n_cb - 1, n_cb), pad, dtype=torch.int64)
|
||||
for cb in range(n_cb):
|
||||
rows[cb:cb + n_frames, cb] = codes[:, cb]
|
||||
embd = sum(tables[cb][rows[:, cb]] for cb in range(n_cb))
|
||||
yield (f"voice.{path.name[:-len('.codes.npy')]}", embd)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
if name.startswith(("a.gen.wav.", "a.spk.")):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
# the speaker encoder ends with the speaker projection of the backbone
|
||||
if name.startswith("speaker_projection."):
|
||||
yield ("a.spk.proj." + name.split(".")[-1], data_torch)
|
||||
return
|
||||
|
||||
if name == "multi_output.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch)
|
||||
return
|
||||
|
||||
assert name.startswith("multi_embedder.")
|
||||
self._embd[name] = data_torch
|
||||
if len(self._embd) < self.n_codebooks + 1:
|
||||
return
|
||||
# every row built by the mmproj has text id = text_vocab, so that row is folded into codebook 0
|
||||
tables = [self._embd[f"multi_embedder.embedders.{cb}.weight"].float() for cb in range(self.n_codebooks)]
|
||||
tables[0] = tables[0] + self._embd[f"multi_embedder.embedders.{self.n_codebooks}.weight"][self.text_vocab].float()
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), torch.cat(tables, dim=0))
|
||||
yield from self._voices([LazyTorchTensor.to_eager(t) for t in tables])
|
||||
self._embd.clear()
|
||||
@@ -167,6 +167,7 @@ models = [
|
||||
{"name": "ufakzeka", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ufakai/ufakzeka-1", },
|
||||
{"name": "mmbert", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jhu-clsp/mmBERT-base", },
|
||||
{"name": "k2-horizon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/IFM/K2-Horizon-36B", },
|
||||
{"name": "inkling2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/thinkingmachines/Inkling-2-Small", },
|
||||
]
|
||||
|
||||
# some models are known to be broken upstream, so we will skip them as exceptions
|
||||
|
||||
+414
-194
@@ -3,9 +3,9 @@
|
||||
> [!NOTE]
|
||||
> Performance and memory optimizations, accuracy validation, broader quantization coverage, broader operator and model support are work in progress.
|
||||
|
||||
[OpenVINO](https://docs.openvino.ai/) is an open-source toolkit for optimizing and deploying high-performance AI inference, specifically designed for Intel hardware, including CPUs, GPUs, and NPUs, in the cloud, on-premises, and on the edge. [OpenVINO backend for llama.cpp](../../ggml/src/ggml-openvino) enables hardware-accelerated inference on **Intel® CPUs, GPUs, and NPUs** while remaining compatible with the existing **GGUF model ecosystem**. The backend translates GGML compute graphs into OpenVINO graphs and leverages graph compilation, kernel fusion, and device-specific optimizations to improve inference performance on supported Intel hardware.
|
||||
[OpenVINO](https://docs.openvino.ai/) is an open-source toolkit for optimizing and deploying high-performance AI inference, specifically designed for Intel hardware, including CPUs, GPUs (integrated and discrete), and NPUs, in the cloud, on-premises, and on the edge. The [OpenVINO backend](../../ggml/src/ggml-openvino) runs the same GGUF model files on **Intel® CPUs, GPUs and NPUs**, without changes to the model or the rest of llama.cpp.
|
||||
|
||||
The OpenVINO backend is implemented in `ggml/src/ggml-openvino` and provides a translation layer for core GGML operations. The OpenVINO backend replaces the standard GGML graph execution path with Intel's OpenVINO inference engine. This approach allows the same GGUF model file to run on Intel CPUs, Intel GPUs (integrated and discrete), and Intel NPUs without changes to the model or the rest of the llama.cpp stack. When a `ggml_cgraph` is dispatched to OpenVINO backend, it:
|
||||
When a `ggml_cgraph` is dispatched to the OpenVINO backend, it:
|
||||
|
||||
- Walks the GGML graph and identifies inputs, outputs, weights, and KV cache tensors.
|
||||
- Translates the GGML operations into an `ov::Model` using OpenVINO's frontend API.
|
||||
@@ -16,9 +16,10 @@ For guidance on contributing to the OpenVINO backend, see the [OpenVINO Backend
|
||||
|
||||
## Contents
|
||||
|
||||
- [Quick Start](#quick-start)
|
||||
- [Supported Devices](#supported-devices)
|
||||
- [Supported Model Precisions](#supported-model-precisions)
|
||||
- [Supported Llama.cpp Tools](#supported-llamacpp-tools)
|
||||
- [Supported llama.cpp Tools](#supported-llamacpp-tools)
|
||||
- [Validated Models](#validated-models)
|
||||
- [Build Instructions](#build-instructions)
|
||||
- [0. Prerequisites](#0-prerequisites)
|
||||
@@ -29,10 +30,44 @@ For guidance on contributing to the OpenVINO backend, see the [OpenVINO Backend
|
||||
- [3. Download Sample Model](#3-download-sample-model)
|
||||
- [4. Run Inference with OpenVINO Backend](#4-run-inference-with-openvino-backend)
|
||||
- [5. Docker Build](#5-docker-build)
|
||||
- [GGML OpenVINO Backend Runtime Configurations](#ggml-openvino-backend-runtime-configurations)
|
||||
- [Runtime Configuration](#runtime-configuration)
|
||||
- [Basic](#basic)
|
||||
- [Plugin properties](#plugin-properties)
|
||||
- [Advanced tuning](#advanced-tuning)
|
||||
- [Compiled model cache](#compiled-model-cache)
|
||||
- [Deprecated](#deprecated)
|
||||
- [Debugging and development](#debugging-and-development)
|
||||
- [Example Usage](#example-usage)
|
||||
- [GPU Inference with Profiling](#gpu-inference-with-profiling)
|
||||
- [Known Limitations](#known-limitations)
|
||||
- [Work in Progress](#work-in-progress)
|
||||
|
||||
## Quick Start
|
||||
|
||||
On Ubuntu, with the [prerequisites](#0-prerequisites) and the [OpenVINO Runtime](#1-install-openvino-runtime) installed:
|
||||
|
||||
```bash
|
||||
# 1. Build
|
||||
git clone https://github.com/ggml-org/llama.cpp
|
||||
cd llama.cpp
|
||||
source /opt/intel/openvino/setupvars.sh
|
||||
cmake -B build/ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON
|
||||
cmake --build build/ReleaseOV --parallel
|
||||
|
||||
# 2. Download a model
|
||||
mkdir -p ~/models/
|
||||
wget https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf \
|
||||
-O ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
|
||||
# 3. Run on CPU, GPU or NPU
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024
|
||||
```
|
||||
|
||||
The [Ubuntu Build Script](#ubuntu-build-script) and the [Windows Build Script](#windows-build-script) do the setup in one step: they install the prerequisites and the OpenVINO Runtime, and build llama.cpp with the OpenVINO backend.
|
||||
|
||||
See [Build Instructions](#build-instructions) for Windows and Docker, and [Runtime Configuration](#runtime-configuration) for the environment variables.
|
||||
|
||||
## Supported Devices
|
||||
|
||||
OpenVINO backend supports the following hardware:
|
||||
@@ -41,7 +76,7 @@ OpenVINO backend supports the following hardware:
|
||||
- Intel GPUs (integrated and discrete)
|
||||
- Intel NPUs
|
||||
|
||||
Although OpenVINO supports a wide range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), the llama.cpp OpenVINO backend has been validated specifically on AI PCs such as the Intel® Core™ Ultra Series 1 and Series 2.
|
||||
Although OpenVINO supports a wide range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), the llama.cpp OpenVINO backend has been validated specifically on AI PCs such as the Intel® Core™ Ultra Series 2 (Lunar Lake) and Series 3 (Panther Lake).
|
||||
|
||||
## Supported Model Precisions
|
||||
|
||||
@@ -51,101 +86,123 @@ Although OpenVINO supports a wide range of [Intel hardware](https://docs.openvin
|
||||
- `Q4_0`
|
||||
- `Q4_1`
|
||||
- `Q4_K`
|
||||
- `Q4_K_M`
|
||||
- `Q5_K` (converted to `Q8_0_C` at runtime by default)
|
||||
- `Q6_K` (converted to `Q8_0_C` at runtime by default)
|
||||
- `Q5_0` (converted at runtime, see the details below)
|
||||
- `Q5_1`
|
||||
- `Q5_K` (converted at runtime, see the details below)
|
||||
- `Q6_K` (converted at runtime, see the details below)
|
||||
|
||||
> [!NOTE]
|
||||
> Accuracy validation and performance optimizations for quantized models are a work in progress.
|
||||
`Q4_K_M` files mix `Q4_K` tensors with `Q6_K` tensors, and sometimes `Q5_K` tensors.
|
||||
|
||||
**CPU and GPU Quantization Details:**
|
||||
- `Q5_K` and `Q6_K` tensors are converted to `Q8_0_C`
|
||||
<details>
|
||||
<summary>Quantization details for each device</summary>
|
||||
|
||||
**NPU Quantization Details:**
|
||||
- Primary supported quantization scheme is `Q4_0`
|
||||
- `Q6_K` tensors are requantized to `Q4_0_128` in general. For embedding weights, `Q6_K` tensors are requantized to `Q8_0_C` except for the token embedding matrix which is dequantized to fp16
|
||||
**All devices:**
|
||||
- `FP16` and `BF16` tensors are used as they are.
|
||||
- A quantized token embedding (`token_embd.weight`) and output weight (`output.weight`) are converted to `Q8_0_C`.
|
||||
|
||||
**CPU and GPU:**
|
||||
- `Q4_0`, `Q4_1`, `Q4_K`, `Q5_1` and `Q8_0` tensors keep their layout: weights, scales and zero points are extracted directly.
|
||||
- `Q5_K` and `Q6_K` tensors are converted to `Q8_0_C`. `GGML_OPENVINO_REQUANT_KQUANT` selects a 4-bit layout instead.
|
||||
- `Q5_0` tensors are converted to `Q4_0_64`.
|
||||
|
||||
**GPU:**
|
||||
- `Q4_K`, `Q5_K` and `Q6_K` MoE expert tensors are converted to `Q4_1_64`. This lets the GPU plugin use its fused MoE op, which is much faster for prompt processing. `GGML_OPENVINO_REQUANT_KQUANT` does not change this. `GGML_OPENVINO_MOE_OP=0` turns the fused MoE op off.
|
||||
|
||||
**NPU:**
|
||||
- All other quantized tensors are requantized to `Q4_0_128`.
|
||||
- For a `Q6_K` token embedding, `GGML_OPENVINO_NPU_TOKEN_EMBD_QUANT` selects another layout.
|
||||
- The validated models use `Q4_K_M` files.
|
||||
|
||||
**Layout names:**
|
||||
- `Q8_0_C`: 8-bit, one scale per row.
|
||||
- `Q4_0_128` and `Q4_0_64`: 4-bit, symmetric, one scale per 128 or 64 weights.
|
||||
|
||||
**Additional Notes:**
|
||||
- Both `Q4_0` and `Q4_1` models use `Q6_K` for the token embedding tensor and the final matmul weight tensor (often the same tensor)
|
||||
- `Q4_0` models may produce some `Q4_1` tensors if an imatrix is provided during quantization using `llama-quantize`
|
||||
- `Q4_K_M` models may include both `Q6_K` and `Q5_K` tensors (observed in Phi-3)
|
||||
- `Q5_1` tensors are dequantized natively (weights, scales, and zero-points extracted directly)
|
||||
|
||||
## Supported Llama.cpp Tools
|
||||
</details>
|
||||
|
||||
The OpenVINO backend integrates with the standard llama.cpp tools listed below.
|
||||
However, all the tools coverage across all devices is not uniform and exhaustive validation is work in progress.
|
||||
## Supported llama.cpp Tools
|
||||
|
||||
The OpenVINO backend works with the standard llama.cpp tools listed below. Tool coverage is not uniform across devices.
|
||||
|
||||
- llama-bench
|
||||
- llama-cli
|
||||
- llama-completion
|
||||
- llama-embedding
|
||||
- llama-perplexity
|
||||
- llama-run
|
||||
- llama-server
|
||||
- llama-simple
|
||||
|
||||
## Validated Models
|
||||
|
||||
Although, the validated models below were tested with `llama-cli` using the `Q4_K_M` quantization format on Intel® Core™ Ultra Series 2 (Lunar Lake), the OpenVINO backend is expected to work across a broader range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), [supported model precisions](#supported-model-precisions), [supported llama.cpp tools](#supported-llamacpp-tools) and additional model architectures.
|
||||
|
||||
> [!NOTE]
|
||||
> Extensive accuracy validation, performance optimizations, and broader architecture coverage are work in progress.
|
||||
The models below were tested with `llama-cli` and `Q4_K_M` files on an Intel® Core™ Ultra Series 2 (Lunar Lake). The OpenVINO backend is expected to work on a broader range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), [model precisions](#supported-model-precisions), [llama.cpp tools](#supported-llamacpp-tools) and model architectures.
|
||||
|
||||
**Legend & Test Configuration:**
|
||||
- **Status:** ✓ = Passed | ~ = Accuracy issues | ✗ = Failed or Unsupported
|
||||
- **Execution Modes:**
|
||||
- **SL** = Stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`)
|
||||
- **SF** = Stateful (`GGML_OPENVINO_STATEFUL_EXECUTION=1`)
|
||||
- Note: The NPU operates in stateless mode only.
|
||||
- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel Graphics Compiler 2.41.5 | Intel OpenCL GPU Driver 26.35.39758.10-0 | Intel NPU Driver 1.38.0.
|
||||
- **Execution mode:** stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`) on every device.
|
||||
- **Validation system:** Intel® Core™ Ultra 7 258V (Lunar Lake) | 32 GB RAM | Ubuntu 26.04.1 | Intel Graphics Compiler 2.42.6 | Intel OpenCL GPU Driver 26.40.40196.8-0 | Intel NPU Driver 1.38.0.
|
||||
- See [Known Limitations](#known-limitations) for context on observed failures.
|
||||
|
||||
| Model | CPU (SL / SF) | GPU (SL / SF) | NPU (SL) |
|
||||
<details open>
|
||||
<summary><b>Validated models</b></summary>
|
||||
|
||||
| Model | CPU | GPU | NPU |
|
||||
| :--- | :---: | :---: | :---: |
|
||||
| [bartowski/Llama-3.2-1B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Llama-3.2-3B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Meta-Llama-3.1-8B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Llama-3.2-1B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Llama-3.2-3B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Meta-Llama-3.1-8B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3.5-0.8B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-0.8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [bartowski/Qwen_Qwen3.5-2B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [bartowski/Qwen_Qwen3.5-4B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-4B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3.5-0.8B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-0.8B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3.5-2B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3.5-4B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-4B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ~ |
|
||||
| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✓ | ✗ / ✗ | ✓ |
|
||||
| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ | ✓ | ✗ |
|
||||
| | | | |
|
||||
| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/microsoft_Phi-4-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/microsoft_Phi-4-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [bartowski/Mistral-7B-Instruct-v0.3-Q4_K_M](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [QuantFactory/Ministral-3b-instruct.Q4_K_M](https://huggingface.co/QuantFactory/Ministral-3b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Ministral-8B-Instruct-2410-Q4_K_M](https://huggingface.co/bartowski/Ministral-8B-Instruct-2410-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Mistral-7B-Instruct-v0.3-Q4_K_M](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [QuantFactory/Ministral-3b-instruct.Q4_K_M](https://huggingface.co/QuantFactory/Ministral-3b-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Ministral-8B-Instruct-2410-Q4_K_M](https://huggingface.co/bartowski/Ministral-8B-Instruct-2410-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ~ / ~ | ✓ |
|
||||
| [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ / ✓ | ~ / ~ | ~ |
|
||||
| [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ | ✓ | ~ |
|
||||
| [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ | ✓ | ✓ |
|
||||
| [openbmb/MiniCPM-V-4_5-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-4_5-gguf) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [LiquidAI/LFM2.5-2.6B-Q4_K_M](https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF) | ✓ | ✓ | ✗ |
|
||||
| [bartowski/inclusionAI_Ling-mini-2.0-Q4_K_M](https://huggingface.co/bartowski/inclusionAI_Ling-mini-2.0-GGUF) | ✓ | ✓ | ✗ |
|
||||
| | | | |
|
||||
| [gpustack/bge-m3-Q4_K_M.gguf](https://huggingface.co/gpustack/bge-m3-GGUF) | ✓ | ✗ | ✗ |
|
||||
| [Qwen/Qwen3-Embedding-0.6B-Q8_0](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B-GGUF) | ✓ | ✓ | ✗ |
|
||||
| [ggml-org/bge-m3-Q8_0](https://huggingface.co/ggml-org/bge-m3-Q8_0-GGUF) | ✓ | ✓ | ✗ |
|
||||
| [nomic-ai/nomic-embed-text-v1.5-Q8_0](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF) | ✓ | ✓ | ✗ |
|
||||
|
||||
The embedding models were tested with `llama-embedding` and `Q8_0` files.
|
||||
|
||||
</details>
|
||||
|
||||
## Build Instructions
|
||||
|
||||
@@ -154,39 +211,51 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
|
||||
- Linux or Windows system with Intel hardware (CPU, GPU, or NPU)
|
||||
- **For Intel GPU or NPU Usage**: Install the appropriate hardware drivers for your Intel GPU or NPU. For detailed instructions, see: [Additional Configurations for Hardware Acceleration](https://docs.openvino.ai/2026/get-started/install-openvino/configurations.html).
|
||||
|
||||
- **Linux:**
|
||||
- Git, CMake, and Ninja software tools are needed for building.
|
||||
```bash
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential libcurl4-openssl-dev libtbb12 cmake ninja-build python3-pip curl wget tar
|
||||
```
|
||||
- OpenCL
|
||||
```bash
|
||||
sudo apt install ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd
|
||||
```
|
||||
<details open>
|
||||
<summary><b>Linux</b></summary>
|
||||
|
||||
- **Windows:**
|
||||
- Download and install [Microsoft Visual Studio 2022 Build Tools](https://aka.ms/vs/17/release/vs_BuildTools.exe). During installation, select the **"Desktop development with C++"** workload.
|
||||
Build tools (Git, CMake, Ninja):
|
||||
|
||||
- Install required tools:
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
winget install Git.Git
|
||||
winget install GNU.Wget
|
||||
winget install Ninja-build.Ninja
|
||||
```
|
||||
```bash
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential libcurl4-openssl-dev libtbb12 cmake ninja-build python3-pip curl wget tar
|
||||
```
|
||||
|
||||
- Install **OpenCL** using **vcpkg**:
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
cd C:\
|
||||
git clone https://github.com/microsoft/vcpkg
|
||||
cd vcpkg
|
||||
.\bootstrap-vcpkg.bat
|
||||
.\vcpkg install opencl
|
||||
# Optional but recommended: Integrate vcpkg with Visual Studio / CMake:
|
||||
.\vcpkg integrate install
|
||||
```
|
||||
OpenCL:
|
||||
|
||||
```bash
|
||||
sudo apt install ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Windows</b></summary>
|
||||
|
||||
1. Download and install [Microsoft Visual Studio 2022 Build Tools](https://aka.ms/vs/17/release/vs_BuildTools.exe). During installation, select the **"Desktop development with C++"** workload.
|
||||
2. Install the required tools:
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
winget install Git.Git
|
||||
winget install GNU.Wget
|
||||
winget install Ninja-build.Ninja
|
||||
```
|
||||
|
||||
3. Install **OpenCL** using **vcpkg**:
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
cd C:\
|
||||
git clone https://github.com/microsoft/vcpkg
|
||||
cd vcpkg
|
||||
.\bootstrap-vcpkg.bat
|
||||
.\vcpkg install opencl
|
||||
# Optional but recommended: Integrate vcpkg with Visual Studio / CMake:
|
||||
.\vcpkg integrate install
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
### 1. Install OpenVINO Runtime
|
||||
|
||||
@@ -199,21 +268,25 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
|
||||
|
||||
### 2. Build llama.cpp with OpenVINO Backend
|
||||
|
||||
Clone llama.cpp repo and build :
|
||||
Clone the llama.cpp repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/ggml-org/llama.cpp
|
||||
cd llama.cpp
|
||||
```
|
||||
|
||||
- **Linux:**
|
||||
Build on Linux:
|
||||
|
||||
```bash
|
||||
source /opt/intel/openvino/setupvars.sh
|
||||
cmake -B build/ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON
|
||||
cmake --build build/ReleaseOV --parallel
|
||||
```
|
||||
|
||||
- **Windows:** Open **x64 Native Tools Command Prompt for VS** (so the MSVC toolchain is on `PATH`), then run:
|
||||
<details>
|
||||
<summary><b>Build on Windows</b></summary>
|
||||
|
||||
Open **x64 Native Tools Command Prompt for VS** (so the MSVC toolchain is on `PATH`), then run:
|
||||
|
||||
```cmd
|
||||
C:\Intel\openvino\setupvars.bat
|
||||
@@ -224,9 +297,11 @@ cmake --build build\ReleaseOV --parallel
|
||||
> [!NOTE]
|
||||
> The Windows install path is `C:\Intel\openvino` (no spaces) to avoid quoting problems some CMake/Ninja toolchains have with `C:\Program Files (x86)\...`. Adjust to wherever you installed OpenVINO Runtime. From `cmd`, run `C:\Intel\openvino\setupvars.bat`; from PowerShell, run `& "C:\Intel\openvino\setupvars.ps1"` instead. Once the build is finished you can launch the binaries from any `cmd` or `PowerShell` window after sourcing the matching `setupvars` script for that shell.
|
||||
|
||||
</details>
|
||||
|
||||
#### Ubuntu Build Script
|
||||
|
||||
For Ubuntu24 users, the following shell script automates the prerequisite installs (build tools, OpenCL ICD), the OpenVINO Runtime download/extract/setup, and the Ninja-based llama.cpp build.
|
||||
For Ubuntu users, the following shell script automates the prerequisite installs (build tools, OpenCL ICD), the OpenVINO Runtime download/extract/setup, and the Ninja-based llama.cpp build.
|
||||
Save the following as `build-llamacpp-ov.sh` next to where you want the `llama.cpp` folder to land, then run it:
|
||||
|
||||
```bash
|
||||
@@ -556,81 +631,102 @@ endlocal
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
### 3. Download Sample Model
|
||||
|
||||
Download sample model for testing.
|
||||
Download a sample model for testing.
|
||||
|
||||
```bash
|
||||
# Linux
|
||||
mkdir -p ~/models/
|
||||
wget https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf \
|
||||
-O ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary><b>Windows</b></summary>
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
mkdir C:\models
|
||||
Invoke-WebRequest -Uri https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf -OutFile C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
```
|
||||
|
||||
# Windows Command Line
|
||||
```cmd
|
||||
REM Windows Command Prompt
|
||||
mkdir C:\models
|
||||
curl -L https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf -o C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
### 4. Run Inference with OpenVINO Backend
|
||||
|
||||
When using the OpenVINO backend, the first inference token may have slightly higher latency due to on-the-fly conversion to the OpenVINO graph. Subsequent tokens and runs will be faster.
|
||||
The first run of a model converts and compiles it for the device. This takes seconds on CPU and GPU, and can take minutes on the NPU for large models. Later tokens are faster, and the [compiled model cache](#compiled-model-cache) makes later runs start faster.
|
||||
|
||||
> [!NOTE]
|
||||
> Default context size is set to the model training context, which may be very large. For example, 131072 for Llama 3.2 1B, which may result in lower performance, especially on edge/laptop devices. Use `-c` to limit context size in supported llama.cpp tools for better performance. For example, `-c 512`.
|
||||
> Default context size is set to the model training context, which may be very large. For example, 131072 for Llama 3.2 1B, which may result in lower performance, especially on edge/laptop devices. Use `-c` to limit context size in supported llama.cpp tools for better performance:
|
||||
> - CPU and GPU: for example `-c 1024`.
|
||||
> - NPU: keep the context small, for example `-c 512`, or `-c 256` for larger models. Decode may slow down as the context grows.
|
||||
|
||||
```bash
|
||||
# If device is unset or unavailable, defaults to CPU.
|
||||
# If the system has multiple GPUs, use GPU.0 or GPU.1 to explicitly target a specific GPU.
|
||||
|
||||
# Linux
|
||||
# If the device is unset, the CPU is used.
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
# Optional: enable stateful execution for improved GPU performance (recommended).
|
||||
export GGML_OPENVINO_STATEFUL_EXECUTION=1
|
||||
# To run llama-simple:
|
||||
./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -n 50 "The story of AI is "
|
||||
# To run in chat mode:
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024
|
||||
# To run llama-bench, -fa 1 is needed
|
||||
GGML_OPENVINO_STATEFUL_EXECUTION=1 GGML_OPENVINO_DEVICE=GPU ./build/ReleaseOV/bin/llama-bench -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -fa 1
|
||||
# To run llama-bench:
|
||||
GGML_OPENVINO_DEVICE=GPU ./build/ReleaseOV/bin/llama-bench -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
|
||||
# NPU: keep context small to avoid failures from very large model context windows.
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 512
|
||||
```
|
||||
|
||||
# Windows Command Line
|
||||
<details>
|
||||
<summary><b>Windows</b></summary>
|
||||
|
||||
```cmd
|
||||
REM Windows Command Prompt
|
||||
set GGML_OPENVINO_DEVICE=GPU
|
||||
# Optional: enable stateful execution for improved GPU performance (recommended).
|
||||
set GGML_OPENVINO_STATEFUL_EXECUTION=1
|
||||
REM To run llama-simple:
|
||||
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
|
||||
REM To run in chat mode:
|
||||
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 1024
|
||||
REM To run llama-bench:
|
||||
build\ReleaseOV\bin\llama-bench.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf"
|
||||
|
||||
REM NPU: keep the context small.
|
||||
set GGML_OPENVINO_DEVICE=NPU
|
||||
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 512
|
||||
```
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
$env:GGML_OPENVINO_DEVICE = "GPU"
|
||||
$env:GGML_OPENVINO_STATEFUL_EXECUTION = "1"
|
||||
|
||||
# To run llama-simple
|
||||
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
|
||||
# To run in chat mode:
|
||||
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 1024
|
||||
# To run llama-bench, -fa 1 is needed
|
||||
build\ReleaseOV\bin\llama-bench.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -fa 1
|
||||
|
||||
# NPU: keep context small to avoid failures from very large model context windows.
|
||||
# Windows Command Line
|
||||
set GGML_OPENVINO_DEVICE=NPU
|
||||
# Windows PowerShell
|
||||
# NPU: keep the context small.
|
||||
$env:GGML_OPENVINO_DEVICE = "NPU"
|
||||
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 512
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
> [!NOTE]
|
||||
> On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU. A device that is not available is an error (no fallback to CPU), and the error message lists the available OpenVINO devices with their names. Run `llama-cli --list-devices` to see the valid values: each OpenVINO device shows the `GGML_OPENVINO_DEVICE=<value>` to set, and `(selected)` marks the active one. Select the OpenVINO device with this variable, not with `-dev`. See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html) for more details.
|
||||
> - Select the OpenVINO device with `GGML_OPENVINO_DEVICE`, not with `-dev`.
|
||||
> - On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU.
|
||||
> - To split a model across several GPUs, set a comma-separated list, e.g. `GPU.0,GPU.1`, and use `-sm layer`.
|
||||
> - A device that is not available is an error (no fallback to CPU). The error message lists the available OpenVINO devices with their names.
|
||||
> - Run `llama-cli --list-devices` to see the valid values: each OpenVINO device shows the `GGML_OPENVINO_DEVICE=<value>` to set, and `(selected)` marks the active one.
|
||||
> - See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html) for more details.
|
||||
|
||||
### 5. Docker Build
|
||||
|
||||
You can build and run llama.cpp with OpenVINO backend using Docker.
|
||||
|
||||
<details>
|
||||
<summary><b>Build the images</b></summary>
|
||||
|
||||
```bash
|
||||
# Build the base runtime image with compiled shared libraries and minimal dependencies.
|
||||
docker build -t llama-openvino:base -f .devops/openvino.Dockerfile .
|
||||
@@ -648,9 +744,12 @@ docker build --target=server -t llama-openvino:server -f .devops/openvino.Docker
|
||||
docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --target=server -t llama-openvino:server -f .devops/openvino.Dockerfile .
|
||||
```
|
||||
|
||||
Run llama.cpp with OpenVINO backend Docker container.
|
||||
Save sample models in `~/models` as [shown above](#3-download-sample-model). It will be mounted to the container in the examples below.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Run a container</b> (CPU, GPU, NPU)</summary>
|
||||
|
||||
Run llama.cpp with OpenVINO backend Docker container. Save sample models in `~/models` as [shown above](#3-download-sample-model). It will be mounted to the container in the examples below.
|
||||
|
||||
```bash
|
||||
# Run Docker container
|
||||
@@ -659,19 +758,20 @@ docker run --rm -it -v ~/models:/models llama-openvino:light --no-warmup -c 1024
|
||||
# With Intel GPU access (iGPU or dGPU)
|
||||
docker run --rm -it -v ~/models:/models \
|
||||
--device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
|
||||
--env=GGML_OPENVINO_DEVICE=GPU --env=GGML_OPENVINO_STATEFUL_EXECUTION=1 \
|
||||
--env=GGML_OPENVINO_DEVICE=GPU \
|
||||
llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
|
||||
# With Intel NPU access
|
||||
docker run --rm -it -v ~/models:/models \
|
||||
--device=/dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
|
||||
--device=/dev/accel --group-add=$(stat -c "%g" /dev/accel/accel* | head -n 1) -u $(id -u):$(id -g) \
|
||||
--env=GGML_OPENVINO_DEVICE=NPU \
|
||||
llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
llama-openvino:light --no-warmup -c 512 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
```
|
||||
|
||||
Run Llama.cpp Server with OpenVINO Backend.
|
||||
> [!NOTE]
|
||||
> `llama-server` with OpenVINO backend supports only one chat session/thread, when `GGML_OPENVINO_STATEFUL_EXECUTION=1` is enabled.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Run llama-server</b> (container or local build) and test it</summary>
|
||||
|
||||
```bash
|
||||
# Run the llama-openvino:server Docker container (CPU)
|
||||
@@ -685,14 +785,14 @@ llama-openvino:server --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_
|
||||
|
||||
# Run the llama-openvino:server Docker container with Intel NPU access
|
||||
docker run --rm -it -v ~/models:/models \
|
||||
--device=/dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
|
||||
--device=/dev/accel --group-add=$(stat -c "%g" /dev/accel/accel* | head -n 1) -u $(id -u):$(id -g) \
|
||||
-p 8080:8080 --env=GGML_OPENVINO_DEVICE=NPU \
|
||||
llama-openvino:server --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --host 0.0.0.0
|
||||
llama-openvino:server --no-warmup -c 512 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --host 0.0.0.0
|
||||
|
||||
# Or Using llama-server executable
|
||||
./build/ReleaseOV/bin/llama-server -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --port 8080 -c 1024
|
||||
|
||||
# Option 1: Open your browser to http://localhost:8080 to access the web UI for the llama.cpp server.
|
||||
# Option 1: Open your browser to http://localhost:8080 to open Llama Chat, the built-in chat app of the llama.cpp server.
|
||||
# Option 2: In a NEW terminal, test the server with curl
|
||||
|
||||
# If you are behind a proxy, make sure to set NO_PROXY to avoid proxy for localhost
|
||||
@@ -706,101 +806,221 @@ curl -X POST "http://localhost:8080/v1/chat/completions" -H "Content-Type: appli
|
||||
-d '{"messages":[{"role":"user","content":"Write a poem about OpenVINO"}],"max_tokens":100}' | jq .
|
||||
```
|
||||
|
||||
## GGML OpenVINO Backend Runtime Configurations
|
||||
</details>
|
||||
|
||||
The OpenVINO backend can be configured using the following environment variables at runtime to control device selection, caching, debugging, and profiling behavior.
|
||||
Boolean flags follow a uniform convention: set to a **positive integer** (e.g. `1`) to enable; unset, empty, `0`, negative, or non-numeric values are treated as disabled.
|
||||
## Runtime Configuration
|
||||
|
||||
| Variable | Type | Default | Description |
|
||||
|-----------------------------------|-----------|------------|-------------------------------------------------------------------------------------------------------------|
|
||||
| `GGML_OPENVINO_DEVICE` | String | `CPU` | Specify the target device (CPU, GPU, NPU). On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU. A device that is not available is an error (no fallback to CPU), and the error message lists the available OpenVINO devices with their names. See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html). When set to **NPU**, static compilation mode is enabled for optimal performance. |
|
||||
| `GGML_OPENVINO_CACHE_DIR` | String | `not set` | Directory for OpenVINO's separate plugin cache. On NPU, this sets `NPUW_CACHE_DIR`. |
|
||||
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` | String | `not set` | Directory for standalone compiled blobs with weights. Dynamic CPU/GPU graphs can import matching blobs on later runs. |
|
||||
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY` | Boolean | `0` | Require an existing compiled blob and skip weight uploads and compilation. Requires Linux or Windows mmap loading and a full dynamic CPU/GPU graph on OpenVINO. |
|
||||
| `GGML_OPENVINO_PREFILL_CHUNK_SIZE`| Integer | `256` | Token chunk size for **NPU** prefill (NPU-only; ignored on CPU/GPU). Must be a positive integer; otherwise the default is used. |
|
||||
| `GGML_OPENVINO_NPU_COMPILE_CONFIG` | String | `not set` | NPU-only compiler mode parameters forwarded to OpenVINO as `NPU_COMPILATION_MODE_PARAMS`, for example `optimization-level=3`. |
|
||||
| `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Keep KV and supported recurrent caches inside the model. Single-slot CPU/GPU execution only. |
|
||||
| `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. |
|
||||
| `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. |
|
||||
| `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | Boolean | `0` | Disable the stateful KV-state sequence-axis relayout (relayout is on by default). It moves the KV state sequence axis from dim 1 to dim 2, so the GPU plugin can append new tokens in place instead of copying the whole state every token, and the reader side no longer transposes the whole accumulated state. Set to `1` to disable. |
|
||||
| `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. |
|
||||
| `GGML_OPENVINO_MEMORY_OPTIMIZE` | Boolean | `0` | Umbrella switch for compile-time memory reductions. Enables `GGML_OPENVINO_REDUCE_COMPILE_MEM` and, on GPU, `GGML_OPENVINO_RELEASE_WEIGHTS` unless those fine-grained variables are explicitly set. |
|
||||
| `GGML_OPENVINO_REDUCE_COMPILE_MEM`| Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` | Reduce compile-time host memory use by streaming weight requantization and avoiding extra weight-node materialization where possible. Set explicitly to override the umbrella switch. |
|
||||
| `GGML_OPENVINO_RELEASE_WEIGHTS` | Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` on GPU | GPU-only. Release host weight buffers after the compiled model cache can reuse the device/plugin copy. Requires stable graph shapes; dynamic workloads that need recompilation should leave this disabled. |
|
||||
| `GGML_OPENVINO_SPILL_DIR` | String | `not set` | Directory for a disk-backed weight buffer. When set, the repacked weight buffer is mapped from an unlinked file on this path instead of anonymous memory, so its pages are reclaimable under memory pressure instead of staying pinned, cutting the load-time host memory peak. Must point at real storage; a tmpfs mount (e.g. `/tmp` on many systems) backs it with RAM and makes the peak worse. |
|
||||
| `GGML_OPENVINO_REQUANT_KQUANT` | String | `not set` | Requantize Q6_K/Q5_K weights (and matching MoE expert weights) to a 4-bit target instead of the default Q8_0_C, trading accuracy for less memory traffic. One of `q4_asym64` (Q6_K/Q5_K only, keeps a real zero point at group 64), `q4_asym64_all` (also requantizes Q4_K), `q4_sym128` (Q6_K/Q5_K only), `q4_sym128_all` (Q4_K too, drops its per-group zero point), or `native` (no requantization). |
|
||||
| `GGML_OPENVINO_PROFILING` | Integer | `0` | `1` logs execution timing; `2` or higher also enables OpenVINO and OpenCL profiling. |
|
||||
| `GGML_OPENVINO_DEBUG_NODE` | String | `not set` | Add the named graph nodes as compiled outputs for debugging. Separate multiple names with commas. |
|
||||
| `GGML_OPENVINO_MOE_OP` | Boolean | `1` | On GPU, set to `0` to keep the unfused GatherMatmul path. |
|
||||
| `GGML_OPENVINO_DUMP_CGRAPH` | Boolean | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. |
|
||||
| `GGML_OPENVINO_DUMP_IR` | Boolean | `0` | Serialize OpenVINO IR files with timestamps. |
|
||||
| `GGML_OPENVINO_DEBUG_INPUT` | Boolean | `0` | Enable input debugging and print input tensor info. |
|
||||
| `GGML_OPENVINO_DEBUG_OUTPUT` | Boolean | `0` | Enable output debugging and print output tensor info. |
|
||||
| `GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS` | Boolean | `0` | Print tensor address map once. |
|
||||
| `GGML_OPENVINO_LOG_UNSUPPORTED_OPS`| Boolean | `0` | Log warning messages with tensor details and rejection reasons for any ops not supported by the OpenVINO backend. Emits at `WARN` level (requires `--log-verbosity >= 2`, enabled by default). |
|
||||
The backend needs no environment variable: it runs on the CPU with its defaults. Set `GGML_OPENVINO_DEVICE` to use the GPU or the NPU. All other variables are optional.
|
||||
|
||||
> [!NOTE]
|
||||
> - `GGML_OPENVINO_STATEFUL_EXECUTION` is an **Experimental** feature for managing caches internally inside the OpenVINO model on CPUs and GPUs. Use a single slot (`-np 1`). KV caches retain the append-based state layout and sequence-axis optimization. Qwen3.5 adds recurrent cache states in their GGML layouts. Qwen3.5 requires an unsplit graph with model caching enabled and no recurrent rollback. A prompt starting at position 0 resets all states. State save/restore, sequence rewind, context shift, and mid-sequence graph replacement are unsupported. Stateful execution is not effective on NPUs.
|
||||
> - `GGML_OPENVINO_LOG_UNSUPPORTED_OPS` emits logs at `WARN` level (`GGML_LOG_WARN`), which requires application log verbosity `--log-verbosity >= 2` (or `-lv 2`).
|
||||
> - With `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1`, use the same compilation settings as the export run. One directory can hold blobs for different models and settings; `GGML_OPENVINO_SPILL_DIR` does not affect the cache key and is ignored in cache-only mode. See [Compiled model cache](../../ggml/src/ggml-openvino/README.md) for the workflow and restrictions.
|
||||
```bash
|
||||
export GGML_OPENVINO_DEVICE=GPU # String
|
||||
export GGML_OPENVINO_PROFILING=1 # Integer
|
||||
export GGML_OPENVINO_EXACT_ZERO_POINT=1 # Boolean: 1 (on) or 0 (off)
|
||||
|
||||
# Show the variables in effect (log verbosity 4)
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024 -lv 4 2>&1 | grep "settings from the environment"
|
||||
```
|
||||
|
||||
- The backend reads the variables once, at start. Unset or empty means the default.
|
||||
- To lower the memory a model needs, see [Tips for Memory Optimization](../../ggml/src/ggml-openvino/dev_doc/OpenVINO-ENV-VARS.md#tips-for-memory-optimization).
|
||||
- A value that is not valid stops the backend. A `GGML_OPENVINO_*` name that the backend does not know gives a warning:
|
||||
|
||||
```text
|
||||
ggml-openvino: GGML_OPENVINO_EXACT_ZERO_POINT=yes is not valid, use 1 (on) or 0 (off)
|
||||
ggml-openvino: unknown environment variable GGML_OPENVINO_DEVCE is ignored
|
||||
```
|
||||
|
||||
### Basic
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_DEVICE` | all | String | `CPU` | Target device: `CPU`, `GPU` or `NPU`. For several GPUs, see the [device notes](#4-run-inference-with-openvino-backend). Examples: `GGML_OPENVINO_DEVICE=NPU`, `GGML_OPENVINO_DEVICE=GPU.0,GPU.1`. |
|
||||
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` | all | String | `not set` | Directory for the backend's cache of compiled models. A later start with the same model and settings loads faster. See [Compiled model cache](#compiled-model-cache). Example: `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache`. |
|
||||
| `GGML_OPENVINO_NPU_CONFIG` | NPU | String | `not set` | OpenVINO properties for the NPU plugin, as comma-separated `KEY=VALUE` pairs. See [Plugin properties](#plugin-properties). Example: `GGML_OPENVINO_NPU_CONFIG="NPU_COMPILER_TYPE=PLUGIN"`. |
|
||||
| `GGML_OPENVINO_GPU_CONFIG` | GPU | String | `not set` | OpenVINO properties for the GPU plugin, as comma-separated `KEY=VALUE` pairs. See [Plugin properties](#plugin-properties). Example: `GGML_OPENVINO_GPU_CONFIG="INFERENCE_PRECISION_HINT=f16"`. |
|
||||
| `GGML_OPENVINO_CPU_CONFIG` | CPU | String | `not set` | OpenVINO properties for the CPU plugin, as comma-separated `KEY=VALUE` pairs. See [Plugin properties](#plugin-properties). Example: `GGML_OPENVINO_CPU_CONFIG="INFERENCE_PRECISION_HINT=f16"`. |
|
||||
| `GGML_OPENVINO_PROFILING` | all | Integer | `0` | `1` logs execution timing. `2` or higher also turns on OpenVINO and OpenCL profiling. Example: `GGML_OPENVINO_PROFILING=1`. |
|
||||
|
||||
### Plugin properties
|
||||
|
||||
`GGML_OPENVINO_NPU_CONFIG`, `GGML_OPENVINO_GPU_CONFIG` and `GGML_OPENVINO_CPU_CONFIG` pass OpenVINO plugin properties to the device in their name.
|
||||
|
||||
```bash
|
||||
# Format: comma-separated KEY=VALUE pairs. The backend uses the variable of the selected device only.
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_NPU_CONFIG="NPU_COMPILER_TYPE=PLUGIN,COMPILATION_NUM_THREADS=4"
|
||||
|
||||
# CPU: compute in FP16, as the GPU and the NPU do. This helps to debug the accuracy of a quantized model.
|
||||
export GGML_OPENVINO_DEVICE=CPU
|
||||
export GGML_OPENVINO_CPU_CONFIG="INFERENCE_PRECISION_HINT=f16"
|
||||
```
|
||||
|
||||
- Property names and values are those of OpenVINO, for example `YES` and `NO`. They are not the `1` and `0` of the `GGML_OPENVINO_*` variables.
|
||||
- The backend applies the variable last, so it overrides the properties that the backend sets.
|
||||
- OpenVINO checks the properties when the model loads. If it rejects one, the load fails and the error names the variable.
|
||||
- All properties and their values are in the OpenVINO documentation: [NPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.html), [GPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html), [CPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device.html).
|
||||
- NPU: the backend sets several properties by default. They are listed in the [developer documentation](../../ggml/src/ggml-openvino/dev_doc/OpenVINO-ENV-VARS.md#npu-plugin-properties).
|
||||
- GPU and CPU: the backend sets no property, except `CACHE_DIR` and `CACHE_MODE=OPTIMIZE_SIZE` when `GGML_OPENVINO_CACHE_DIR` is set.
|
||||
|
||||
`NPU_COMPILER_TYPE` selects the NPU compiler. Both forms below work; use the first one. If both are set, `GGML_OPENVINO_NPU_CONFIG` takes precedence.
|
||||
|
||||
```bash
|
||||
# Values: DRIVER (the compiler in the NPU driver; backend default), PLUGIN (the compiler in OpenVINO),
|
||||
# PREFER_PLUGIN (OpenVINO default: PLUGIN when it is available, DRIVER if not).
|
||||
export GGML_OPENVINO_NPU_CONFIG="NPU_COMPILER_TYPE=PLUGIN" # recommended form
|
||||
export GGML_OPENVINO_NPU_COMPILER_TYPE=PLUGIN # older name, still works
|
||||
```
|
||||
|
||||
### Advanced tuning
|
||||
|
||||
The defaults fit most models. The variables below trade one property for another, for example speed for accuracy.
|
||||
|
||||
<details>
|
||||
<summary><b>Advanced tuning variables</b></summary>
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_PREFILL_CHUNK_SIZE` | NPU | Integer | `256` | Number of tokens that the NPU processes in one prefill step. A larger value can make long prompts faster, and makes the first load longer. Zero or a negative value means the default. Example: `GGML_OPENVINO_PREFILL_CHUNK_SIZE=512`. |
|
||||
| `GGML_OPENVINO_NPU_TOKEN_EMBD_QUANT` | NPU | String | `q8_0_c` | How a `Q6_K` `token_embd.weight` is stored for the NPU: `q8_0_c` (8-bit, one scale per row), `f16`, `q4_0_128` (4-bit, one scale per 128 weights) or `q4_0_c` (4-bit, one scale per row). The 4-bit layouts use less memory at some cost in accuracy; `f16` uses more memory. Keep the default unless memory is tight. |
|
||||
| `GGML_OPENVINO_GDN_DECOMPOSE` | all | Boolean | auto | Translate `GATED_DELTA_NET` (Qwen3.5) into MatMul operations instead of the OpenVINO `GatedDeltaNet` op. Unset: on for the Lunar Lake NPU, off elsewhere. `0` turns it off, `1` turns it on for any device. Try `1` for Qwen3.5 on the Panther Lake NPU. |
|
||||
| `GGML_OPENVINO_MANUAL_GQA_ATTN` | CPU, GPU | Boolean | on for GPU | Build grouped-query attention from separate MatMul operations. Unset: on for GPU, off for CPU and NPU. `1` turns it on, `0` turns it off. Try `1` on the CPU for faster decode. |
|
||||
| `GGML_OPENVINO_REQUANT_KQUANT` | CPU, GPU | String | `not set` | Requantize `Q6_K` and `Q5_K` weights to a 4-bit layout instead of `Q8_0_C`: faster decode for lower accuracy. One of `q4_asym64`, `q4_asym64_all` (also `Q4_K`), `q4_sym128`, `q4_sym128_all` (also `Q4_K`) or `native` (no requantization). On GPU with `Q4_K_M` files use `q4_asym64_all`, because `q4_asym64` does not compile there. |
|
||||
| `GGML_OPENVINO_EXACT_ZERO_POINT` | CPU, GPU | Boolean | `0` | Keep the exact zero point of asymmetric weights (`Q4_1`, `Q4_K`, `Q5_1`, `Q5_K`) instead of rounding it to an integer. This improves accuracy and lowers speed. Use it when accuracy matters more than speed. |
|
||||
|
||||
</details>
|
||||
|
||||
### Compiled model cache
|
||||
|
||||
The cache keeps compiled models on disk, so later starts of the same model load faster. Use the same model and the same settings in every run.
|
||||
|
||||
```bash
|
||||
# CPU or GPU: the first run compiles the model and writes the cache. Later runs read it.
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024
|
||||
|
||||
# CPU or GPU, later runs (optional): require the cached model and skip the weight upload and the compilation.
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1
|
||||
|
||||
# NPU: add the weightless bank.
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache
|
||||
export GGML_OPENVINO_NPU_WEIGHTLESS_BANK=1
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 512
|
||||
|
||||
# NPU, later runs (optional): use less memory. The run fails if a cached model is missing.
|
||||
export GGML_OPENVINO_SELF_CONTAINED_MMAP=1
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary><b>Cache variables</b></summary>
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY` | CPU, GPU | Boolean | `0` | Require an existing cached model and skip the weight upload and the compilation. Needs mmap model loading (the default on Linux and Windows) and a model that runs fully on OpenVINO. |
|
||||
| `GGML_OPENVINO_NPU_WEIGHTLESS_BANK` | NPU | Boolean | `0` | Requires `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR`. Write the prefill and decode models without weights, plus one shared weights file. Later runs read the weights from that file and keep one copy in memory. |
|
||||
| `GGML_OPENVINO_SELF_CONTAINED_MMAP` | NPU | Boolean | `0` | Later runs only. Requires `GGML_OPENVINO_NPU_WEIGHTLESS_BANK=1` and an existing cache. Use the memory-mapped GGUF file as the model buffer, which lowers the host memory peak. Fails if a cached model is missing; run once without it to build the cache. |
|
||||
| `GGML_OPENVINO_CACHE_DIR` | all | String | `not set` | Directory for the cache of the OpenVINO plugin itself. `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` is the recommended cache. |
|
||||
|
||||
- One directory can hold the cached models of different models and settings.
|
||||
- NPU: each prefill length enters the cache in a run without `GGML_OPENVINO_SELF_CONTAINED_MMAP`.
|
||||
- See [Compiled model cache](../../ggml/src/ggml-openvino/README.md#compiled-model-cache) for the workflow and the restrictions.
|
||||
|
||||
</details>
|
||||
|
||||
### Deprecated
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_STATEFUL_EXECUTION` | CPU, GPU | Boolean | `0` | **Deprecated**, to be removed. Keep the KV cache inside the OpenVINO model. Use the default, stateless execution (`0`). |
|
||||
|
||||
<details>
|
||||
<summary>Limits of stateful execution</summary>
|
||||
|
||||
- CPU and GPU only; it has no effect on the NPU.
|
||||
- One slot only (`-np 1`).
|
||||
- State save and restore, sequence rewind, context shift and recurrent rollback are not supported.
|
||||
- Qwen3.5 needs a graph that runs fully on OpenVINO, with model caching on.
|
||||
- A prompt that starts at position 0 resets all states.
|
||||
|
||||
</details>
|
||||
|
||||
### Debugging and development
|
||||
|
||||
Debug output, switches that turn a default optimization off, experimental options and older variable names are in the [developer documentation](../../ggml/src/ggml-openvino/dev_doc/OpenVINO-ENV-VARS.md).
|
||||
|
||||
### Example Usage
|
||||
|
||||
#### GPU Inference with Profiling
|
||||
|
||||
```bash
|
||||
# If the system has multiple GPUs, use GPU.0 or GPU.1 to explicitly target a specific GPU.
|
||||
|
||||
# Linux
|
||||
export GGML_OPENVINO_CACHE_DIR=/tmp/ov_cache
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache
|
||||
export GGML_OPENVINO_PROFILING=1
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
export GGML_OPENVINO_STATEFUL_EXECUTION=1
|
||||
|
||||
./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -n 50 "The story of AI is "
|
||||
```
|
||||
|
||||
# Windows Command Line
|
||||
set GGML_OPENVINO_CACHE_DIR=C:\tmp\ov_cache
|
||||
<details>
|
||||
<summary><b>Windows</b></summary>
|
||||
|
||||
```cmd
|
||||
REM Windows Command Prompt
|
||||
set GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=C:\tmp\ov_cache
|
||||
set GGML_OPENVINO_PROFILING=1
|
||||
set GGML_OPENVINO_DEVICE=GPU
|
||||
set GGML_OPENVINO_STATEFUL_EXECUTION=1
|
||||
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
|
||||
```
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
$env:GGML_OPENVINO_CACHE_DIR = "C:\tmp\ov_cache"
|
||||
$env:GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR = "C:\tmp\ov_cache"
|
||||
$env:GGML_OPENVINO_PROFILING = "1"
|
||||
$env:GGML_OPENVINO_DEVICE = "GPU"
|
||||
$env:GGML_OPENVINO_STATEFUL_EXECUTION = "1"
|
||||
|
||||
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
|
||||
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
## Known Limitations
|
||||
|
||||
**General (all devices)**
|
||||
|
||||
- Llama.cpp OpenVINO backend currently supports a subset of GGML ops and text-only models. Unsupported ops or unsupported op shapes/cases fail during OpenVINO translation.
|
||||
- Multimodal features (audio/image/video) are a work in progress.
|
||||
- Limited Embedding and Reranking model support.
|
||||
- Embedding models: reranking models are not validated.
|
||||
- Llama.cpp tool coverage across CPU/GPU/NPU is not uniform.
|
||||
|
||||
**Tool-specific**
|
||||
|
||||
- `llama-bench`: requires `-fa 1` (flash-attention).
|
||||
- `llama-cli --context-shift`: stateless only (`GGML_OPENVINO_STATEFUL_EXECUTION=0`). In stateful mode the KV cache is owned by the OpenVINO model and cannot be shifted externally.
|
||||
- `llama-server`: only one chat session/thread when `GGML_OPENVINO_STATEFUL_EXECUTION=1`.
|
||||
|
||||
**GPU-specific**
|
||||
|
||||
- `llama-server -np > 1`: concurrent requests are batched together, which may slightly reduce per-request throughput.
|
||||
- Multiple GPUs (`GGML_OPENVINO_DEVICE=GPU.0,GPU.1` with `-sm layer`):
|
||||
- `-sm tensor` is not supported.
|
||||
- Copies between GPUs go through host memory. There is no asynchronous copy and no peer-to-peer copy.
|
||||
- Only GPUs can be combined. A list with several NPUs, or with a GPU and the CPU or the NPU, is rejected.
|
||||
- MoE models use the GPU plugin's fused MoE op, which runs in f16 only:
|
||||
- granitemoe models (e.g. `granite-3.1-1b-a400m`) give wrong output.
|
||||
- **Workaround:** `GGML_OPENVINO_MOE_OP=0`. Output is correct, but prompt processing is much slower.
|
||||
- `GGML_OPENVINO_GPU_CONFIG="INFERENCE_PRECISION_HINT=f32"` fails to compile.
|
||||
- `llama-perplexity` with MoE models: set `-b` to the same value as `-c`, so that each batch holds one sequence.
|
||||
|
||||
**NPU-specific**
|
||||
|
||||
- Default context resolves to the model's training context (e.g. 131072 for Llama 3.2 1B), which can OOM or fail or degrade performance on NPU. Inspect the resolved value with `-lv 3`.
|
||||
- **Workaround:** Pass an explicit `-c <N>`, e.g. `-c 1024`.
|
||||
- **Workaround:** Pass an explicit `-c <N>`, e.g. `-c 512`.
|
||||
- NPU device uses a static graph with a fixed prefill chunk size (defaults to 256), configurable with `GGML_OPENVINO_PREFILL_CHUNK_SIZE`. Large prefill/batch settings may need tuning.
|
||||
- `llama-server -np > 1` (multiple parallel sequences) is not supported.
|
||||
- `llama-perplexity`: requires `-b 512` or smaller.
|
||||
- Flash attention must stay on, which is the default (`-fa auto`). With `-fa off`, the run fails on the NPU.
|
||||
- `llama-embedding` fails on the NPU.
|
||||
- Lunar Lake NPU: decode slows down sharply when the context is large. The context size at which this starts depends on the model.
|
||||
- **Workaround:** Pass the smallest context that fits the use, e.g. `-c 512`, or `-c 256` for larger models.
|
||||
- Panther Lake NPU: the load of a Qwen3.5 model can end with a segmentation fault in the NPU compiler of OpenVINO 2026.4.1.
|
||||
- **Workaround:** Set `GGML_OPENVINO_NPU_CONFIG=COMPILATION_NUM_THREADS=1`.
|
||||
- Panther Lake NPU: `GGML_OPENVINO_NPU_TOKEN_EMBD_QUANT=f16` and `q4_0_128` can give wrong output. Keep the default.
|
||||
- Lunar Lake NPU: `granite-4.0-1b` gives wrong output. The activations of the model exceed the FP16 range, and the NPU computes in FP16. The model runs correctly on CPU and GPU.
|
||||
- Lunar Lake NPU: `gemma-4-12B` does not run. The first inference fails with `ZE_RESULT_ERROR_UNINITIALIZED` from the NPU driver. The model runs on CPU and GPU.
|
||||
- The first load of a model on the NPU compiles it for the device, which can take minutes for large models.
|
||||
- **Workaround:** Use the [compiled model cache](#compiled-model-cache).
|
||||
|
||||
> [!NOTE]
|
||||
> The OpenVINO backend is actively under development. Fixes and improvements are underway, and this document will continue to be updated.
|
||||
|
||||
@@ -117,6 +117,13 @@ Block-32 formats use a tile size of 32 rows x 32 columns (`K_cols = 32`):
|
||||
Kernel converts E8M0 bias to IEEE FP16 bias via `vh = clamp(e - 112, 0, 30) << 10` to form a power-of-two FP16 scale.
|
||||
Each 4-bit FP4 nibble is unpacked via `mxfp4_to_fp16_lut` and multiplied by this scale.
|
||||
|
||||
- **NVFP4** (576 bytes per tile, `HTP_TENSOR_TILE_SIZE_NVFP4`, lossless tiling with same size as `block_nvfp4`):
|
||||
- A 64-element block spans 2 consecutive k-tiles; k-tile `h` holds sub-blocks `2h` and `2h+1`.
|
||||
- **Quants** (bytes 0..511): same nibble transposition as `Q4_0`, holding E2M1 values in element order
|
||||
(within a 16-element sub-block, `qs` byte `j` holds elements `j` and `j+8`).
|
||||
- **Scales** (bytes 512..575): UE4M3 bytes, `tile[512 + 32*u + row]` for sub-block `u = 0..1` of the k-tile.
|
||||
- The per-tensor F32 scale is not part of the tile.
|
||||
|
||||
- **Q4_1** (640 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q4_1`):
|
||||
- **Quants** (bytes 0..511, 512 bytes):
|
||||
Same 4-bit nibble transposition and storage layout as `Q4_0`.
|
||||
@@ -192,6 +199,10 @@ Byte ordering: `Q2_K`, `Q3_K`, and `Q6_K` use `vrmpy` byte order within each 128
|
||||
- `bytes 256..319`: raw `scales[]` bytes (`min << 4 | sc`): `[0..31]` for k 0..15, `[32..63]` for k 16..31.
|
||||
- `bytes 320..335`: 8 FP16 in tile `j` (0..7) of super-block: `d` (`j < 4`) or `-dmin` (`j >= 4`) for rows `8*(j%4)..8*(j%4)+7`.
|
||||
|
||||
- **IQ2_XXS** (264 bytes per tile, `HTP_TENSOR_TILE_SIZE_IQ2_XXS`, lossless tiling with same size as `block_iq2_xxs`):
|
||||
- `bytes 0..255`: `qs[4*kt .. 4*kt+3]` (grid indices, signs and scales of one 32-element group) of row `r` at `8*r`.
|
||||
- `bytes 256..263`: 4 FP16 `d` in tile `j` (0..7) of super-block: rows `4*j..4*j+3`.
|
||||
|
||||
- **Q3_K** (512 bytes per tile, `HTP_TENSOR_TILE_SIZE_Q3_K`):
|
||||
- `bytes 0..255` (vec 0..1): low 2 bits; vec `m` holds groups `4m..4m+3` at bit offsets 0, 2, 4, 6.
|
||||
- `bytes 256..383` (vec 2): sign mask, bit `g` set when `hmask` is 0 (quant = `low2 - 4`).
|
||||
|
||||
@@ -0,0 +1,469 @@
|
||||
# Text-to-speech (TTS)
|
||||
|
||||
llama.cpp supports audio generation via `libmtmd`. The following tools can be used:
|
||||
- [llama-tts](../../tools/tts/README.md), see its README for common usage and params
|
||||
- [llama-server](../../tools/server/README.md) via the `/tts` endpoint and the OpenAI-compatible `/v1/audio/speech` API
|
||||
|
||||
Supported models:
|
||||
- [Qwen3-TTS](#qwen3-tts)
|
||||
- [Pocket TTS](#pocket-tts)
|
||||
- [Soprano](#soprano)
|
||||
- [LFM3-Audio](#lfm3-audio)
|
||||
- [Chatterbox](#chatterbox)
|
||||
- [Drama 3](#drama-3)
|
||||
- [ZONOS3](#zonos3)
|
||||
- [Fish Audio S2 Pro](#fish-audio-s2-pro)
|
||||
- [YuE2](#yue2) (songs)
|
||||
- [MiniMax Music 3](#minimax-music-3) (songs)
|
||||
- [Kokoro](#kokoro)
|
||||
- [tiny-voca](#tiny-voca)
|
||||
|
||||
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
|
||||
|
||||
## Qwen3-TTS
|
||||
|
||||
Available params:
|
||||
- `--tts-lang` can be `zh`, `en`, `de`, `it`, `pt`, `es`, `ja`, `ko`, `fr`, `ru` (default: `en`)
|
||||
- `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3)
|
||||
- `--tts-speaker-text` is the transcript of the speaker file. With it, the speech continues from the reference codes (the "ICL" mode of the reference), which clones the voice more closely. It needs a clip of 30 s or less and an mmproj converted with the speech tokenizer encoder, otherwise it falls back to the speaker embedding only
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \
|
||||
-p "Hello world" \
|
||||
--tts-lang english \
|
||||
--tts-speaker-file speaker.mp3 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Voice cloning with the transcript of the reference:
|
||||
|
||||
```sh
|
||||
llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \
|
||||
-p "Hello world" \
|
||||
--tts-speaker-file speaker.wav \
|
||||
--tts-speaker-text "The words spoken in speaker.wav." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
## Pocket TTS
|
||||
|
||||
Available params:
|
||||
- `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3). It is required, the model produces almost no audio without it
|
||||
- `--tts-voice` selects one of the predefined voices shipped with the weights (e.g. `giovanni`), when the GGUF holds them. The converter reads them from `embeddings/*.safetensors` next to the weights, of both repositories, and logs how closely each one is reproduced (some packs ship voice states computed with other weights, the converter warns above 5% and skips a voice only when it does not belong to the model at all)
|
||||
- Note: `lang` is not used, the language is a property of the weights
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m pocket-tts.gguf \
|
||||
-mm mmproj-pocket-tts.gguf \
|
||||
-p "Hello world" \
|
||||
--tts-speaker-file speaker.mp3 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
The weights [without voice cloning](https://huggingface.co/kyutai/pocket-tts-without-voice-cloning) ship a set of predefined voices (`embeddings/*.safetensors`) next to each language; the converter stores them in the mmproj, as the voice prompt embeddings, and leaves out the zeroed voice encoder. With such a model `--tts-speaker-file` is rejected and a voice is selected with `--tts-voice` (the first one when it is not given):
|
||||
|
||||
```sh
|
||||
llama-tts -m pocket-tts.gguf \
|
||||
-mm mmproj-pocket-tts.gguf \
|
||||
-p "Ciao a tutti" \
|
||||
--tts-voice giovanni \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The [upstream repository](https://huggingface.co/kyutai/pocket-tts) holds one complete model per language under `languages/`, next to a set of shared files at the root. Convert one of the `languages/<name>` directories, **not** the root directory:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/pocket-tts/languages/english --outfile pocket-tts.gguf
|
||||
python convert_hf_to_gguf.py path/to/pocket-tts/languages/english --mmproj --outfile mmproj-pocket-tts.gguf
|
||||
```
|
||||
|
||||
## Soprano
|
||||
|
||||
Available params:
|
||||
- `--tts-speaker-file` is not supported; the model uses a fixed voice
|
||||
- Note: `lang` is not used. Supply normalized English text
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m soprano.gguf \
|
||||
-mm mmproj-soprano.gguf \
|
||||
-p "Hello world" \
|
||||
--temp 0 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
Both [Soprano-80M](https://huggingface.co/ekwek/Soprano-80M) and [Soprano-1.1-80M](https://huggingface.co/ekwek/Soprano-1.1-80M) are supported. Convert both files from the same model directory, including `decoder.pth`:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --outtype f32 --outfile soprano.gguf
|
||||
python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --mmproj --outtype f16 --outfile mmproj-soprano.gguf
|
||||
```
|
||||
|
||||
## LFM3-Audio
|
||||
|
||||
Available params:
|
||||
- `--tts-lang` can be `ar`, `da`, `de`, `el`, `en`, `en-gb`, `es`, `fi`, `fr`, `he`, `it`, `ja`, `ko`, `nl`, `pl`, `pt`, `sv`, `tr`, `zh` (default: `en`), other values are used as the language name
|
||||
- `--tts-voice` can be `female` (default) or `male`
|
||||
- Note: `--tts-speaker-file` is not used, the model has no voice cloning
|
||||
- Note: `--top-k` and `--top-p` are not used, audio frames are sampled with the settings of the model card (LFM3-Audio: temperature 1.0, top-k 128, repetition penalty 1.1 over 4096 frames; LFM3-Omni: temperature 0.4, repetition penalty 1.2 over 64 frames), the `audio_*` keys of `generation_config.json` override them at conversion
|
||||
- Each frame is 80 ms, generation stops at the end-of-audio frame
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m lfm3-audio.gguf \
|
||||
-mm mmproj-lfm3-audio.gguf \
|
||||
-p "Hello world" \
|
||||
--tts-voice male \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
The mmproj holds the audio encoder (for ASR), the depthformer and the codec decoder, the latter two are exported when the checkpoint has an `audio_tokenizer` directory:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/lfm3-audio --outfile lfm3-audio.gguf
|
||||
python convert_hf_to_gguf.py path/to/lfm3-audio --mmproj --outfile mmproj-lfm3-audio.gguf
|
||||
```
|
||||
|
||||
## Chatterbox
|
||||
|
||||
Two variants are supported: [Chatterbox Turbo](https://huggingface.co/ResembleAI/chatterbox-turbo) (English) and [Chatterbox Multilingual V3](https://huggingface.co/ResembleAI/chatterbox) (23 languages).
|
||||
|
||||
Available params:
|
||||
- `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3). Optional, the predefined voice is used without it
|
||||
- `--tts-voice` can be `default`, the voice shipped with the model (`conds.pt`)
|
||||
- `--tts-lang` sets the language of the multilingual variant, for example `fr` (default: none). It is not used by Turbo
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m chatterbox-mtl.gguf \
|
||||
-mm mmproj-chatterbox-mtl.gguf \
|
||||
-p "Bonjour tout le monde" \
|
||||
--tts-lang fr \
|
||||
--tts-speaker-file speaker.mp3 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Note: the multilingual variant runs classifier-free guidance with a second sequence. `llama-tts` sets this up by itself, but `llama-server` does not support it yet.
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The upstream repositories ship raw checkpoints without `config.json`. Download the files below into a directory, then convert it as usual (the variant is detected from the talker file):
|
||||
- Turbo: `t3_turbo_v1.safetensors`, `s3gen_meanflow.safetensors`, `ve.safetensors`, `conds.pt` and the GPT-2 tokenizer files (`vocab.json`, `merges.txt`, `added_tokens.json`, `special_tokens_map.json`, `tokenizer_config.json`)
|
||||
- Multilingual V3: `t3_mtl23ls_v3.safetensors`, `s3gen_v3.safetensors`, `ve.safetensors`, `conds.pt`, `mtl_tokenizer.json`
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/chatterbox --outfile chatterbox.gguf
|
||||
python convert_hf_to_gguf.py path/to/chatterbox --mmproj --outfile mmproj-chatterbox.gguf
|
||||
```
|
||||
|
||||
## Drama 3
|
||||
|
||||
[Fish Audio Drama 3](https://huggingface.co/fishaudio/drama-3): a Qwen3.5-MoE backbone (35B, 3B active) samples each frame's main token, a codebook tower samples the 9 other codes, and the Fish DAC decodes them to 44.1 kHz audio.
|
||||
|
||||
Available params:
|
||||
- Speaker tags `<|speaker:0|>` to `<|speaker:15|>` in the text give each character its own voice, a text without a tag starts with `<|speaker:0|>`
|
||||
- The language is the one of the text, `--tts-lang` is not used
|
||||
- Note: `--tts-speaker-file` is not supported yet (voice cloning)
|
||||
- Note: `--top-k` and `--top-p` are not used, frames are sampled with the settings of the reference (main token: temperature 1.0, top-p 0.9, repetition-aware redraw over the last 16 frames; other codes: temperature 1.0, top-k 30, top-p 0.9)
|
||||
- Each frame is 2048 samples (about 46 ms), generation stops at the end-of-speech token
|
||||
- The codebook tower keeps the whole sequence in its KV cache, the prompt and the generated frames are limited to 4096 positions
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m drama3.gguf \
|
||||
-mm mmproj-drama3.gguf \
|
||||
-p "<|speaker:0|> Did you remember to buy the bread? <|speaker:1|> Of course I did, it is on the kitchen table." \
|
||||
-n 1000 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The routed experts of the checkpoint are FP8, they are dequantized at conversion. The mmproj holds the codebook tower and the decoder half of `codec.pth`, it is best kept in BF16:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/drama-3 --outtype q8_0 --outfile drama3.gguf
|
||||
python convert_hf_to_gguf.py path/to/drama-3 --mmproj --outtype bf16 --outfile mmproj-drama3.gguf
|
||||
```
|
||||
|
||||
## ZONOS3
|
||||
|
||||
[Zyphra ZONOS3 Mini](https://huggingface.co/Zyphra/ZONOS3-Mini): a hybrid Gated DeltaNet / attention MoE backbone (800M, 100M active) predicts the 10 codebooks of each frame with an inter-codebook delay, and the ZAC codec decodes them to 44.1 kHz audio.
|
||||
|
||||
Available params:
|
||||
- `--tts-voice` picks one of the predefined voices of the release (`Savannah_en-American_F_1` by default); `llama-server` lists them in `GET /props`. A voice is the prompt prefix of its reference codes, the default speaker mode of the reference
|
||||
- The language is the one of the text; `--tts-lang` (`lang` in `llama-server`) only selects the text normalization
|
||||
- Numbers are spelled out by the `tts` chat template of the GGUF: cardinals, decimals, money, percents, clock times, common ordinal forms (`1st`, `1er`, `1º`), English and German years. A 4-digit number is read as a year only after a word like `in`, `since`, a month name, or `im Jahr`, `seit` in German (`in 1990` -> "in nineteen ninety", `1234 people` -> "one thousand two hundred thirty-four people"). Languages: `en`, `fr`, `de`, `es`, `it`, `pt`, `ja` and `zh` (or `cmn`), the region is ignored (`fr_fr`, `pt_br`); English when the language is empty, other languages are left as written. Set the language to the one of the text: the separators differ, e.g. `1.234` is a thousand in German
|
||||
- Note: the reference uses NeMo text normalization, which also covers units, dates, abbreviations and Korean; supply normalized text for them
|
||||
- `--tts-speaker-file` clones the voice of a clip (the first 60 s are used) with the speaker encoder of the release, the "embedding" speaker mode of the reference
|
||||
- `llama-server` also takes `speed` (relative to 15 UTF-8 bytes per second) and `loudness` (LUFS), see the `/tts` endpoint; they pick the speaking-rate and loudness condition buckets of the checkpoint, the defaults leave them unknown
|
||||
- Note: `loudness` has a strong effect with a cloned voice; a predefined voice mostly keeps the loudness of its reference clip
|
||||
- Note: `--top-k` and `--top-p` are not used, frames are sampled with the settings of the reference: temperature 0.75 (`--temp` changes it), top-k 80, top-p 0.69, repetition penalty 2.4 over the last 96 frames of codebooks 0 to 5
|
||||
- Each frame is 2940 samples (about 67 ms), generation stops at the end-of-audio code
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m zonos3-mini.gguf \
|
||||
-mm mmproj-zonos3-mini.gguf \
|
||||
-p "The old lighthouse keeper climbed the spiral stairs every evening." \
|
||||
--tts-voice Imogen_en-British_F_1 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
Convert the root of the release repository, not the `lm/` subdirectory. The mmproj holds the codebook heads, the predefined voices, the speaker encoder and the `keep2_plain` codec decoder, it is best kept in F32:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/ZONOS3-Mini --outtype q8_0 --outfile zonos3-mini.gguf
|
||||
python convert_hf_to_gguf.py path/to/ZONOS3-Mini --mmproj --outtype f32 --outfile mmproj-zonos3-mini.gguf
|
||||
```
|
||||
|
||||
## Fish Audio S2 Pro
|
||||
|
||||
[Fish Audio S2 Pro](https://huggingface.co/fishaudio/s2-pro): a Qwen3 backbone (4B) samples each frame's semantic token, a 4-layer fast AR predicts the 9 other codes, and the Fish DAC (same as Drama 3) decodes them to 44.1 kHz audio.
|
||||
|
||||
Available params:
|
||||
- Speaker tags `<|speaker:0|>`, `<|speaker:1|>`, ... in the text give each character its own voice, a text without a tag starts with `<|speaker:0|>`
|
||||
- Inline instructions in brackets control the delivery, for example `[laughing]`, `[whisper]` or `[pause]`
|
||||
- The language is the one of the text, `--tts-lang` is not used
|
||||
- Voice cloning: `--tts-speaker-file` with the reference clip, optionally `--tts-speaker-text` with its transcript (it is put in the prompt as the reference does). The clip is encoded by the codec encoder half of `codec.pth`, it is truncated to the first 30 s
|
||||
- Note: `--temp`, `--top-k` and `--top-p` are not used (only `--seed`), frames are sampled with the settings of the reference (semantic token: repetition penalty 1.1 over the last 16 frames, top-k 30, top-p 0.8, temperature 0.8, or top-p 0.9 and temperature 1.0 when the last 4 frames repeat a token; other codes: greedy)
|
||||
- Each frame is 2048 samples (about 46 ms), generation stops at the end-of-speech token
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m s2-pro.gguf \
|
||||
-mm mmproj-s2-pro.gguf \
|
||||
-p "<|speaker:0|>Did you remember to buy the bread? <|speaker:1|>Of course I did [laughing] it is on the kitchen table." \
|
||||
-n 1000 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Voice cloning (`--tts-speaker-text` can be left out):
|
||||
|
||||
```sh
|
||||
llama-tts -m s2-pro.gguf \
|
||||
-mm mmproj-s2-pro.gguf \
|
||||
--tts-speaker-file reference.wav \
|
||||
--tts-speaker-text "The words spoken in reference.wav." \
|
||||
-p "Hello, this is a cloned voice." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The mmproj holds the fast AR and `codec.pth` (the encoder half for voice cloning, the decoder half for the audio), it is best kept in BF16. The codec encoder is always stored in F16/F32, small errors in it change the codes picked by its VQ:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/s2-pro --outtype q8_0 --outfile s2-pro.gguf
|
||||
python convert_hf_to_gguf.py path/to/s2-pro --mmproj --outtype bf16 --outfile mmproj-s2-pro.gguf
|
||||
```
|
||||
|
||||
## YuE2
|
||||
|
||||
[m-a-p/YuE2-3B](https://huggingface.co/m-a-p/YuE2-3B) generates songs (vocals and accompaniment) from a style prompt and lyrics, as 48 kHz stereo. The backbone (a Qwen3 text model) writes an ABC score, then one semantic code per 40 ms frame; the NAR half of the checkpoint turns the codes into VAE latents by flow matching, attending to the keys and values of the backbone, and the [YuE2-Vae](https://huggingface.co/m-a-p/YuE2-Vae) decoder makes the audio.
|
||||
|
||||
The prompt is the text of the reference request:
|
||||
|
||||
```
|
||||
[Tags]
|
||||
City Pop, upbeat, danceable, groovy bass, electric guitar, synth
|
||||
[Lyrics]
|
||||
[Verse]
|
||||
...
|
||||
```
|
||||
|
||||
Available params:
|
||||
- `--tts-voice` picks the planning mode of the reference (`cot`): `full` (default, a chord-annotated score), `melody` (a melody-only score) or `off` (no score)
|
||||
- Cover mode: append the score to the prompt as `\n[ABC]\n<score>`, with `--tts-voice melody` or `full`; the model then goes straight to the codes
|
||||
- The generation stops at the limits of the reference (4096 score tokens, then 9000 codes, 6 minutes); `-n` sets a lower limit on the score tokens and the codes together
|
||||
- The context must keep the default size (24576 positions, the one of the reference): a smaller `-c` is rejected when the model is loaded
|
||||
- Note: `--top-k` and `--top-p` are not used, both phases are sampled with the settings of the reference (`yue2_generation_config.json`); `--temp` changes the temperature of the codes
|
||||
- Note: the backbone context must use an F16 KV cache (default): the NAR pass loads its keys and values
|
||||
- The codes are sampled with classifier-free guidance like the reference: scale 1.01 with `--tts-voice off`, off (1.0) otherwise. `llama-server` takes `cfg_scale` to change it, the reference suggests 1.2 for a stronger text guidance. The uncond prompt runs as a second sequence, so the context must also hold it (the instruction, the score and the codes once more)
|
||||
- The NAR pass solves the flow-matching ODE in 32 midpoint steps like the reference; `llama-server` takes `flow_steps` to change it. The NAR time is about proportional to it: 16 steps are about 35% faster for the NAR and the VAE together, with a small deviation from the reference (about 0.2 dB of spectral difference); it grows with fewer steps (about 0.4 dB at 8, 0.8 dB at 4)
|
||||
- The NAR pass runs on chunks of codes like the reference: one chunk for a song that fits in 24576 positions, so the audio arrives after the last code. When streaming, each 30 s of codes is a chunk, processed during the generation, so the audio arrives in 30 s steps; the chunks are independent (each sees the prompt, the score and its own codes), which may be audible at the boundaries
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m yue2-3b.gguf \
|
||||
-mm mmproj-yue2-3b.gguf \
|
||||
-f song.txt \
|
||||
--output song.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The VAE is a separate repository, download it into the `vae` subdirectory of the model:
|
||||
|
||||
```sh
|
||||
hf download m-a-p/YuE2-3B --local-dir YuE2-3B
|
||||
hf download m-a-p/YuE2-Vae --local-dir YuE2-3B/vae
|
||||
python convert_hf_to_gguf.py YuE2-3B --outfile yue2-3b.gguf
|
||||
python convert_hf_to_gguf.py YuE2-3B --mmproj --outfile mmproj-yue2-3b.gguf
|
||||
```
|
||||
|
||||
The mmproj holds the NAR transformer layers (loaded with libllama as a Qwen3 text model, in the `--outtype` type), and the latent projections and the VAE decoder, always in F32.
|
||||
|
||||
## MiniMax Music 3
|
||||
|
||||
[MiniMaxAI/MiniMax-Music3](https://huggingface.co/MiniMaxAI/MiniMax-Music3) generates songs (vocals and accompaniment) from a music description and lyrics, as 44.1 kHz stereo. For each 40 ms frame, the backbone (a Qwen3 text model) samples the semantic code and the RVQ depth decoder the 7 other codes; the hidden states of both go to a flow-matching DiT, then a DAC decoder makes the audio. Both use the settings of the reference (the diffusers pipeline).
|
||||
|
||||
The prompt has the same format as YuE2, the description goes under `[Tags]`:
|
||||
|
||||
```
|
||||
[Tags]
|
||||
Genre: acoustic pop. BPM: 96. Key: C major. Vocals: soft female lead. Arrangement: fingerpicked guitar and soft piano.
|
||||
[Lyrics]
|
||||
[verse]
|
||||
Morning light filtering through the pine
|
||||
[chorus]
|
||||
Softly the world begins to breathe
|
||||
```
|
||||
|
||||
Available params:
|
||||
- The description and the lyrics are cleaned up like the reference: markdown is removed, a line that starts with structure tags (e.g. `[Verse]`) keeps only the tags
|
||||
- The generation stops at the end-of-audio token or at the limit of the reference (9000 frames, 6 minutes); `-n` sets a lower limit in frames (25 per second)
|
||||
- `--temp` changes the temperature of the codes (default 1.0); `--top-k` and `--top-p` are not used, the codes are sampled with top-k 50 like the reference
|
||||
- The codes are sampled with classifier-free guidance, scale 1.5 like the reference; `llama-server` takes `cfg_scale` to change it (1.0 disables it). The uncond prompt runs as a second sequence that shares the context, so a song needs about 2 x (prompt tokens + frames) positions: the default context (10240) holds about 3 minutes with a short description, use a larger `-c` (e.g. 20480) for longer songs
|
||||
- The flow matching runs 30 Euler steps like the reference; `llama-server` takes `flow_steps` to change it
|
||||
- The audio is made in windows of 200 frames (8 s) every 100 frames, like the reference: a window is processed during the generation once the frame after it exists, so with streaming the audio arrives in steps of about 4 s
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m minimax-music3.gguf \
|
||||
-mm mmproj-minimax-music3.gguf \
|
||||
-f song.txt \
|
||||
--output song.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The conversion reads the diffusers layout of the repository, the `qwen_7B` directory and the `.pth` files are not needed:
|
||||
|
||||
```sh
|
||||
hf download MiniMaxAI/MiniMax-Music3 --local-dir MiniMax-Music3 --exclude "qwen_7B/*" --exclude "*.pth"
|
||||
python convert_hf_to_gguf.py MiniMax-Music3 --outfile minimax-music3.gguf
|
||||
python convert_hf_to_gguf.py MiniMax-Music3 --mmproj --outfile mmproj-minimax-music3.gguf
|
||||
```
|
||||
|
||||
The mmproj holds the depth decoder and the DiT layers (in the `--outtype` type), and the condition encoder, the small DiT projections and the DAC decoder, always in F32.
|
||||
|
||||
## Kokoro
|
||||
|
||||
Available params:
|
||||
- `--tts-voice` selects one of the voices shipped with the weights (default: `af_heart`); `llama-tts` lists them when the name is unknown. The first letter of a voice is its language: `a` American English, `b` British English, `e` Spanish, `f` French, `h` Hindi, `i` Italian, `p` Brazilian Portuguese, `j` Japanese, `z` Mandarin
|
||||
- `--tts-lang` can be `en-us` or `en-gb`; when it is not given, the voices named `b*` speak British English and those named `a*` American English
|
||||
- Note: `--tts-speaker-file` is not supported, the model has no voice cloning
|
||||
- Note: sampling params are not used, the model is not autoregressive. `--seed` only changes the noise of the vocoder source
|
||||
- `speed` (`llama-server` only) changes the speaking rate, e.g. `1.5`; `loudness` is not supported
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m kokoro.gguf \
|
||||
-mm mmproj-kokoro.gguf \
|
||||
-p "Hello world" \
|
||||
--tts-voice am_michael \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
The model speaks phonemes. Only English has a text to phonemes step (the model's `tts` chat template, with the pronunciation dictionary stored in the model):
|
||||
- the pronunciation of a word missing from the dictionary (mostly rare names) is predicted from its spelling, by a model that the converter learns from the dictionary. The reference uses espeak-ng for these words; the prediction is about as accurate, but it is not the same, so such a word can sound different from the reference
|
||||
- a text longer than 510 phonemes is split at sentence ends and spoken segment by segment
|
||||
|
||||
**Other languages, or your own phonemes:**
|
||||
|
||||
The prompt of a voice of another language must already be phonemes, in the symbols of the model (see `vocab` in its `config.json`). Run the text to phonemes step in your own code, for example with [espeak-ng](https://github.com/espeak-ng/espeak-ng), which the reference uses for Spanish, French, Hindi, Italian and Brazilian Portuguese:
|
||||
|
||||
```sh
|
||||
PHONEMES=$(echo "El gobierno anunció nuevas medidas." | espeak-ng -q --ipa -v es --stdin | tr '\n' ' ')
|
||||
llama-tts -m kokoro.gguf \
|
||||
-mm mmproj-kokoro.gguf \
|
||||
-p "$PHONEMES" \
|
||||
--tts-voice ef_dora \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Notes:
|
||||
- espeak-ng drops the punctuation, which the model uses for pauses: phonemize the text between the marks and put them back for better prosody
|
||||
- the reference also replaces a few sounds by single symbols (e.g. `tʃ` by `ʧ`), see `espeak.py` of [misaki](https://github.com/hexgrad/misaki)
|
||||
- with `llama-server`, send `"raw_prompt": true` to pass phonemes with any voice, including the English ones
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
Download `config.json`, `kokoro-v1_0.pth` and the `voices/*.pt` files you want from [Kokoro-82M](https://huggingface.co/hexgrad/Kokoro-82M), every voice in the directory is stored in the mmproj:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/Kokoro-82M --outfile kokoro.gguf
|
||||
python convert_hf_to_gguf.py path/to/Kokoro-82M --mmproj --outfile mmproj-kokoro.gguf
|
||||
```
|
||||
|
||||
The pronunciation dictionary and the part-of-speech tagger of the `tts` template come from the `misaki` and `nltk` python packages. The converter also learns the spelling-to-sound model for unknown words from that dictionary, which adds about 15 seconds:
|
||||
|
||||
```sh
|
||||
pip install misaki nltk
|
||||
python -m nltk.downloader averaged_perceptron_tagger_eng
|
||||
```
|
||||
|
||||
Without them the converter writes no `tts` template, and the prompt must be a phoneme string (as produced by misaki).
|
||||
|
||||
## tiny-voca
|
||||
|
||||
[tiny-voca](https://huggingface.co/n4ze3m/tiny-voca): a 4.5M-parameter English TTS model. A phoneme encoder (the text model), then a duration predictor, two frame transformers and a source-filter synthesizer (in the mmproj).
|
||||
|
||||
Available params:
|
||||
- `--tts-voice` can be `M2` (default) or `F2`
|
||||
- Note: `--tts-speaker-file` is not supported, the model has no voice cloning
|
||||
- Note: sampling params are not used, the model is not autoregressive. `--seed` only changes the noise of the synthesizer
|
||||
- `speed` (`llama-server` only) changes the speaking rate, from `0.25` to `4`; `loudness` is not supported
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m tiny-voca.gguf \
|
||||
-mm mmproj-tiny-voca.gguf \
|
||||
-p "Hello! The meeting moved to 3:30 pm." \
|
||||
--tts-voice F2 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
The model speaks phonemes. The text to phonemes step is the model's `tts` chat template, a port of [dizione](https://github.com/n4ze3m/dizione) (the reference one), with its pronunciation dictionary and tables stored in the model:
|
||||
- the choice of a pronunciation for each word: exceptions and names, the dictionary, function words and their sentence stress, words spelled alike and read differently ("I read it yesterday", "to read"), hyphenated words, capitals read letter by letter, mixed case ("iPhone"), stretched words ("soooo"), accents, chat spellings ("lol", "btw")
|
||||
- a word missing from the dictionary is built from a known word when it can be (endings, prefixes, compounds), else it goes through the unknown-word model of dizione (stored in the mmproj) and its conversion rules
|
||||
- a pronunciation can be given in the text, in CMUdict notation or as phonemes: `Say {HH AH0 L OW1} to Samir{s ə m ˈɪɹ}`
|
||||
- the text rules are ported too: numbers, money, times, dates, telephone numbers, versions, codes, units, fractions, ranges, math signs and comparisons, web addresses and file names, abbreviations, chat spellings
|
||||
|
||||
With `llama-server`, send `"raw_prompt": true` to pass phonemes directly, in the format printed by dizione: symbols separated by spaces, `<sp>` between words, a stress mark (`ˈ` or `ˌ`) before a stressed vowel, e.g. `h ə l ˈoʊ <sp> w ˈɜː l d .`
|
||||
|
||||
Each sentence is spoken as one utterance. The template splits the text as the reference does (before a capital or a figure, not after a title or an initial), ends each sentence with `<eos>`, and adds a full stop when the text ends without a mark. Phonemes without `<eos>` are cut after each `.`, `?`, `!` or `…`, and a sentence without a final mark gets a full stop.
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
Convert the root of the repository (it needs `config.json`, `tiny-voca.pt` and `phonemizer/lexicon.tsv`). The other tables of the `tts` template come from the `dizione` python package; without it the converter writes no `tts` template, and the prompt must be phonemes:
|
||||
|
||||
```sh
|
||||
pip install git+https://github.com/n4ze3m/dizione
|
||||
```
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/tiny-voca --outfile tiny-voca.gguf
|
||||
python convert_hf_to_gguf.py path/to/tiny-voca --mmproj --outfile mmproj-tiny-voca.gguf
|
||||
```
|
||||
+122
-119
@@ -12,122 +12,125 @@ Legend:
|
||||
- 🟡 Partially supported by this backend
|
||||
- ❌ Not supported by this backend
|
||||
|
||||
| Operation | BLAS | CANN | CPU | CUDA | ET | HTP | MTL | OpenCL | SYCL | Vulkan | WebGPU | ZenDNN | zDNN |
|
||||
|-----------|------|------|------|------|------|------|------|------|------|------|------|------|------|
|
||||
| ABS | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ACC | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ADD1 | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| ADD_ID | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CLAMP | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| COL2IM_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONCAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| CONT | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| CONV_1D_DW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ |
|
||||
| CONV_2D | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
|
||||
| CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CONV_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| COS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CROSS_ENTROPY_LOSS_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DSV4_HC_COMB | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DSV4_HC_POST | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DSV4_HC_PRE | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DUP | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| FLASH_ATTN_EXT | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GATED_LINEAR_ATTN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ |
|
||||
| GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ |
|
||||
| GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| IM2COL | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | 🟡 | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| LIGHTNING_INDEXER | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| LOG | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 |
|
||||
| MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 |
|
||||
| MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ |
|
||||
| MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ |
|
||||
| PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ |
|
||||
| POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| REPEAT_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| RMS_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RMS_NORM_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROLL | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROPE | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ROPE_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROUND | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RWKV_WKV6 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| RWKV_WKV7 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| SCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SET | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SET_ROWS | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| SGN | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SILU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| SIN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
|
||||
| SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SQRT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ❌ |
|
||||
| STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SUB | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SUM | ❌ | 🟡 | ✅ | 🟡 | ❌ | 🟡 | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| SUM_ROWS | ❌ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SWIGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SWIGLU_CLAMP | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| SWIGLU_OAI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TANH | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TIMESTEP_EMBEDDING | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
|
||||
| TOP_K_MASK | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| UPSCALE | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| XIELU | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| Operation | BLAS | CANN | CPU | CUDA | ET | HTP | MTL | OPENVINO | OpenCL | SYCL | Vulkan | WebGPU | ZenDNN | zDNN |
|
||||
|-----------|------|------|------|------|------|------|------|------|------|------|------|------|------|------|
|
||||
| ABS | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ACC | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ADD1 | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| ADD_ID | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CLAMP | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| COL2IM_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONCAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| CONT | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| CONV_1D_DW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ |
|
||||
| CONV_2D | ❌ | ❌ | 🟡 | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
|
||||
| CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CONV_3D | ❌ | ❌ | ✅ | ✅ | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| COS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CROSS_ENTROPY_LOSS_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DSV4_HC_COMB | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DSV4_HC_POST | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DSV4_HC_PRE | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DUP | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| FLASH_ATTN_EXT | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GATED_LINEAR_ATTN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ |
|
||||
| GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ |
|
||||
| GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| IM2COL | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| LIGHTNING_INDEXER | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| LOG | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 |
|
||||
| MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 |
|
||||
| MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ |
|
||||
| MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_PREC_SRC1_BF16 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_PREC_SRC1_BF16_SRC0 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_PREC_SRC1_F32 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ |
|
||||
| PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ |
|
||||
| POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| REPEAT_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| RMS_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RMS_NORM_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROLL | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROPE | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ROPE_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROUND | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RWKV_WKV6 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| RWKV_WKV7 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| SCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SET | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SET_ROWS | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| SGN | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SILU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| SIN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
|
||||
| SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SQRT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ❌ |
|
||||
| STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SUB | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SUM | ❌ | 🟡 | ✅ | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| SUM_ROWS | ❌ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SWIGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SWIGLU_CLAMP | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| SWIGLU_OAI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TANH | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TIMESTEP_EMBEDDING | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
|
||||
| TOP_K_MASK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| UPSCALE | ❌ | 🟡 | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| XIELU | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
|
||||
+7280
-5968
File diff suppressed because it is too large
Load Diff
+23719
File diff suppressed because it is too large
Load Diff
+6
-6
@@ -19,8 +19,8 @@
|
||||
"OpenCL","GELU","type=f16,ne_a=[5,7,11,13],v=0","support","0","no","OpenCL"
|
||||
"OpenCL","GELU_QUICK","type=f16,ne_a=[128,2,2,2],v=0","support","0","no","OpenCL"
|
||||
"OpenCL","GELU_QUICK","type=f16,ne_a=[5,7,11,13],v=0","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[128,2,2,2],v=0","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[5,7,11,13],v=0","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[128,2,2,2],v=0","support","1","yes","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[5,7,11,13],v=0","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSWISH","type=f16,ne_a=[128,2,2,2],v=0","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSWISH","type=f16,ne_a=[5,7,11,13],v=0","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSIGMOID","type=f16,ne_a=[128,2,2,2],v=0","support","1","yes","OpenCL"
|
||||
@@ -61,8 +61,8 @@
|
||||
"OpenCL","GELU","type=f16,ne_a=[5,7,11,13],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","GELU_QUICK","type=f16,ne_a=[128,2,2,2],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","GELU_QUICK","type=f16,ne_a=[5,7,11,13],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[128,2,2,2],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[5,7,11,13],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[128,2,2,2],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[5,7,11,13],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSWISH","type=f16,ne_a=[128,2,2,2],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSWISH","type=f16,ne_a=[5,7,11,13],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSIGMOID","type=f16,ne_a=[128,2,2,2],v=1","support","1","yes","OpenCL"
|
||||
@@ -145,8 +145,8 @@
|
||||
"OpenCL","GELU","type=f32,ne_a=[5,7,11,13],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","GELU_QUICK","type=f32,ne_a=[128,2,2,2],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","GELU_QUICK","type=f32,ne_a=[5,7,11,13],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f32,ne_a=[128,2,2,2],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f32,ne_a=[5,7,11,13],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f32,ne_a=[128,2,2,2],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","SILU","type=f32,ne_a=[5,7,11,13],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSWISH","type=f32,ne_a=[128,2,2,2],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSWISH","type=f32,ne_a=[5,7,11,13],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSIGMOID","type=f32,ne_a=[128,2,2,2],v=1","support","1","yes","OpenCL"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -19439,3 +19439,31 @@
|
||||
"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=1,type_K=q4_1","support","0","no","Vulkan"
|
||||
"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=1,type_K=q4_0","support","0","no","Vulkan"
|
||||
"Vulkan0","LIGHTNING_INDEXER","hsk=128,nh=64,kv=256,nb=512,ns=4,nm=1,type_K=iq4_nl","support","0","no","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=256,n_blk=4,n_pick=64,n_top=16,n_extra=3,score=1,nb=32,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=1001,n_blk=4,n_pick=70,n_top=16,n_extra=3,score=1,nb=7,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=256,n_blk=4,n_pick=64,n_top=64,n_extra=0,score=1,nb=33,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=512,n_blk=8,n_pick=16,n_top=32,n_extra=7,score=1,nb=5,nh=1,ns=4,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=4096,n_blk=4,n_pick=1024,n_top=512,n_extra=3,score=1,nb=1,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=4096,n_blk=4,n_pick=1024,n_top=512,n_extra=3,score=1,nb=64,nh=1,ns=2,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=256,n_blk=0,n_pick=256,n_top=16,n_extra=0,score=0,nb=32,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=1001,n_blk=0,n_pick=1001,n_top=64,n_extra=0,score=0,nb=7,nh=1,ns=3,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=4096,n_blk=0,n_pick=4096,n_top=2048,n_extra=0,score=0,nb=5,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=512,n_blk=0,n_pick=512,n_top=32,n_extra=5,score=1,nb=9,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=512,n_blk=16,n_pick=32,n_top=4,n_extra=0,score=1,nb=6,nh=4,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=1024,n_blk=8,n_pick=128,n_top=16,n_extra=8,score=0,nb=3,nh=2,ns=2,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=1001,n_blk=4,n_pick=70,n_top=16,n_extra=3,score=1,nb=7,nh=1,ns=1,real=0,bcast=1","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f16,n_kv=512,n_blk=8,n_pick=16,n_top=32,n_extra=7,score=1,nb=5,nh=1,ns=4,real=0,bcast=1","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=256,n_blk=4,n_pick=64,n_top=16,n_extra=3,score=1,nb=32,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=1001,n_blk=4,n_pick=70,n_top=16,n_extra=3,score=1,nb=7,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=256,n_blk=4,n_pick=64,n_top=64,n_extra=0,score=1,nb=33,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=512,n_blk=8,n_pick=16,n_top=32,n_extra=7,score=1,nb=5,nh=1,ns=4,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=4096,n_blk=4,n_pick=1024,n_top=512,n_extra=3,score=1,nb=1,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=4096,n_blk=4,n_pick=1024,n_top=512,n_extra=3,score=1,nb=64,nh=1,ns=2,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=256,n_blk=0,n_pick=256,n_top=16,n_extra=0,score=0,nb=32,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=1001,n_blk=0,n_pick=1001,n_top=64,n_extra=0,score=0,nb=7,nh=1,ns=3,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=4096,n_blk=0,n_pick=4096,n_top=2048,n_extra=0,score=0,nb=5,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=512,n_blk=0,n_pick=512,n_top=32,n_extra=5,score=1,nb=9,nh=1,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=512,n_blk=16,n_pick=32,n_top=4,n_extra=0,score=1,nb=6,nh=4,ns=1,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=1024,n_blk=8,n_pick=128,n_top=16,n_extra=8,score=0,nb=3,nh=2,ns=2,real=0,bcast=0","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=1001,n_blk=4,n_pick=70,n_top=16,n_extra=3,score=1,nb=7,nh=1,ns=1,real=0,bcast=1","support","1","yes","Vulkan"
|
||||
"Vulkan0","TOP_K_MASK","type=f32,n_kv=512,n_blk=8,n_pick=16,n_top=32,n_extra=7,score=1,nb=5,nh=1,ns=4,real=0,bcast=1","support","1","yes","Vulkan"
|
||||
|
||||
|
Can't render this file because it is too large.
|
+1
-1
@@ -12351,7 +12351,7 @@
|
||||
"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.000000,step=0.250000","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.250000,step=0.250000","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=1048576.000000,step=1.000000","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","TIMESTEP_EMBEDDING","type=f32,ne_a=[2,1,1,1],dim=320,max_period=10000","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","TIMESTEP_EMBEDDING","type=f32,ne_a=[2,1,1,1],dim=320,max_period=10000","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","LEAKY_RELU","type=f32,ne_a=[10,5,4,3],negative_slope=0.100000","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","CUMSUM","type=f32,ne=[10,5,4,3]","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","CUMSUM","type=f32,ne=[127,5,4,3]","support","1","yes","WebGPU"
|
||||
|
||||
|
Can't render this file because it is too large.
|
+53
-3
@@ -74,13 +74,41 @@ llama-server -m Qwen3-4B.gguf -md Qwen3-4B-DFlash.gguf \
|
||||
|
||||
`--spec-draft-n-max` is clamped to the draft model's trained block size.
|
||||
|
||||
DFlash2, DSpark and LiliCorr drafts are DFlash backbones with a head on the block, and the head is read from the draft model.
|
||||
`draft-dflash` runs each of them with the driver of its head.
|
||||
The other spec types must match the head of the draft model: `draft-dspark` needs the Markov head, `draft-dflash2` the candidate selector and `draft-lilicorr` the candidate scorer. On a mismatch, speculative decoding is not initialized.
|
||||
Without `--spec-type`, the type is detected from the draft model.
|
||||
|
||||
See:
|
||||
|
||||
- #22105
|
||||
|
||||
### DFlash2 (`draft-dflash2`)
|
||||
|
||||
[DFlash2](https://inco.ai/blog/dflash2/) adds a local convolution to the DFlash layers and a _candidate selector_ on the block.
|
||||
|
||||
DFlash2 drafts convert like DFlash drafts, with `--target-model-dir`:
|
||||
|
||||
```bash
|
||||
python convert_hf_to_gguf.py <dflash2-draft-dir> \
|
||||
--target-model-dir <target-model-dir> --outtype bf16 --outfile DFlash2.gguf
|
||||
|
||||
llama-server -m target.gguf -md DFlash2.gguf \
|
||||
--spec-type draft-dflash2 --spec-draft-n-max 7 -fa on --jinja
|
||||
```
|
||||
|
||||
`--spec-draft-p-min P` truncates the draft at the first step whose selector probability falls below `P`.
|
||||
|
||||
With `--spec-draft-sampling probabilistic`, the draft samples the path at the request's temperature
|
||||
and the target verifies it by rejection sampling.
|
||||
|
||||
See:
|
||||
|
||||
- #27816, #30663
|
||||
|
||||
### DSpark (`draft-dspark`)
|
||||
|
||||
DSpark extends DFlash with a semi-autoregressive _Markov head_: the draft still emits a whole
|
||||
[DSpark](https://arxiv.org/pdf/2607.05147) extends DFlash with a semi-autoregressive _Markov head_: the draft still emits a whole
|
||||
block per forward pass, but each block position's logits are biased by a low-rank term keyed on
|
||||
the previous token, chained in-graph across the block. This keeps drafting at one decode per
|
||||
block while recovering some of the left-to-right signal that pure block diffusion loses.
|
||||
@@ -115,6 +143,26 @@ See:
|
||||
|
||||
- #25173
|
||||
|
||||
### LiliCorr (`draft-lilicorr`)
|
||||
|
||||
[LiliCorr](https://arxiv.org/pdf/2608.20530) puts a _candidate scorer_ on the DFlash block: a small transformer over the top-k candidates
|
||||
of each block position, conditioned on the target's hidden state at the last position, scores the paths through the block like the DFlash2 selector.
|
||||
|
||||
LiliCorr drafts (`LiLiCorrDraftModel`) convert with `--target-model-dir`:
|
||||
|
||||
```bash
|
||||
python convert_hf_to_gguf.py <lilicorr-draft-dir> \
|
||||
--target-model-dir <target-model-dir> --outtype bf16 --outfile LiliCorr.gguf
|
||||
|
||||
llama-server -m target.gguf -md LiliCorr.gguf --spec-type draft-lilicorr -fa on --jinja
|
||||
```
|
||||
|
||||
`--spec-draft-p-min P` truncates the draft at the first step whose scorer probability falls below `P`.
|
||||
|
||||
See:
|
||||
|
||||
- #30506
|
||||
|
||||
### n-gram Cache (`ngram-cache`)
|
||||
|
||||
An n-gram is a sequence of n tokens. The n-gram cache implementation maintains statistics about short n-gram sequences.
|
||||
@@ -225,7 +273,7 @@ Use exactly one of these options:
|
||||
### General Speculative Parameters
|
||||
|
||||
```
|
||||
--spec-type [none|draft-simple|draft-eagle3|draft-dflash|draft-dspark|draft-mtp|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod]
|
||||
--spec-type [none|draft-simple|draft-eagle3|draft-dflash|draft-dflash2|draft-dspark|draft-lilicorr|draft-mtp|ngram-cache|ngram-simple|ngram-map-k|ngram-map-k4v|ngram-mod]
|
||||
comma-separated list of types of speculative decoding to use
|
||||
(default: none)
|
||||
(env: LLAMA_ARG_SPEC_TYPE)
|
||||
@@ -368,8 +416,10 @@ Specifies a comma-separated list of speculative decoding types to use.
|
||||
| `none` | No speculative decoding (default) |
|
||||
| `draft-simple` | Use a simple draft model for speculation |
|
||||
| `draft-eagle3` | Use an EAGLE-3 draft model that reads the target's hidden states |
|
||||
| `draft-dflash` | Use a DFlash block-diffusion draft model that emits a block per step |
|
||||
| `draft-dflash` | Use a DFlash block-diffusion draft model that emits a block per step (also DFlash2, DSpark and LiliCorr drafts, resolved from the model) |
|
||||
| `draft-dflash2` | Use a DFlash2 draft model (DFlash + local convolution + candidate selector) |
|
||||
| `draft-dspark` | Use a DSpark draft model (DFlash backbone + semi-autoregressive Markov head) |
|
||||
| `draft-lilicorr` | Use a LiliCorr draft model (DFlash + candidate scorer conditioned on the target's hidden state) |
|
||||
| `draft-mtp` | Use Multi Token Prediction (MTP) heads from the main model |
|
||||
| `ngram-cache` | Use n-gram cache lookup |
|
||||
| `ngram-simple` | Use simple n-gram pattern matching |
|
||||
|
||||
@@ -7,6 +7,10 @@ Jinja has no regex support, and exposing `std::regex` / Python `re` would not he
|
||||
- `tts_norm_scan(text, rules)`: split text into matched and unmatched segments
|
||||
- `tts_norm_sub(text, pattern, repl)`: replace every match
|
||||
|
||||
A third function, `tts_dict(name, key)`, gives the template access to dictionaries stored in the model (e.g. a pronunciation lexicon).
|
||||
|
||||
A fourth one, `tts_g2p(word)`, gives the pronunciation of a word by the letter-to-sound model of the mmproj, when the caller provides one (see below).
|
||||
|
||||
Implementations:
|
||||
- C++: [common/chat-tts-norm.cpp](../common/chat-tts-norm.cpp), injected into task templates by `common_chat_template_render()` in [common/chat.cpp](../common/chat.cpp). They are not available to chat templates.
|
||||
- Python reference: [scripts/chat_tts_norm.py](../scripts/chat_tts_norm.py), for use with `jinja2` (`env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan`)
|
||||
@@ -69,6 +73,33 @@ Replaces every match of `pattern`, with the same scanning rules as `tts_norm_sca
|
||||
|
||||
Any other `%` sequence in `repl`, or a reference to a group that does not exist, is an error.
|
||||
|
||||
### `tts_dict(name, key)`
|
||||
|
||||
Looks up `key` in the dictionary `name` of the model and returns its value, or `none` if the key is missing. An unknown dictionary name is an error. A model can ship any number of dictionaries, for example a pronunciation lexicon per language:
|
||||
|
||||
```jinja
|
||||
{%- set e = tts_dict('lexicon_en', word) -%}
|
||||
{%- if e is none -%}{{ spell(word) }}
|
||||
{%- elif e is mapping -%}{{ e[tag] if tag in e else e.DEFAULT }}
|
||||
{%- else -%}{{ e }}
|
||||
{%- endif -%}
|
||||
```
|
||||
|
||||
The lookup is exact: no case folding and no normalization, do them in the template.
|
||||
|
||||
The GGUF lists the names of the dictionaries in `tokenizer.tts_dicts`. Each one is stored as two arrays of strings of the same length: `tokenizer.tts_dict.<name>.keys`, sorted by the UTF-8 bytes of the key, and `tokenizer.tts_dict.<name>.values`. A value is a JSON document (string, number, list, object, ...) and is returned decoded. `GGUFWriter.add_tts_dicts(dicts)` writes all of this.
|
||||
|
||||
### `tts_g2p(word)`
|
||||
|
||||
Pronunciation of `word` by a letter-to-sound model, as phonemes separated by spaces, in the notation of the model (e.g. CMUdict with stress digits for tiny-voca), or `none`. The template converts it to the symbols of the model. It is `none` when the model has no letter-to-sound model, or when the caller gives none: `common_chat_get_tts_prompt()` takes it as an optional callback, `llama-tts` and `llama-server` pass `mtmd_gen_audio_g2p()`. A template must keep working without it, e.g. by spelling the word.
|
||||
|
||||
```jinja
|
||||
{%- set g = tts_g2p(word) -%}
|
||||
{%- if g is none -%}{{ spell(word) }}{%- else -%}{{ from_cmudict(word, g.split(' ')) }}{%- endif -%}
|
||||
```
|
||||
|
||||
There is no Python reference for it: a test passes its own function (`env.globals["tts_g2p"]`).
|
||||
|
||||
## Pattern syntax
|
||||
|
||||
Patterns operate on Unicode codepoints. Invalid UTF-8 bytes (C++ only) are seen as U+FFFD, one per byte, and preserved in the output.
|
||||
|
||||
@@ -87,36 +87,32 @@ for id: llama_token in tokens {
|
||||
|
||||
print("\n")
|
||||
|
||||
var batch = llama_batch_init(max(Int32(tokens.count), Int32(n_parallel)), 0, 1)
|
||||
let batch = llama_batch_ext_init(context)
|
||||
defer {
|
||||
llama_batch_free(batch)
|
||||
llama_batch_ext_free(batch)
|
||||
}
|
||||
|
||||
// evaluate the initial prompt
|
||||
batch.n_tokens = Int32(tokens.count)
|
||||
|
||||
for (i, token) in tokens.enumerated() {
|
||||
batch.token[i] = token
|
||||
batch.pos[i] = Int32(i)
|
||||
batch.n_seq_id[i] = 1
|
||||
// batch.seq_id[i][0] = 0
|
||||
// TODO: is this the proper way to do this?
|
||||
if let seq_id = batch.seq_id[i] {
|
||||
seq_id[0] = 0
|
||||
let idx = llama_batch_ext_add_token(batch, 0, token)
|
||||
if idx < 0 {
|
||||
print("llama_batch_ext_add_token() failed: \(idx)")
|
||||
exit(1)
|
||||
}
|
||||
batch.logits[i] = 0
|
||||
var pos = llama_pos(i)
|
||||
llama_batch_ext_set_pos(batch, idx, &pos)
|
||||
}
|
||||
|
||||
// llama_decode will output logits only for the last token of the prompt
|
||||
batch.logits[Int(batch.n_tokens) - 1] = 1
|
||||
// llama_process will output logits only for the last token of the prompt
|
||||
llama_batch_ext_set_output_logits(batch, Int32(tokens.count) - 1, true)
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed")
|
||||
exit(1)
|
||||
}
|
||||
|
||||
for i in 1 ..< n_parallel {
|
||||
llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, batch.n_tokens)
|
||||
llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, Int32(tokens.count))
|
||||
}
|
||||
|
||||
if n_parallel > 1 {
|
||||
@@ -125,16 +121,17 @@ if n_parallel > 1 {
|
||||
|
||||
var streams: [String] = .init(repeating: "", count: n_parallel)
|
||||
var streamBuffers: [[CChar]] = .init(repeating: [], count: n_parallel)
|
||||
var i_batch = [Int32](repeating: batch.n_tokens - 1, count: n_parallel)
|
||||
var i_batch = [Int32](repeating: Int32(tokens.count) - 1, count: n_parallel)
|
||||
|
||||
var n_cur = batch.n_tokens
|
||||
var n_cur = Int32(tokens.count)
|
||||
var n_decode = 0
|
||||
|
||||
let t_main_start = ggml_time_us()
|
||||
|
||||
while n_cur <= n_len {
|
||||
// prepare the next batch
|
||||
batch.n_tokens = 0
|
||||
llama_batch_ext_clear(batch)
|
||||
var n_tokens: Int32 = 0
|
||||
|
||||
// sample the next token for each parallel sequence / stream
|
||||
for i in 0 ..< n_parallel {
|
||||
@@ -165,31 +162,32 @@ while n_cur <= n_len {
|
||||
streams[i] += nextStringPiece
|
||||
|
||||
// push this new token for next evaluation
|
||||
batch.token[Int(batch.n_tokens)] = new_token_id
|
||||
batch.pos[Int(batch.n_tokens)] = n_cur
|
||||
batch.n_seq_id[Int(batch.n_tokens)] = 1
|
||||
if let seq_id = batch.seq_id[Int(batch.n_tokens)] {
|
||||
seq_id[0] = Int32(i)
|
||||
let idx = llama_batch_ext_add_token(batch, Int32(i), new_token_id)
|
||||
if idx < 0 {
|
||||
print("llama_batch_ext_add_token() failed: \(idx)")
|
||||
exit(1)
|
||||
}
|
||||
batch.logits[Int(batch.n_tokens)] = 1
|
||||
var pos = n_cur
|
||||
llama_batch_ext_set_pos(batch, idx, &pos)
|
||||
llama_batch_ext_set_output_logits(batch, idx, true)
|
||||
|
||||
i_batch[i] = batch.n_tokens
|
||||
i_batch[i] = idx
|
||||
|
||||
batch.n_tokens += 1
|
||||
n_tokens += 1
|
||||
|
||||
n_decode += 1
|
||||
}
|
||||
|
||||
// all streams are finished
|
||||
if batch.n_tokens == 0 {
|
||||
if n_tokens == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
n_cur += 1
|
||||
|
||||
// evaluate the current batch with the transformer model
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed")
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,20 +5,16 @@ enum LlamaError: Error {
|
||||
case couldNotInitializeContext
|
||||
}
|
||||
|
||||
func llama_batch_clear(_ batch: inout llama_batch) {
|
||||
batch.n_tokens = 0
|
||||
}
|
||||
|
||||
func llama_batch_add(_ batch: inout llama_batch, _ id: llama_token, _ pos: llama_pos, _ seq_ids: [llama_seq_id], _ logits: Bool) {
|
||||
batch.token [Int(batch.n_tokens)] = id
|
||||
batch.pos [Int(batch.n_tokens)] = pos
|
||||
batch.n_seq_id[Int(batch.n_tokens)] = Int32(seq_ids.count)
|
||||
for i in 0..<seq_ids.count {
|
||||
batch.seq_id[Int(batch.n_tokens)]![Int(i)] = seq_ids[i]
|
||||
@discardableResult
|
||||
func llama_batch_add(_ batch: OpaquePointer, _ id: llama_token, _ pos: llama_pos, _ seq_id: llama_seq_id, _ logits: Bool) -> Int32 {
|
||||
let idx = llama_batch_ext_add_token(batch, seq_id, id)
|
||||
if idx < 0 {
|
||||
return idx
|
||||
}
|
||||
batch.logits [Int(batch.n_tokens)] = logits ? 1 : 0
|
||||
|
||||
batch.n_tokens += 1
|
||||
var pos = pos
|
||||
llama_batch_ext_set_pos(batch, idx, &pos)
|
||||
llama_batch_ext_set_output_logits(batch, idx, logits)
|
||||
return idx
|
||||
}
|
||||
|
||||
actor LlamaContext {
|
||||
@@ -26,7 +22,7 @@ actor LlamaContext {
|
||||
private var context: OpaquePointer
|
||||
private var vocab: OpaquePointer
|
||||
private var sampling: UnsafeMutablePointer<llama_sampler>
|
||||
private var batch: llama_batch
|
||||
private var batch: OpaquePointer
|
||||
private var tokens_list: [llama_token]
|
||||
var is_done: Bool = false
|
||||
|
||||
@@ -42,7 +38,7 @@ actor LlamaContext {
|
||||
self.model = model
|
||||
self.context = context
|
||||
self.tokens_list = []
|
||||
self.batch = llama_batch_init(512, 0, 1)
|
||||
self.batch = llama_batch_ext_init(context)
|
||||
self.temporary_invalid_cchars = []
|
||||
let sparams = llama_sampler_chain_default_params()
|
||||
self.sampling = llama_sampler_chain_init(sparams)
|
||||
@@ -53,7 +49,7 @@ actor LlamaContext {
|
||||
|
||||
deinit {
|
||||
llama_sampler_free(sampling)
|
||||
llama_batch_free(batch)
|
||||
llama_batch_ext_free(batch)
|
||||
llama_model_free(model)
|
||||
llama_free(context)
|
||||
llama_backend_free()
|
||||
@@ -110,10 +106,6 @@ actor LlamaContext {
|
||||
return SwiftString
|
||||
}
|
||||
|
||||
func get_n_tokens() -> Int32 {
|
||||
return batch.n_tokens;
|
||||
}
|
||||
|
||||
func completion_init(text: String) {
|
||||
print("attempting to complete \"\(text)\"")
|
||||
|
||||
@@ -133,25 +125,27 @@ actor LlamaContext {
|
||||
print(String(cString: token_to_piece(token: id) + [0]))
|
||||
}
|
||||
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_ext_clear(batch)
|
||||
|
||||
for i1 in 0..<tokens_list.count {
|
||||
let i = Int(i1)
|
||||
llama_batch_add(&batch, tokens_list[i], Int32(i), [0], false)
|
||||
}
|
||||
batch.logits[Int(batch.n_tokens) - 1] = 1 // true
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed")
|
||||
let idx = llama_batch_add(batch, tokens_list[i], Int32(i), 0, i == tokens_list.count - 1)
|
||||
if idx < 0 {
|
||||
print("llama_batch_add() failed: \(idx)")
|
||||
}
|
||||
}
|
||||
|
||||
n_cur = batch.n_tokens
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed")
|
||||
}
|
||||
|
||||
n_cur = Int32(tokens_list.count)
|
||||
}
|
||||
|
||||
func completion_loop() -> String {
|
||||
var new_token_id: llama_token = 0
|
||||
|
||||
new_token_id = llama_sampler_sample(sampling, context, batch.n_tokens - 1)
|
||||
new_token_id = llama_sampler_sample(sampling, context, -1)
|
||||
|
||||
if llama_vocab_is_eog(vocab, new_token_id) || n_cur == n_len {
|
||||
print("\n")
|
||||
@@ -178,13 +172,13 @@ actor LlamaContext {
|
||||
print(new_token_str)
|
||||
// tokens_list.append(new_token_id)
|
||||
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_add(&batch, new_token_id, n_cur, [0], true)
|
||||
llama_batch_ext_clear(batch)
|
||||
llama_batch_add(batch, new_token_id, n_cur, 0, true)
|
||||
|
||||
n_decode += 1
|
||||
n_cur += 1
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("failed to evaluate llama!")
|
||||
}
|
||||
|
||||
@@ -201,21 +195,20 @@ actor LlamaContext {
|
||||
for _ in 0..<nr {
|
||||
// bench prompt processing
|
||||
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_ext_clear(batch)
|
||||
|
||||
let n_tokens = pp
|
||||
|
||||
for i in 0..<n_tokens {
|
||||
llama_batch_add(&batch, 0, Int32(i), [0], false)
|
||||
llama_batch_add(batch, 0, Int32(i), 0, i == n_tokens - 1)
|
||||
}
|
||||
batch.logits[Int(batch.n_tokens) - 1] = 1 // true
|
||||
|
||||
llama_memory_clear(llama_get_memory(context), false)
|
||||
|
||||
let t_pp_start = DispatchTime.now().uptimeNanoseconds / 1000;
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed during prompt")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed during prompt")
|
||||
}
|
||||
llama_synchronize(context)
|
||||
|
||||
@@ -228,14 +221,14 @@ actor LlamaContext {
|
||||
let t_tg_start = DispatchTime.now().uptimeNanoseconds / 1000;
|
||||
|
||||
for i in 0..<tg {
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_ext_clear(batch)
|
||||
|
||||
for j in 0..<pl {
|
||||
llama_batch_add(&batch, 0, Int32(i), [Int32(j)], true)
|
||||
llama_batch_add(batch, 0, Int32(i), Int32(j), true)
|
||||
}
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed during text generation")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed during text generation")
|
||||
}
|
||||
llama_synchronize(context)
|
||||
}
|
||||
|
||||
@@ -116,7 +116,12 @@ int main(int argc, char ** argv) {
|
||||
// init the speculator
|
||||
const auto & params_spec = params.speculative;
|
||||
|
||||
struct common_speculative * spec = common_speculative_init(params.speculative, 1);
|
||||
struct common_speculative * spec = nullptr;
|
||||
try {
|
||||
spec = common_speculative_init(params.speculative, 1);
|
||||
} catch (const std::exception & e) {
|
||||
LOG_ERR("%s: %s\n", __func__, e.what());
|
||||
}
|
||||
|
||||
if (spec == nullptr) {
|
||||
LOG_ERR("%s", "failed to initialize speculative decoding\n");
|
||||
|
||||
@@ -164,6 +164,11 @@
|
||||
}
|
||||
// lib.optionalAttrs (system == "x86_64-linux") {
|
||||
rocm = config.legacyPackages.llamaPackagesRocm.llama-cpp;
|
||||
|
||||
sycl = config.packages.default.override {
|
||||
useSycl = true;
|
||||
syclToolkit = pkgs.intel-oneapi-toolkit;
|
||||
};
|
||||
};
|
||||
|
||||
# Packages exposed in `.#checks` will be built by the CI and by
|
||||
|
||||
@@ -228,6 +228,7 @@ option(GGML_VULKAN_MEMORY_DEBUG "ggml: enable Vulkan memory debug ou
|
||||
option(GGML_VULKAN_SHADER_DEBUG_INFO "ggml: enable Vulkan shader debug info" OFF)
|
||||
option(GGML_VULKAN_VALIDATE "ggml: enable Vulkan validation" OFF)
|
||||
option(GGML_VULKAN_RUN_TESTS "ggml: run Vulkan tests" OFF)
|
||||
set (GGML_VULKAN_SPIRV_INCLUDE_DIR "" CACHE PATH "ggml: override the SPIRV-Headers include directory")
|
||||
option(GGML_WEBGPU "ggml: use WebGPU" OFF)
|
||||
option(GGML_WEBGPU_DEBUG "ggml: enable WebGPU debug output" OFF)
|
||||
option(GGML_WEBGPU_CPU_PROFILE "ggml: enable WebGPU profiling (CPU)" OFF)
|
||||
|
||||
@@ -1727,6 +1727,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
|
||||
split->i_start = 0;
|
||||
split->n_inputs = 0;
|
||||
int cur_backend_id = split->backend_id;
|
||||
bool split_has_nodes = false;
|
||||
for (; i < graph->n_nodes; i++) {
|
||||
struct ggml_tensor * node = graph->nodes[i];
|
||||
|
||||
@@ -1739,8 +1740,9 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
|
||||
GGML_ASSERT(node_backend_id != -1); // all nodes should be assigned by now, this can happen if there is no CPU fallback
|
||||
|
||||
// check if we should start a new split based on the sources of the current node
|
||||
// this also applies to a split without inputs, e.g. one that follows a split that only writes to a host buffer
|
||||
bool need_new_split = false;
|
||||
if (node_backend_id == cur_backend_id && split->n_inputs > 0) {
|
||||
if (node_backend_id == cur_backend_id && split_has_nodes) {
|
||||
for (int j = 0; j < GGML_MAX_SRC; j++) {
|
||||
struct ggml_tensor * src = node->src[j];
|
||||
if (src == NULL) {
|
||||
@@ -1777,6 +1779,7 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
|
||||
split->n_inputs = 0;
|
||||
cur_backend_id = node_backend_id;
|
||||
}
|
||||
split_has_nodes = true;
|
||||
|
||||
// find inputs that are not on the same backend
|
||||
for (int j = 0; j < GGML_MAX_SRC; j++) {
|
||||
|
||||
@@ -2601,6 +2601,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
|
||||
}
|
||||
case GGML_OP_SUM:
|
||||
return ggml_is_contiguous_rows(op->src[0]);
|
||||
case GGML_OP_RMS_NORM:
|
||||
// the cached gamma of ones does not track its type, so F32 and BF16 can not share it
|
||||
return op->src[0]->type != GGML_TYPE_BF16;
|
||||
case GGML_OP_L2_NORM:
|
||||
case GGML_OP_CROSS_ENTROPY_LOSS:
|
||||
case GGML_OP_DUP:
|
||||
@@ -2617,7 +2620,6 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
|
||||
case GGML_OP_SUB:
|
||||
case GGML_OP_MUL:
|
||||
case GGML_OP_DIV:
|
||||
case GGML_OP_RMS_NORM:
|
||||
case GGML_OP_SQR:
|
||||
case GGML_OP_SQRT:
|
||||
case GGML_OP_CLAMP:
|
||||
|
||||
@@ -698,21 +698,39 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
#endif
|
||||
}
|
||||
|
||||
#if defined(__AVX2__)
|
||||
#if defined(__AVX2__) || defined(__AVX512F__)
|
||||
// four consecutive Q2_0 weights per byte: byte b becomes b & 3, (b >> 2) & 3, (b >> 4) & 3, (b >> 6) & 3 (OR, never add: the overlapping copies only touch masked-off bits)
|
||||
static inline __m256i q2_0_unpack_32_avx2(const uint8_t * qs) {
|
||||
#if defined(__AVX512VBMI__) && defined(__AVX512VL__)
|
||||
const __m256i shifts = _mm256_setr_epi8(
|
||||
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
|
||||
32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62);
|
||||
const __m256i bits = _mm256_broadcastq_epi64(_mm_loadl_epi64((const __m128i *) qs));
|
||||
return _mm256_and_si256(_mm256_multishift_epi64_epi8(shifts, bits), _mm256_set1_epi8(3));
|
||||
#else
|
||||
__m256i v = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *) qs));
|
||||
v = _mm256_or_si256(v, _mm256_slli_epi32(v, 12));
|
||||
v = _mm256_or_si256(v, _mm256_slli_epi32(v, 6));
|
||||
return _mm256_and_si256(v, _mm256_set1_epi32(0x03030303));
|
||||
#endif
|
||||
}
|
||||
|
||||
// 32 weights against one Q8_0 block: sum((v - 1) * y) = sum(v * y) - sum(y), exact for every int8 y
|
||||
static inline __m256 q2_0_dot_32_avx2(const uint8_t * qs, const int8_t * qy) {
|
||||
const __m256i v = q2_0_unpack_32_avx2(qs);
|
||||
const __m256i y = _mm256_loadu_si256((const __m256i *) qy);
|
||||
#if defined(__AVX512VNNI__) && defined(__AVX512VL__)
|
||||
const __m256i p = _mm256_dpbusd_epi32(_mm256_setzero_si256(), v, y);
|
||||
const __m256i s = _mm256_dpbusd_epi32(_mm256_setzero_si256(), _mm256_set1_epi8(1), y);
|
||||
return _mm256_cvtepi32_ps(_mm256_sub_epi32(p, s));
|
||||
#elif defined(__AVXVNNI__)
|
||||
const __m256i p = _mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), v, y);
|
||||
const __m256i s = _mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), _mm256_set1_epi8(1), y);
|
||||
return _mm256_cvtepi32_ps(_mm256_sub_epi32(p, s));
|
||||
#else
|
||||
const __m256i p = _mm256_sub_epi16(_mm256_maddubs_epi16(v, y), _mm256_maddubs_epi16(_mm256_set1_epi8(1), y));
|
||||
return _mm256_cvtepi32_ps(_mm256_madd_epi16(p, _mm256_set1_epi16(1)));
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __m256 q2_0_acc_block_avx2(const block_q2_0 * x, const block_q8_0 * y, __m256 acc) {
|
||||
|
||||
@@ -1309,14 +1309,12 @@ class tinyBLAS_RVV {
|
||||
// QUANT ZERO MATRIX MULTIPLICATION
|
||||
|
||||
#if defined(__VXE__) || defined(__VXE2__)
|
||||
template <typename TA>
|
||||
template <typename TA, typename TB>
|
||||
class tinyBLAS_Q0_VXE {
|
||||
static_assert(sizeof(TA::qs) == QK8_0, "VXE weight loads require 32 quantized bytes per block");
|
||||
|
||||
public:
|
||||
tinyBLAS_Q0_VXE(int64_t k,
|
||||
const TA * A, int64_t lda,
|
||||
const block_q8_0 * B, int64_t ldb,
|
||||
const TB * B, int64_t ldb,
|
||||
float * C, int64_t ldc,
|
||||
int32_t ith, int32_t nth)
|
||||
: A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
|
||||
@@ -1333,44 +1331,54 @@ class tinyBLAS_Q0_VXE {
|
||||
for (int64_t job = start; job < end; ++job) {
|
||||
const int64_t ii = tile_size * (job % row_tiles);
|
||||
const int64_t jj = tile_size * (job / row_tiles);
|
||||
|
||||
float sum_min[tile_size][tile_size] = {};
|
||||
float32x4_t v_sum[tile_size][tile_size] = {};
|
||||
|
||||
for (int64_t l = 0; l < k; ++l) {
|
||||
int8x16_t v_xl[tile_size], v_xh[tile_size], v_yl[tile_size], v_yh[tile_size];
|
||||
int8x16_t v_xl[tile_size], v_xh[tile_size];
|
||||
int8x16_t v_yl[tile_size], v_yh[tile_size];
|
||||
|
||||
float x_scale[tile_size], y_scale[tile_size];
|
||||
float x_min[tile_size], y_sum[tile_size];
|
||||
|
||||
for (int32_t i = 0; i < tile_size; ++i) {
|
||||
const TA & x = A[lda * (ii + i) + l];
|
||||
const block_q8_0 & y = B[ldb * (jj + i) + l];
|
||||
const TB & y = B[ldb * (jj + i) + l];
|
||||
|
||||
load(x, v_xl[i], v_xh[i]);
|
||||
load(y, v_yl[i], v_yh[i]);
|
||||
|
||||
const int8_t * x_qs = x.qs;
|
||||
const int8_t * y_qs = y.qs;
|
||||
v_xl[i] = vec_xl(0, x_qs);
|
||||
v_xh[i] = vec_xl(0, x_qs + QK8_0 / 2);
|
||||
v_yl[i] = vec_xl(0, y_qs);
|
||||
v_yh[i] = vec_xl(0, y_qs + QK8_0 / 2);
|
||||
x_scale[i] = unhalf(x.d);
|
||||
y_scale[i] = unhalf(y.d);
|
||||
x_min[i] = min(x);
|
||||
y_sum[i] = sum(y);
|
||||
}
|
||||
|
||||
for (int32_t j = 0; j < tile_size; ++j) {
|
||||
for (int32_t i = 0; i < tile_size; ++i) {
|
||||
const int32x4_t v_dot = dot_s8_wide(v_xl[i], v_yl[j]) + dot_s8_wide(v_xh[i], v_yh[j]);
|
||||
const int32x4_t v_dot = dot(v_xl[i], v_xh[i], v_yl[j], v_yh[j]);
|
||||
const float32x4_t v_scale = vec_splats(x_scale[i] * y_scale[j]);
|
||||
v_sum[j][i] = vec_madd(vec_float(v_dot), v_scale, v_sum[j][i]);
|
||||
if (has_min) {
|
||||
sum_min[j][i] += x_min[i] * y_sum[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (int32_t j = 0; j < tile_size; ++j) {
|
||||
for (int32_t i = 0; i < tile_size; ++i) {
|
||||
C[ldc * (jj + j) + ii + i] = hsum(v_sum[j][i]);
|
||||
C[ldc * (jj + j) + ii + i] = hsum(v_sum[j][i]) + sum_min[j][i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
// Q4_1 and Q5_1 weights have a block minimum, it pairs with the Q8_1 block sum.
|
||||
static constexpr bool has_min = std::is_same<TB, block_q8_1>::value;
|
||||
|
||||
static int32x4_t dot_s8_wide(int8x16_t v_x, int8x16_t v_y) {
|
||||
const int16x8_t v_even = vec_mule(v_x, v_y);
|
||||
const int16x8_t v_odd = vec_mulo(v_x, v_y);
|
||||
@@ -1378,8 +1386,86 @@ class tinyBLAS_Q0_VXE {
|
||||
return vec_unpackh(v_even) + vec_unpackl(v_even) + vec_unpackh(v_odd) + vec_unpackl(v_odd);
|
||||
}
|
||||
|
||||
static int32x4_t dot(int8x16_t v_xl, int8x16_t v_xh, int8x16_t v_yl, int8x16_t v_yh) {
|
||||
// block_q8_0 path
|
||||
if (std::is_same<TA, block_q8_0>::value) {
|
||||
return dot_s8_wide(v_xl, v_yl) + dot_s8_wide(v_xh, v_yh);
|
||||
}
|
||||
|
||||
// block_q8_1 path
|
||||
const int16x8_t v_one = vec_splats((int16_t)1);
|
||||
const int16x8_t v_xy = vec_meadd(v_xl, v_yl, vec_moadd(v_xl, v_yl, vec_meadd(v_xh, v_yh, vec_mulo(v_xh, v_yh))));
|
||||
return vec_meadd(v_xy, v_one, vec_mulo(v_xy, v_one));
|
||||
}
|
||||
|
||||
static void load(const block_q4_0 & x, int8x16_t & v_xl, int8x16_t & v_xh) {
|
||||
const uint8_t * x_qs = x.qs;
|
||||
const uint8x16_t v_x = vec_xl(0, x_qs);
|
||||
v_xl = (int8x16_t)vec_and(v_x, vec_splats((uint8_t)0x0F)) - vec_splats((int8_t)8);
|
||||
v_xh = (int8x16_t)vec_sr(v_x, 4) - vec_splats((int8_t)8);
|
||||
}
|
||||
|
||||
static void load(const block_q4_1 & x, int8x16_t & v_xl, int8x16_t & v_xh) {
|
||||
const uint8_t * x_qs = x.qs;
|
||||
const uint8x16_t v_x = vec_xl(0, x_qs);
|
||||
v_xl = (int8x16_t)vec_and(v_x, vec_splats((uint8_t)0x0F));
|
||||
v_xh = (int8x16_t)vec_sr(v_x, 4);
|
||||
}
|
||||
|
||||
static void load(const block_q5_0 & x, int8x16_t & v_xl, int8x16_t & v_xh) {
|
||||
const uint8_t * x_qs = x.qs;
|
||||
const uint8x16_t v_x = vec_xl(0, x_qs);
|
||||
uint8x16_t v_qhl, v_qhh;
|
||||
load_qh(x.qh, v_qhl, v_qhh);
|
||||
v_xl = (int8x16_t)vec_or(vec_and(v_x, vec_splats((uint8_t)0x0F)), v_qhl) - vec_splats((int8_t)16);
|
||||
v_xh = (int8x16_t)vec_or(vec_sr(v_x, 4), v_qhh) - vec_splats((int8_t)16);
|
||||
}
|
||||
|
||||
static void load(const block_q5_1 & x, int8x16_t & v_xl, int8x16_t & v_xh) {
|
||||
const uint8_t * x_qs = x.qs;
|
||||
const uint8x16_t v_x = vec_xl(0, x_qs);
|
||||
uint8x16_t v_qhl, v_qhh;
|
||||
load_qh(x.qh, v_qhl, v_qhh);
|
||||
v_xl = (int8x16_t)vec_or(vec_and(v_x, vec_splats((uint8_t)0x0F)), v_qhl);
|
||||
v_xh = (int8x16_t)vec_or(vec_sr(v_x, 4), v_qhh);
|
||||
}
|
||||
|
||||
static void load(const block_q8_0 & x, int8x16_t & v_xl, int8x16_t & v_xh) {
|
||||
const int8_t * x_qs = x.qs;
|
||||
v_xl = vec_xl(0, x_qs);
|
||||
v_xh = vec_xl(0, x_qs + QK8_0 / 2);
|
||||
}
|
||||
|
||||
static void load(const block_q8_1 & x, int8x16_t & v_xl, int8x16_t & v_xh) {
|
||||
const int8_t * x_qs = x.qs;
|
||||
v_xl = vec_xl(0, x_qs);
|
||||
v_xh = vec_xl(0, x_qs + QK8_1 / 2);
|
||||
}
|
||||
|
||||
// expand the 32 high bits to one byte each, with value 0 or 16
|
||||
static void load_qh(const uint8_t * qh, uint8x16_t & v_qhl, uint8x16_t & v_qhh) {
|
||||
// qh is a native 32-bit value, on big-endian its lowest bits are in byte 3
|
||||
const uint8x16_t v_bytel = { 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2 };
|
||||
const uint8x16_t v_byteh = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0 };
|
||||
const uint8x16_t v_bit = { 1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128 };
|
||||
|
||||
uint32_t tmp;
|
||||
memcpy(&tmp, qh, sizeof(tmp));
|
||||
const uint8x16_t v_qh = (uint8x16_t)vec_splats(tmp);
|
||||
|
||||
v_qhl = (uint8x16_t)vec_cmpeq(vec_and(vec_perm(v_qh, v_qh, v_bytel), v_bit), v_bit) & vec_splats((uint8_t)16);
|
||||
v_qhh = (uint8x16_t)vec_cmpeq(vec_and(vec_perm(v_qh, v_qh, v_byteh), v_bit), v_bit) & vec_splats((uint8_t)16);
|
||||
}
|
||||
|
||||
static float min(const block_q4_1 & x) { return unhalf(x.m); }
|
||||
static float min(const block_q5_1 & x) { return unhalf(x.m); }
|
||||
template <typename T> static float min(const T &) { return 0.0f; }
|
||||
|
||||
static float sum(const block_q8_0 & ) { return 0.0f; }
|
||||
static float sum(const block_q8_1 & y) { return unhalf(y.s); }
|
||||
|
||||
const TA * const A;
|
||||
const block_q8_0 * const B;
|
||||
const TB * const B;
|
||||
float * const C;
|
||||
const int64_t k;
|
||||
const int64_t lda;
|
||||
@@ -4262,7 +4348,7 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
|
||||
if (m % 2 || n % 2) {
|
||||
return false;
|
||||
}
|
||||
tinyBLAS_Q0_VXE<block_q8_0> tb{
|
||||
tinyBLAS_Q0_VXE<block_q8_0, block_q8_0> tb{
|
||||
k, (const block_q8_0 *)A, lda,
|
||||
(const block_q8_0 *)B, ldb,
|
||||
(float *)C, ldc,
|
||||
@@ -4306,6 +4392,17 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#elif defined(__VXE__) || defined(__VXE2__)
|
||||
if (m % 2 || n % 2) {
|
||||
return false;
|
||||
}
|
||||
tinyBLAS_Q0_VXE<block_q4_0, block_q8_0> tb{
|
||||
k, (const block_q4_0 *)A, lda,
|
||||
(const block_q8_0 *)B, ldb,
|
||||
(float *)C, ldc,
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#elif defined(__MMA__)
|
||||
//TO-DO: Remove this condition once gemv forwarding is enabled.
|
||||
if (n < 8 && n != 4)
|
||||
@@ -4335,6 +4432,57 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#elif defined(__VXE__) || defined(__VXE2__)
|
||||
if (m % 2 || n % 2) {
|
||||
return false;
|
||||
}
|
||||
tinyBLAS_Q0_VXE<block_q5_0, block_q8_0> tb{
|
||||
k, (const block_q5_0 *)A, lda,
|
||||
(const block_q8_0 *)B, ldb,
|
||||
(float *)C, ldc,
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
case GGML_TYPE_Q4_1: {
|
||||
if (Btype != GGML_TYPE_Q8_1) {
|
||||
return false;
|
||||
}
|
||||
#if defined(__VXE__) || defined(__VXE2__)
|
||||
if (m % 2 || n % 2) {
|
||||
return false;
|
||||
}
|
||||
tinyBLAS_Q0_VXE<block_q4_1, block_q8_1> tb{
|
||||
k, (const block_q4_1 *)A, lda,
|
||||
(const block_q8_1 *)B, ldb,
|
||||
(float *)C, ldc,
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
case GGML_TYPE_Q5_1: {
|
||||
if (Btype != GGML_TYPE_Q8_1) {
|
||||
return false;
|
||||
}
|
||||
#if defined(__VXE__) || defined(__VXE2__)
|
||||
if (m % 2 || n % 2) {
|
||||
return false;
|
||||
}
|
||||
tinyBLAS_Q0_VXE<block_q5_1, block_q8_1> tb{
|
||||
k, (const block_q5_1 *)A, lda,
|
||||
(const block_q8_1 *)B, ldb,
|
||||
(float *)C, ldc,
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user