mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-17 08:17:40 -06:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7f106b5459 | ||
|
|
d92ea9bb12 | ||
|
|
8d1599fca1 | ||
|
|
054a1babf3 | ||
|
|
513ad7ae07 | ||
|
|
67cf8967eb | ||
|
|
4df3f2959c | ||
|
|
a49db739fb | ||
|
|
eada89c315 | ||
|
|
8f647f1fdb | ||
|
|
dde9de82cb | ||
|
|
e0a9118fe9 | ||
|
|
140a279874 | ||
|
|
c145db2728 | ||
|
|
ede6106780 | ||
|
|
9214bdb554 | ||
|
|
d3c3e242c1 | ||
|
|
9581154a22 | ||
|
|
1ef8c5d684 | ||
|
|
e8fbcad855 | ||
|
|
95e837aaf1 | ||
|
|
d998c676ff | ||
|
|
92192d31e4 | ||
|
|
ad5cbfa911 | ||
|
|
d261e49e7e | ||
|
|
4ddf2b9470 | ||
|
|
4303cd94ae | ||
|
|
c7518dbec5 | ||
|
|
a68c6136bb | ||
|
|
3a7660371d | ||
|
|
e8698ac163 | ||
|
|
7804be0634 | ||
|
|
ef6bc6f367 | ||
|
|
a06044b934 | ||
|
|
62a6d3a302 | ||
|
|
ef3848f9d7 | ||
|
|
47fe133318 | ||
|
|
301db63370 | ||
|
|
a001af915e | ||
|
|
164ab018f9 | ||
|
|
5cb9516798 | ||
|
|
8bfd64b5ac | ||
|
|
e1925b3695 | ||
|
|
95db855bce | ||
|
|
1d2e880bad | ||
|
|
adf554d239 | ||
|
|
6b01a4767f | ||
|
|
f477c1b90a | ||
|
|
21fa9c043f | ||
|
|
a946eb541a | ||
|
|
05db09a4d3 | ||
|
|
9fba51d2c9 | ||
|
|
b4c0e00a4f | ||
|
|
1719dbb0ee | ||
|
|
fc504f91d8 | ||
|
|
18b74fa41d | ||
|
|
7ae3c6b231 | ||
|
|
f4d0673917 | ||
|
|
a681a53b60 | ||
|
|
75fd7f9ec9 | ||
|
|
a68799c609 | ||
|
|
56fd8a375f | ||
|
|
b54752df40 | ||
|
|
900f01ba09 | ||
|
|
de32b268c4 | ||
|
|
4e5116f476 | ||
|
|
278a89cb78 | ||
|
|
dd099a6bf3 | ||
|
|
73f53317eb | ||
|
|
429460dc0e | ||
|
|
58a4a6b49c | ||
|
|
5a4afd8077 | ||
|
|
6d3568e31d | ||
|
|
e574a17b06 | ||
|
|
5c008e0008 | ||
|
|
9f26fe11c7 | ||
|
|
cf9a9957da | ||
|
|
dca216ad63 | ||
|
|
f8cc5774e7 | ||
|
|
acf5d54562 | ||
|
|
3e41ae69db | ||
|
|
3bc612fe23 | ||
|
|
d8f30a5b6d | ||
|
|
e964175653 | ||
|
|
d20426bbbe | ||
|
|
19f31e61b0 | ||
|
|
07037ba133 | ||
|
|
0cc522e45d | ||
|
|
f08bca8a46 | ||
|
|
3c311796f2 | ||
|
|
8b4ac2496c | ||
|
|
1ae047c926 | ||
|
|
1c2adc75f6 | ||
|
|
a956a852c9 | ||
|
|
c6c0ec6d5c |
@@ -19,12 +19,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${CANN_BASE_IMAGE} AS build
|
||||
@@ -40,7 +43,7 @@ WORKDIR /app
|
||||
# -- Copy project files --
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
# -- Set CANN environment variables (required for compilation) --
|
||||
# Using ENV instead of `source` allows environment variables to persist across the entire image layer
|
||||
|
||||
@@ -9,12 +9,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -28,7 +31,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON; \
|
||||
|
||||
@@ -17,12 +17,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_CUDA_DEV_CONTAINER} AS build
|
||||
@@ -40,7 +43,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "${CUDA_DOCKER_ARCH}" != "default" ]; then \
|
||||
export CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CUDA_DOCKER_ARCH}"; \
|
||||
|
||||
@@ -11,12 +11,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build
|
||||
@@ -41,7 +44,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "${GGML_SYCL_F16}" = "ON" ]; then \
|
||||
echo "GGML_SYCL_F16 is set" \
|
||||
|
||||
@@ -15,12 +15,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_MUSA_DEV_CONTAINER} AS build
|
||||
@@ -45,7 +48,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "${MUSA_DOCKER_ARCH}" != "default" ]; then \
|
||||
export CMAKE_ARGS="-DMUSA_ARCHITECTURES=${MUSA_DOCKER_ARCH}"; \
|
||||
|
||||
+138
-41
@@ -22,12 +22,19 @@
|
||||
spirv-headers,
|
||||
nodejs,
|
||||
importNpmLock,
|
||||
syclToolkit ? null,
|
||||
level-zero,
|
||||
ocl-icd,
|
||||
intel-graphics-compiler,
|
||||
intel-compute-runtime,
|
||||
makeBinaryWrapper,
|
||||
useBlas ?
|
||||
builtins.all (x: !x) [
|
||||
useCuda
|
||||
useMetalKit
|
||||
useRocm
|
||||
useVulkan
|
||||
useSycl
|
||||
]
|
||||
&& blas.meta.available,
|
||||
useCuda ? config.cudaSupport,
|
||||
@@ -38,11 +45,28 @@
|
||||
rocmGpuTargets ? builtins.concatStringsSep ";" rocmPackages.clr.gpuTargets,
|
||||
useVulkan ? false,
|
||||
useRpc ? false,
|
||||
useSycl ? false,
|
||||
# Rebuild, not a runtime switch. cmake default is off.
|
||||
syclF16 ? false,
|
||||
# null keeps JIT. AOT makes icpx run ocloc, which DPC++ does not ship.
|
||||
syclDeviceArch ? null,
|
||||
syclMaxParallelLinkJobs ? null,
|
||||
llamaVersion ? "0.0.0", # Arbitrary version, substituted by the flake
|
||||
|
||||
# It's necessary to consistently use backendStdenv when building with CUDA support,
|
||||
# otherwise we get libstdc++ errors downstream.
|
||||
effectiveStdenv ? if useCuda then cudaPackages.backendStdenv else stdenv,
|
||||
# CUDA needs its backend stdenv or libstdc++ mismatches downstream.
|
||||
# SYCL needs icpx which is provided by the oneAPI stdenv
|
||||
effectiveStdenv ?
|
||||
if useCuda && useSycl then
|
||||
throw "llama-cpp: useCuda and useSycl are mutually exclusive"
|
||||
else if useCuda then
|
||||
cudaPackages.backendStdenv
|
||||
else if useSycl then
|
||||
if syclToolkit == null then
|
||||
throw "llama-cpp: useSycl requires syclToolkit, consider setting it to intel-oneapi-toolkit"
|
||||
else
|
||||
syclToolkit.stdenv
|
||||
else
|
||||
stdenv,
|
||||
enableStatic ? effectiveStdenv.hostPlatform.isStatic,
|
||||
precompileMetalShaders ? false,
|
||||
useWebUi ? true,
|
||||
@@ -65,7 +89,8 @@ let
|
||||
++ lib.optionals useMetalKit [ "MetalKit" ]
|
||||
++ lib.optionals useMpi [ "MPI" ]
|
||||
++ lib.optionals useRocm [ "ROCm" ]
|
||||
++ lib.optionals useVulkan [ "Vulkan" ];
|
||||
++ lib.optionals useVulkan [ "Vulkan" ]
|
||||
++ lib.optionals useSycl [ "SYCL" ];
|
||||
|
||||
pnameSuffix =
|
||||
strings.optionalString (suffices != [ ])
|
||||
@@ -108,6 +133,26 @@ let
|
||||
shaderc
|
||||
spirv-headers
|
||||
];
|
||||
|
||||
syclAot = useSycl && syclDeviceArch != null && syclDeviceArch != "";
|
||||
|
||||
# source ocloc binary from intel-compute-runtime
|
||||
oclocAot = runCommand "ocloc-aot" { nativeBuildInputs = [ makeBinaryWrapper ]; } ''
|
||||
mkdir -p "$out/bin"
|
||||
shopt -s nullglob
|
||||
exact=(${lib.getBin intel-compute-runtime}/bin/ocloc)
|
||||
vers=(${lib.getBin intel-compute-runtime}/bin/ocloc-*)
|
||||
if [ ''${#exact[@]} -eq 1 ]; then
|
||||
src=''${exact[0]}
|
||||
elif [ ''${#vers[@]} -eq 1 ]; then
|
||||
src=''${vers[0]}
|
||||
else
|
||||
echo "expected one ocloc binary from intel-compute-runtime" >&2
|
||||
exit 1
|
||||
fi
|
||||
makeWrapper "$src" "$out/bin/ocloc" \
|
||||
--prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath [ intel-graphics-compiler ]}
|
||||
'';
|
||||
in
|
||||
|
||||
effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
@@ -133,11 +178,13 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
src = lib.cleanSource ../../.;
|
||||
};
|
||||
|
||||
# Builds the webui locally, taking care not to require updating any sha256 hash.
|
||||
# Builds Llama Chat locally, taking care not to require updating any sha256 hash.
|
||||
webui = stdenvNoCC.mkDerivation {
|
||||
pname = "webui";
|
||||
version = llamaVersion;
|
||||
src = lib.cleanSource ../../tools/ui;
|
||||
# the app compiles Llama UI from its source in ../../lib/ui, so the source is ui/ as a whole
|
||||
src = lib.cleanSource ../../ui;
|
||||
sourceRoot = "source/apps/chat";
|
||||
|
||||
nativeBuildInputs = [
|
||||
nodejs
|
||||
@@ -146,7 +193,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
|
||||
# no sha256 required when using buildNodeModules
|
||||
npmDeps = importNpmLock.buildNodeModules {
|
||||
npmRoot = ../../tools/ui;
|
||||
npmRoot = ../../ui/apps/chat;
|
||||
inherit nodejs;
|
||||
};
|
||||
|
||||
@@ -156,8 +203,8 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
};
|
||||
|
||||
postPatch = lib.optionalString useWebUi ''
|
||||
cp -r ${finalAttrs.webui} tools/ui/dist
|
||||
chmod -R u+w tools/ui/dist
|
||||
cp -r ${finalAttrs.webui} ui/apps/chat/dist
|
||||
chmod -R u+w ui/apps/chat/dist
|
||||
'';
|
||||
|
||||
# With PR#6015 https://github.com/ggml-org/llama.cpp/pull/6015,
|
||||
@@ -168,6 +215,13 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
# see https://github.com/ggml-org/llama.cpp/pull/6118 for discussion
|
||||
__noChroot = effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders;
|
||||
|
||||
# Do not fortify under SYCL as it breaks compiled memcpy invocations
|
||||
# _FORTIFY_SOURCE rewrites memcpy -> __memcpy_chk
|
||||
hardeningDisable = optionals useSycl [
|
||||
"fortify"
|
||||
"fortify3"
|
||||
];
|
||||
|
||||
nativeBuildInputs =
|
||||
[
|
||||
cmake
|
||||
@@ -181,7 +235,12 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
autoAddDriverRunpath
|
||||
]
|
||||
++ optionals (effectiveStdenv.hostPlatform.isGnu && enableStatic) [ glibc.static ]
|
||||
++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ];
|
||||
++ optionals (effectiveStdenv.hostPlatform.isDarwin && useMetalKit && precompileMetalShaders) [ xcrunHost ]
|
||||
++ optionals useSycl [
|
||||
autoAddDriverRunpath
|
||||
makeBinaryWrapper
|
||||
]
|
||||
++ optionals syclAot [ oclocAot ];
|
||||
|
||||
buildInputs =
|
||||
optionals effectiveStdenv.hostPlatform.isDarwin darwinBuildInputs
|
||||
@@ -190,39 +249,53 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
++ optionals useRocm rocmBuildInputs
|
||||
++ optionals useBlas [ blas ]
|
||||
++ optionals useVulkan vulkanBuildInputs
|
||||
++ optionals useSycl [
|
||||
level-zero
|
||||
ocl-icd
|
||||
intel-graphics-compiler
|
||||
]
|
||||
++ [ openssl ];
|
||||
|
||||
cmakeFlags =
|
||||
[
|
||||
(cmakeBool "LLAMA_BUILD_SERVER" true)
|
||||
(cmakeBool "LLAMA_BUILD_WEBUI" useWebUi)
|
||||
(cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
|
||||
(cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
|
||||
(cmakeBool "GGML_NATIVE" false)
|
||||
(cmakeBool "GGML_BLAS" useBlas)
|
||||
(cmakeBool "GGML_CUDA" useCuda)
|
||||
(cmakeBool "GGML_HIP" useRocm)
|
||||
(cmakeBool "GGML_METAL" useMetalKit)
|
||||
(cmakeBool "GGML_VULKAN" useVulkan)
|
||||
(cmakeBool "GGML_STATIC" enableStatic)
|
||||
(cmakeBool "GGML_RPC" useRpc)
|
||||
]
|
||||
++ optionals useCuda [
|
||||
(
|
||||
with cudaPackages.flags;
|
||||
cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
|
||||
builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
|
||||
)
|
||||
cmakeFlags = [
|
||||
(cmakeBool "LLAMA_BUILD_SERVER" true)
|
||||
(cmakeBool "LLAMA_BUILD_WEBUI" useWebUi)
|
||||
(cmakeBool "BUILD_SHARED_LIBS" (!enableStatic))
|
||||
(cmakeBool "CMAKE_SKIP_BUILD_RPATH" true)
|
||||
(cmakeBool "GGML_NATIVE" false)
|
||||
(cmakeBool "GGML_BLAS" useBlas)
|
||||
(cmakeBool "GGML_CUDA" useCuda)
|
||||
(cmakeBool "GGML_HIP" useRocm)
|
||||
(cmakeBool "GGML_METAL" useMetalKit)
|
||||
(cmakeBool "GGML_VULKAN" useVulkan)
|
||||
(cmakeBool "GGML_STATIC" enableStatic)
|
||||
(cmakeBool "GGML_RPC" useRpc)
|
||||
]
|
||||
++ optionals useCuda [
|
||||
(
|
||||
with cudaPackages.flags;
|
||||
cmakeFeature "CMAKE_CUDA_ARCHITECTURES" (
|
||||
builtins.concatStringsSep ";" (map dropDot cudaCapabilities)
|
||||
)
|
||||
]
|
||||
++ optionals useRocm [
|
||||
(cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
|
||||
(cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmGpuTargets)
|
||||
]
|
||||
++ optionals useMetalKit [
|
||||
(lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
|
||||
(cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
|
||||
];
|
||||
)
|
||||
]
|
||||
++ optionals useRocm [
|
||||
(cmakeFeature "CMAKE_HIP_COMPILER" "${rocmPackages.llvm.clang}/bin/clang")
|
||||
(cmakeFeature "CMAKE_HIP_ARCHITECTURES" rocmGpuTargets)
|
||||
]
|
||||
++ optionals useMetalKit [
|
||||
(lib.cmakeFeature "CMAKE_C_FLAGS" "-D__ARM_FEATURE_DOTPROD=1")
|
||||
(cmakeBool "GGML_METAL_EMBED_LIBRARY" (!precompileMetalShaders))
|
||||
]
|
||||
++ optionals useSycl [
|
||||
(cmakeBool "GGML_SYCL" true)
|
||||
(cmakeBool "GGML_SYCL_F16" syclF16)
|
||||
]
|
||||
++ optionals syclAot [
|
||||
(cmakeFeature "GGML_SYCL_DEVICE_ARCH" syclDeviceArch)
|
||||
]
|
||||
++ optionals (syclAot && syclMaxParallelLinkJobs != null) [
|
||||
(cmakeFeature "GGML_SYCL_MAX_PARALLEL_LINK_JOBS" (toString syclMaxParallelLinkJobs))
|
||||
];
|
||||
|
||||
# Environment variables needed for ROCm
|
||||
env = optionalAttrs useRocm {
|
||||
@@ -237,11 +310,35 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
cp $src/include/llama.h $out/include/
|
||||
'';
|
||||
|
||||
# Fix SYCL binary calling missing important libraries
|
||||
# Also use Level Zero + sysman INSTEAD of OpenCL
|
||||
# due to OpenCL reporting free memory as total memory,
|
||||
# breaking `-fit` functionality
|
||||
postFixup = lib.optionalString useSycl ''
|
||||
driverLibs=${
|
||||
lib.makeLibraryPath [
|
||||
intel-graphics-compiler
|
||||
ocl-icd
|
||||
]
|
||||
}
|
||||
for b in "$out"/bin/*; do
|
||||
[ -f "$b" ] && [ -x "$b" ] || continue
|
||||
patchelf --print-interpreter "$b" >/dev/null 2>&1 || continue
|
||||
patchelf --add-needed libOpenCL.so.1 --add-needed libigc.so.2 \
|
||||
--add-rpath "$driverLibs" "$b"
|
||||
wrapProgram "$b" \
|
||||
--set-default ONEAPI_DEVICE_SELECTOR level_zero:gpu \
|
||||
--set-default ZES_ENABLE_SYSMAN 1
|
||||
done
|
||||
'';
|
||||
|
||||
meta = {
|
||||
# Configurations we don't want even the CI to evaluate. Results in the
|
||||
# "unsupported platform" messages. This is mostly a no-op, because
|
||||
# cudaPackages would've refused to evaluate anyway.
|
||||
badPlatforms = optionals useCuda lib.platforms.darwin;
|
||||
badPlatforms =
|
||||
optionals useCuda lib.platforms.darwin
|
||||
++ optionals useSycl (lib.subtractLists [ "x86_64-linux" ] lib.platforms.all);
|
||||
|
||||
# Configurations that are known to result in build failures. Can be
|
||||
# overridden by importing Nixpkgs with `allowBroken = true`.
|
||||
|
||||
@@ -28,12 +28,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
## Build Image
|
||||
@@ -83,7 +86,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
# Build Stage
|
||||
RUN bash -c "source ${OpenVINO_DIR}/setupvars.sh && \
|
||||
|
||||
@@ -17,12 +17,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
### Build image
|
||||
@@ -52,7 +55,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \
|
||||
cmake -S . -B build \
|
||||
|
||||
@@ -9,12 +9,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -31,7 +34,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN cmake -B build -DGGML_NATIVE=OFF -DGGML_VULKAN=ON -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON && \
|
||||
cmake --build build --config Release -j$(nproc)
|
||||
|
||||
@@ -9,12 +9,15 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
# the app compiles Llama UI from its source next to it
|
||||
COPY ui/lib/ui/package.json /app/ui/lib/ui/package.json
|
||||
COPY ui/lib/ui/src/lib/ /app/ui/lib/ui/src/lib/
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -28,7 +31,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_ZENDNN=ON && \
|
||||
cmake --build build -j $(nproc)
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
|
||||
build*/
|
||||
|
||||
tools/ui/node_modules/
|
||||
ui/apps/chat/node_modules/
|
||||
|
||||
models/*
|
||||
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ insert_final_newline = unset
|
||||
[vendor/**.patch]
|
||||
trim_trailing_whitespace = unset
|
||||
|
||||
[tools/ui/**]
|
||||
[ui/**]
|
||||
indent_style = unset
|
||||
indent_size = unset
|
||||
end_of_line = unset
|
||||
|
||||
@@ -14,7 +14,9 @@ runs:
|
||||
run: |
|
||||
BUILD_NUMBER="$(git rev-list --count HEAD)"
|
||||
SHORT_HASH="$(git rev-parse --short=7 HEAD)"
|
||||
if [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
if [[ "$GITHUB_REF_TYPE" == "tag" && "${{ env.BRANCH_NAME }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "name=${{ env.BRANCH_NAME }}" >> $GITHUB_OUTPUT
|
||||
elif [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
|
||||
|
||||
+1
-2
@@ -48,7 +48,6 @@ examples:
|
||||
- all-globs-to-all-files:
|
||||
- '!tools/server/**'
|
||||
- '!tools/mtmd/**'
|
||||
- '!tools/ui/**'
|
||||
testing:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
@@ -72,7 +71,7 @@ android:
|
||||
server/ui:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- tools/ui/**
|
||||
- ui/**
|
||||
server:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
|
||||
@@ -98,7 +98,7 @@ jobs:
|
||||
|
||||
hip:
|
||||
runs-on: ubuntu-22.04
|
||||
container: rocm/dev-ubuntu-22.04:6.1.2
|
||||
container: rocm/dev-ubuntu-22.04:6.3.4
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
|
||||
@@ -16,6 +16,10 @@ on:
|
||||
description: "Skip the s390x build target (useful for fast test runs that do not need the IBM Z runner)"
|
||||
type: boolean
|
||||
default: false
|
||||
skip_ref_check:
|
||||
description: "Skip the requirement to run from a tag (allows branch refs)"
|
||||
type: boolean
|
||||
default: false
|
||||
schedule:
|
||||
# Rebuild daily rather than on every push because it is expensive
|
||||
- cron: '12 4 * * *'
|
||||
@@ -38,6 +42,14 @@ jobs:
|
||||
source_tag: ${{ steps.srctag.outputs.name }}
|
||||
|
||||
steps:
|
||||
- name: Check manual run ref
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.skip_ref_check }}
|
||||
run: |
|
||||
if [[ "$GITHUB_REF_TYPE" != "tag" ]]; then
|
||||
echo "::error::Manual runs of this workflow must be started from a tag"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
@@ -170,7 +182,7 @@ jobs:
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
# - name: Set up QEMU
|
||||
# if: ${{ contains(matrix.config.platforms, 'linux/amd64') }}
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
BODY: ${{ github.event.pull_request.body }}
|
||||
HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'video demo') }}
|
||||
run: |
|
||||
# sections written by demo:pr (see tools/ui/demo/cli.ts): the managed comment
|
||||
# sections written by demo:pr (see ui/apps/chat/demo/cli.ts): the managed comment
|
||||
# markers, or a preview linking to a demo video uploaded for this PR
|
||||
has_demo=false
|
||||
if { grep -qF '<!-- demo:start -->' <<< "$BODY" && grep -qF '<!-- demo:end -->' <<< "$BODY"; } ||
|
||||
|
||||
@@ -130,7 +130,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -196,7 +196,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -274,7 +274,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -381,7 +381,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -479,7 +479,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v5
|
||||
@@ -571,7 +571,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -624,7 +624,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -672,7 +672,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -894,7 +894,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Install Ninja
|
||||
run: |
|
||||
@@ -1460,7 +1460,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -1532,7 +1532,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Free up disk space
|
||||
uses: ggml-org/free-disk-space@v1.3.1
|
||||
@@ -1647,7 +1647,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
|
||||
@@ -7,6 +7,40 @@ on:
|
||||
description: 'Commit SHA1 to build'
|
||||
required: false
|
||||
type: string
|
||||
spec_methods:
|
||||
description: 'Speculative methods to test'
|
||||
required: true
|
||||
default: 'compact'
|
||||
type: choice
|
||||
options:
|
||||
- compact
|
||||
- nightly
|
||||
- all
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/server-speculative.yml',
|
||||
'common/speculative.*',
|
||||
'src/models/dflash*.cpp',
|
||||
'src/models/dspark.cpp',
|
||||
'src/models/eagle3.cpp',
|
||||
'src/models/lilicorr.cpp',
|
||||
'src/models/qwen35.cpp',
|
||||
'tools/server/tests/unit/test_spec_methods.py'
|
||||
]
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/server-speculative.yml',
|
||||
'common/speculative.*',
|
||||
'src/models/dflash*.cpp',
|
||||
'src/models/dspark.cpp',
|
||||
'src/models/eagle3.cpp',
|
||||
'src/models/lilicorr.cpp',
|
||||
'src/models/qwen35.cpp',
|
||||
'tools/server/tests/unit/test_spec_methods.py'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
@@ -56,8 +90,9 @@ jobs:
|
||||
|
||||
- name: Test speculative decoding
|
||||
env:
|
||||
SPEC_METHODS: compact
|
||||
SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || format('{0}/spec-models', runner.temp) }}
|
||||
SPEC_METHODS: ${{ github.event.inputs.spec_methods || 'compact' }}
|
||||
# pull requests from forks do not get repository variables
|
||||
SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || '/mnt/cache/spec-models' }}
|
||||
N_GPU_LAYERS: 99
|
||||
run: |
|
||||
cd tools/server/tests
|
||||
|
||||
@@ -23,20 +23,20 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build application
|
||||
run: npm run build
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Upload built UI
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
retention-days: 1
|
||||
|
||||
@@ -37,26 +37,26 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build application
|
||||
env:
|
||||
LLAMA_BUILD_NUMBER: ${{ steps.version.outputs.ui_version }}
|
||||
run: npm run build
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run PWA unit tests (versioned build output)
|
||||
run: npx vitest --project=unit --run tests/unit/pwa.spec.ts
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Upload built UI
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
retention-days: 1
|
||||
|
||||
@@ -33,15 +33,15 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
run-id: ${{ inputs.run_id }}
|
||||
github-token: ${{ github.token }}
|
||||
|
||||
- name: Create distribution archive
|
||||
run: |
|
||||
tar -czf dist.tar.gz -C tools/ui/dist .
|
||||
tar -czf dist.tar.gz -C ui/apps/chat/dist .
|
||||
sha256sum dist.tar.gz > dist.tar.gz.sha256
|
||||
mv dist.tar.gz dist.tar.gz.sha256 tools/ui/dist/
|
||||
mv dist.tar.gz dist.tar.gz.sha256 ui/apps/chat/dist/
|
||||
|
||||
- name: Install Hugging Face Hub CLI
|
||||
run: pip install -U huggingface_hub
|
||||
@@ -52,12 +52,12 @@ jobs:
|
||||
- name: Sync built files to Hugging Face bucket (version tag)
|
||||
run: |
|
||||
# Upload the built files to the Hugging Face bucket under the release version
|
||||
hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet
|
||||
hf buckets sync ui/apps/chat/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet
|
||||
|
||||
- name: Sync built files to Hugging Face bucket (latest)
|
||||
run: |
|
||||
# Also upload to the 'latest' directory for fallback downloads
|
||||
hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet
|
||||
hf buckets sync ui/apps/chat/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet
|
||||
|
||||
- name: Verify upload
|
||||
run: |
|
||||
|
||||
@@ -17,7 +17,8 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -25,7 +26,8 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/ui/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
@@ -67,33 +69,33 @@ jobs:
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Run type checking
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run linting
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run lint
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Client tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:client
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Unit tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:unit
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
@@ -109,25 +111,25 @@ jobs:
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Build Storybook
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run build-storybook
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run UI tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:ui -- --testTimeout=60000
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run E2E tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:e2e
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
+46
-17
@@ -13,7 +13,8 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -21,7 +22,8 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'ui/lib/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
@@ -66,46 +68,73 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
if: ${{ steps.node.conclusion == 'success' }}
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run type checking
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run linting
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run lint
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npx playwright install --with-deps
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Client tests
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:client
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Unit tests (uses pre-built dist/ from ui-build)
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:unit
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
lib-checks:
|
||||
name: Library checks
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Run type checking
|
||||
run: npm run check
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Build the package
|
||||
run: npm run prepack
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
@@ -124,38 +153,38 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
if: ${{ steps.node.conclusion == 'success' }}
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Download built UI artifacts (reuses ui-build)
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npx playwright install --with-deps
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build Storybook
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run build-storybook
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run UI tests
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:ui -- --testTimeout=60000
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run E2E tests (uses pre-built dist/ from ui-build)
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:e2e
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# date: Wed Nov 4 12:03:31 EET 2026
|
||||
# date: Mon Nov 16 10:05:56 EET 2026
|
||||
# this file is auto-generated by scripts/gen-authors.sh
|
||||
|
||||
Нияз Гарифзянов <112617865+garrnizon@users.noreply.github.com>
|
||||
@@ -43,7 +43,9 @@ Aarni Koskela <akx@iki.fi>
|
||||
Aaron Miller <apage43@ninjawhale.com>
|
||||
Aaron Teo <57927438+taronaeo@users.noreply.github.com>
|
||||
Aaron Teo <aaron.teo1@ibm.com>
|
||||
Aaron Thomas <84493696+arxhr007@users.noreply.github.com>
|
||||
Aaryaman Vasishta <aaryaman.vasishta@amd.com>
|
||||
Aayan Karmali <44980366+StockerMC@users.noreply.github.com>
|
||||
Abheek Gulati <abheekg@hotmail.com>
|
||||
abhijain1204fujitsu <139222713+abhijain1204fujitsu@users.noreply.github.com>
|
||||
Abhijit Ramesh <abhijitramesh2k@gmail.com>
|
||||
@@ -112,10 +114,12 @@ Aleksander Grygier <admin@serveurperso.com>
|
||||
Aleksander Grygier <aleksander.grygier@gmail.com>
|
||||
Aleksander Grygier <thichthat@gmail.com>
|
||||
Aleksei Nikiforov <103434461+AlekseiNikiforovIBM@users.noreply.github.com>
|
||||
Alen Požun <kalyx.solutions@proton.me>
|
||||
Alessandro de Oliveira Faria (A.K.A.CABELO) <cabelo@opensuse.org>
|
||||
Alessandro98-git <61804547+Alessandro98-git@users.noreply.github.com>
|
||||
Alex <18387287+wadealexc@users.noreply.github.com>
|
||||
Alex <59368173+AlexGabbia@users.noreply.github.com>
|
||||
Alex <alexzkhr@gmail.com>
|
||||
Alex <awhill19@icloud.com>
|
||||
Alex Azarov <alex@azarov.by>
|
||||
Alex Azarov <alexander.azarov@mapbox.com>
|
||||
@@ -132,6 +136,7 @@ Alex Trotta <44127594+Ahajha@users.noreply.github.com>
|
||||
Alex Tuddenham <61622354+AlexsCode@users.noreply.github.com>
|
||||
Alex von Gluck IV <kallisti5@unixzen.com>
|
||||
Alex Wu <dindinw@users.noreply.github.com>
|
||||
Alex-JP-93 <Jianpeng.Li@amd.com>
|
||||
alex-spacemit <jinghui.huang@spacemit.com>
|
||||
Alexander Batischev <eual.jp@gmail.com>
|
||||
Alexander Heisler <126129661+heislera763@users.noreply.github.com>
|
||||
@@ -143,6 +148,7 @@ Alexis Williams <typedrat@users.noreply.github.com>
|
||||
alexpinel <93524949+alexpinel@users.noreply.github.com>
|
||||
Alfred <zxu3@clemson.edu>
|
||||
Ali Chraghi <63465728+alichraghi@users.noreply.github.com>
|
||||
Ali F. Al-Ramadan <55710790+aalramadan@users.noreply.github.com>
|
||||
Ali Nehzat <ali.nehzat@thanks.dev>
|
||||
Ali Tariq <ali.tariq@10xengineers.ai>
|
||||
Ali Tariq <alitariq4589@gmail.com>
|
||||
@@ -158,6 +164,7 @@ Aman Karki <amankarki151@gmail.com>
|
||||
Aman Karki <itsamankarki@gmail.com>
|
||||
amd-dwang <dong.wang@amd.com>
|
||||
amd-lalithnc <lalithnc@amd.com>
|
||||
AMIR <127450356+amirthn6533@users.noreply.github.com>
|
||||
Amir <amir_zia@outlook.com>
|
||||
amirai21 <89905406+amirai21@users.noreply.github.com>
|
||||
AmirAli Mirian <37371367+amiralimi@users.noreply.github.com>
|
||||
@@ -213,6 +220,7 @@ Antoine Viallon <antoine@lesviallon.fr>
|
||||
Anton I <inflyer@hotmail.com>
|
||||
Anton Mitkov <anton_b_mitkov@abv.bg>
|
||||
Anton Mitkov <anton.mitkov@codeplay.com>
|
||||
Anton Sokolchenko <wsevendays@gmail.com>
|
||||
Antonis Makropoulos <benuix@gmail.com>
|
||||
anubhavfujitsu <Anubhav.Srivastava@fujitsu.com>
|
||||
Anudit Nagar <nagaranudit@gmail.com>
|
||||
@@ -233,10 +241,12 @@ arlo-phoenix <140345165+arlo-phoenix@users.noreply.github.com>
|
||||
Armen Kaleshian <kriation@users.noreply.github.com>
|
||||
Arsen Arutunan <58118221+limloop@users.noreply.github.com>
|
||||
Arseniy Obolenskiy <good.doog.2012@gmail.com>
|
||||
ArshM17-NV <amaknojia@nvidia.com>
|
||||
Artem <guinmoon@gmail.com>
|
||||
Artem Zinnatullin <ceo@abstractny.gay>
|
||||
Artyom Lebedev <vagran.ast@gmail.com>
|
||||
aryantandon01 <80969509+aryantandon01@users.noreply.github.com>
|
||||
Asahi-Prv <246092086+Asahi-Prv@users.noreply.github.com>
|
||||
Asahi-Prv <asahihori7@gmail.com>
|
||||
asbelin <71226900+asbelin@users.noreply.github.com>
|
||||
Asbjørn Olling <asbjornolling@gmail.com>
|
||||
@@ -258,6 +268,7 @@ AustinMroz <austinmroz@utexas.edu>
|
||||
AUTOMATIC1111 <16777216c@gmail.com>
|
||||
automaticcat <daogiatuank54@gmail.com>
|
||||
awatuna <23447591+awatuna@users.noreply.github.com>
|
||||
Awss <gitj.18@alt.awss1i.com>
|
||||
b4b4o <zwbao@foxmail.com>
|
||||
Bach Le <bach@bullno1.com>
|
||||
BADR <contact@pythops.com>
|
||||
@@ -351,6 +362,7 @@ CarterLi999 <664681047@qq.com>
|
||||
Casey Primozic <casey@cprimozic.net>
|
||||
Casey Primozic <me@ameo.link>
|
||||
CausalLM <148736309+CausalLM@users.noreply.github.com>
|
||||
CC <65151926+realMoana@users.noreply.github.com>
|
||||
ccbinn <ccbinn@163.com>
|
||||
cduk <19917266+cduk@users.noreply.github.com>
|
||||
cebtenzzre <cebtenzzre@gmail.com>
|
||||
@@ -379,6 +391,7 @@ Cheng Shao <terrorjack@type.dance>
|
||||
Chenguang Li <757486878@qq.com>
|
||||
Chenguang Li <87689256+noemotiovon@users.noreply.github.com>
|
||||
chiheb ben cheikh <100644963+ChihebBENCHEIKH1@users.noreply.github.com>
|
||||
chinmaykamjith001 <chinmaykamjith@gmail.com>
|
||||
Chipmunk <101038159+CHIPMUNK-T0T@users.noreply.github.com>
|
||||
chiranko <96988916+chiranko@users.noreply.github.com>
|
||||
Chris Danis <cdanis@gmail.com>
|
||||
@@ -434,6 +447,7 @@ crasm <crasm@git.vczf.us>
|
||||
crat0z <11581854+crat0z@users.noreply.github.com>
|
||||
CRD716 <crd716@gmail.com>
|
||||
CrispStrobe <154636388+CrispStrobe@users.noreply.github.com>
|
||||
Cristian Adam <cristian.adam@gmail.com>
|
||||
Cristiano Pinto <140563307+crowmoed@users.noreply.github.com>
|
||||
crsawyer <7572190+crsawyer@users.noreply.github.com>
|
||||
Csaba Kecskemeti <csaba.kecskemeti@gmail.com>
|
||||
@@ -527,6 +541,7 @@ Djip007 <3705339+Djip007@users.noreply.github.com>
|
||||
Djip007 <djip.perois@free.fr>
|
||||
dm4 <dm4@secondstate.io>
|
||||
dm4 <sunrisedm4@gmail.com>
|
||||
Dmitrii Panchev <dmnobunaga@users.noreply.github.com>
|
||||
Dmitry Atamanov <data-man@users.noreply.github.com>
|
||||
Dmytro Minochkin <dmytro.minochkin@gmail.com>
|
||||
Dmytro Romanov <casteldazur@gmail.com>
|
||||
@@ -567,6 +582,7 @@ Ed Addario <29247825+EAddario@users.noreply.github.com>
|
||||
Ed Lee <edilee@mozilla.com>
|
||||
Ed Lepedus <ed.lepedus@googlemail.com>
|
||||
Eddie-Wang <wangjinheng1120@163.com>
|
||||
eddiechen <Echen1246@gmail.com>
|
||||
Eden <aa9736195201@gmail.com>
|
||||
eduardopessin <100053075+eduardopessin@users.noreply.github.com>
|
||||
Edward Taylor <edeetee@gmail.com>
|
||||
@@ -598,6 +614,7 @@ Erik Winter <93544977+ewintr@users.noreply.github.com>
|
||||
Ervin Áron Tasnádi <etasnadi@protonmail.com>
|
||||
Esko Toivonen <eskot98@gmail.com>
|
||||
Ethan Guo <ethanguo.dev@gmail.com>
|
||||
Ethan Lin <linsen458@gmail.com>
|
||||
Ethan Turner <eturner64@gmail.com>
|
||||
Ettore Di Giacinto <mudler@users.noreply.github.com>
|
||||
EugeoSynthesisThirtyTwo <gabriel.dhimoila@gmail.com>
|
||||
@@ -625,6 +642,7 @@ fanyang <fanyang89@outlook.com>
|
||||
Farbod Bijary <110523279+farbodbj@users.noreply.github.com>
|
||||
Fathi Boudra <fathi.boudra@linaro.org>
|
||||
Fattire <528174+fat-tire@users.noreply.github.com>
|
||||
Feiteng <lifeiteng0422@gmail.com>
|
||||
felix <felix314159@users.noreply.github.com>
|
||||
Felix <stenbackfelix@gmail.com>
|
||||
Felix Ye <felixyjs@gmail.com>
|
||||
@@ -632,6 +650,7 @@ fengerhu1 <2748250768@qq.com>
|
||||
feni6 <fenissova@proton.me>
|
||||
fidoriel <49869342+fidoriel@users.noreply.github.com>
|
||||
fiesh <fiesh@zefix.tv>
|
||||
figkcoci-debug <figkcoci@gmail.com>
|
||||
FINAL-Bench <kkms1116@koreacu.ac.kr>
|
||||
Finn Voorhees <finnvoorhees@gmail.com>
|
||||
Firat <firatkiral@gmail.com>
|
||||
@@ -646,6 +665,7 @@ Foad Abo Dahood <32059146+masterFoad@users.noreply.github.com>
|
||||
Folko-Ven <71110216+Folko-Ven@users.noreply.github.com>
|
||||
forforever73 <63285796+forforever73@users.noreply.github.com>
|
||||
Foul-Tarnished <107711110+Foul-Tarnished@users.noreply.github.com>
|
||||
fox5611 <66219952+fox5611@users.noreply.github.com>
|
||||
Francisco Herrera <ppaanncchhoo507@gmail.com>
|
||||
Francisco Melo <43780565+francis2tm@users.noreply.github.com>
|
||||
Francois Dugast <francois.dugast@intel.com>
|
||||
@@ -709,6 +729,7 @@ GiviMAD <GiviMAD@users.noreply.github.com>
|
||||
gliptic <gliptic@users.noreply.github.com>
|
||||
gn64 <yukikaze.jp@gmail.com>
|
||||
goerch <jhr.walter@t-online.de>
|
||||
Gotya-OFFCIAL <zhengfreeman@gmail.com>
|
||||
Govlzkoy <gotope@users.noreply.github.com>
|
||||
grahameth <96447521+grahameth@users.noreply.github.com>
|
||||
Gregor Jasny <gjasny@googlemail.com>
|
||||
@@ -749,6 +770,7 @@ Haowei Wu <breadcyanide@icloud.com>
|
||||
Haoxiang Fei <tonyfettes@tonyfettes.com>
|
||||
Harald Fernengel <harald.fernengel@here.com>
|
||||
Harapan Rachman <harapanrachman@gmail.com>
|
||||
Harkanwal Bedi <bedi1887@gmail.com>
|
||||
Harkirat Gill <harkirat.gill@amd.com>
|
||||
HarrisonSec <gzxharrison@gmail.com>
|
||||
Hatsune Miku <129688334+at8u@users.noreply.github.com>
|
||||
@@ -768,6 +790,7 @@ Henrik Forstén <henrik.forsten@gmail.com>
|
||||
Henry Linjamäki <henry.linjamaki@gmail.com>
|
||||
Henry Linjamäki <henry.mikael.linjamaki@intel.com>
|
||||
Henry147147 <44851451+Henry147147@users.noreply.github.com>
|
||||
Hera <58063257+hera2019@users.noreply.github.com>
|
||||
Herman Semenoff <GermanAizek@yandex.ru>
|
||||
Herman Semenov <GermanAizek@yandex.ru>
|
||||
Hesen Peng <hesen.peng@gmail.com>
|
||||
@@ -846,6 +869,7 @@ Imad Saddik <79410781+ImadSaddik@users.noreply.github.com>
|
||||
iMil <imil@NetBSD.org>
|
||||
Incarnas <119618389+bit-incarnas@users.noreply.github.com>
|
||||
InflexCZE <martin.pavlicek11@gmail.com>
|
||||
Iñigo Imaz <49730431+inimaz@users.noreply.github.com>
|
||||
Intel AI Get-to Market Customer Success and Solutions <ai.gtm.css@gmail.com>
|
||||
intelmatt <61025942+intelmatt@users.noreply.github.com>
|
||||
iohub <rickyang.pro@gmail.com>
|
||||
@@ -857,7 +881,9 @@ IsaacDynamo <61521674+IsaacDynamo@users.noreply.github.com>
|
||||
Ishaan Gandhi <Ishaangandhi@gmail.com>
|
||||
iSma <ismail.senhaji@gmail.com>
|
||||
Ismail <115064057+AlrIsmail@users.noreply.github.com>
|
||||
IsNot <209144869@qq.com>
|
||||
issixx <46835150+issixx@users.noreply.github.com>
|
||||
Iswarya Alex <47045679+iswaryaalex@users.noreply.github.com>
|
||||
itsnotoger <19309683+itsnotoger@users.noreply.github.com>
|
||||
itterative <190138728+itterative@users.noreply.github.com>
|
||||
Ivan <nekotekina@gmail.com>
|
||||
@@ -888,6 +914,7 @@ James Francis <6763899+JamesFranc@users.noreply.github.com>
|
||||
James O'Leary <65884233+jpohhhh@users.noreply.github.com>
|
||||
James Reynolds <magnusviri@users.noreply.github.com>
|
||||
jameswu2014 <545426914@qq.com>
|
||||
JamieBordoni <jamie.bordoni@protonmail.com>
|
||||
Jan Boon <jan.boon@kaetemi.be>
|
||||
Jan Boon <kaetemi@gmail.com>
|
||||
Jan Ekström <jeebjp@gmail.com>
|
||||
@@ -956,8 +983,10 @@ Jinyang He <hejinyang@loongson.cn>
|
||||
jinzihao <jinzihao1996@gmail.com>
|
||||
Jiří Podivín <66251151+jpodivin@users.noreply.github.com>
|
||||
Jiří Sejkora <Sejseloid@gmail.com>
|
||||
Jiří Svítil <32205472+mistrjirka@users.noreply.github.com>
|
||||
Jiwoong Song <55657581+jwsong98@users.noreply.github.com>
|
||||
JJJYmmm <92386084+JJJYmmm@users.noreply.github.com>
|
||||
jjperry69 <jjperry69@gmail.com>
|
||||
jklincn <985765408@qq.com>
|
||||
jklincn <jklincn@outlook.com>
|
||||
JM Robles <roblesjm@gmail.com>
|
||||
@@ -970,6 +999,7 @@ Joe Rowell <joerowell4@gmail.com>
|
||||
Joe Todd <joe.todd@codeplay.com>
|
||||
joecryptotoo <80373433+joecryptotoo@users.noreply.github.com>
|
||||
Joel <agnel.joel@gmail.com>
|
||||
Joel Fernandes <joelagnelf@nvidia.com>
|
||||
Johan <JohanAR@users.noreply.github.com>
|
||||
Johannes Gäßler <johannesg@5d6.de>
|
||||
Johannes Rudolph <johannes.rudolph@gmail.com>
|
||||
@@ -988,6 +1018,7 @@ Jonas J <111707981+John-194@users.noreply.github.com>
|
||||
Jonas Jankaitis <111707981+John-194@users.noreply.github.com>
|
||||
Jonas Wunderlich <32615971+jonas-w@users.noreply.github.com>
|
||||
Jonathan <47618606+jbuchananr@users.noreply.github.com>
|
||||
Jonathan Buck <jonathan.j.buck@gmail.com>
|
||||
Jonathan Clohessy <jonathan.clohessy@arm.com>
|
||||
Jonathan Graehl <99024+graehl@users.noreply.github.com>
|
||||
Jorge A <161275481+jorgealias@users.noreply.github.com>
|
||||
@@ -1097,6 +1128,7 @@ klosax <131523366+klosax@users.noreply.github.com>
|
||||
KnightYao <KnightYao@users.noreply.github.com>
|
||||
KokerZhou <111279477+KokerZhou@users.noreply.github.com>
|
||||
Kolen Cheung <ickc@users.noreply.github.com>
|
||||
Kong Jiale <2049447146@qq.com>
|
||||
kononnable <kononnable@gmail.com>
|
||||
Konrad Moren <kmoren@nvidia.com>
|
||||
konradmb <konradmb@o2.pl>
|
||||
@@ -1167,6 +1199,7 @@ li-lizhe <82501526+li-lizhe@users.noreply.github.com>
|
||||
liangliangchang <lichang@amd.com>
|
||||
liminfei-amd <91481003+liminfei-amd@users.noreply.github.com>
|
||||
limitedAtonement <limitedAtonement@users.noreply.github.com>
|
||||
Lines <linellazatin@gmail.com>
|
||||
lingyezhixing <144504450+lingyezhixing@users.noreply.github.com>
|
||||
linsen458-spec <linsen458@gmail.com>
|
||||
Linux User <linuxusergd@outlook.com>
|
||||
@@ -1227,6 +1260,7 @@ makomk <makosoft@googlemail.com>
|
||||
manayang <jackmanayang@gmail.com>
|
||||
manikbhandari <mbbhandarimanik2@gmail.com>
|
||||
Manuel <44313466+makuche@users.noreply.github.com>
|
||||
ManuelPDM <153872964+ManuelPDM@users.noreply.github.com>
|
||||
maor-ps <154728172+maor-ps@users.noreply.github.com>
|
||||
Maphist0 <28743569+Maphist0@users.noreply.github.com>
|
||||
Marc Köhlbrugge <subscriptions@marckohlbrugge.com>
|
||||
@@ -1336,6 +1370,7 @@ Michael Hueschen <m@mhueschen.dev>
|
||||
Michael Kesper <mkesper@schokokeks.org>
|
||||
Michael Klimenko <mklimenko29@gmail.com>
|
||||
Michael Lamothe <michael.lamothe@gmail.com>
|
||||
Michael Paulauski <mike10010100@gmail.com>
|
||||
Michael Podvitskiy <podvitskiymichael@gmail.com>
|
||||
Michael Potter <NanoTekGuy@Gmail.com>
|
||||
Michael Taylor <162068037+mctylr-gh@users.noreply.github.com>
|
||||
@@ -1455,6 +1490,7 @@ Nikolay Popov <131475237+npopov-vst@users.noreply.github.com>
|
||||
Nils Gladitz <nilsgladitz@gmail.com>
|
||||
Nindaleth <Nindaleth@users.noreply.github.com>
|
||||
ningshanwutuobang <ningshanwutuobang@gmail.com>
|
||||
Nisbenz <benazzaanis783@gmail.com>
|
||||
Noah <99681487+NoahOksuz@users.noreply.github.com>
|
||||
nold <Nold360@users.noreply.github.com>
|
||||
nopperl <54780682+nopperl@users.noreply.github.com>
|
||||
@@ -1462,6 +1498,7 @@ nullname <chraac@gmail.com>
|
||||
Nuno <rare-magma@posteo.eu>
|
||||
nuri <yoonuri1@gmail.com>
|
||||
nusu-github <29514220+nusu-github@users.noreply.github.com>
|
||||
nv-vankit <162085734+nv-vankit@users.noreply.github.com>
|
||||
nwyin <tommynguyen0512@gmail.com>
|
||||
o7si <32285332+o7si@users.noreply.github.com>
|
||||
Oğuzhan Akkaya <oakkaya@ymail.com>
|
||||
@@ -1485,6 +1522,7 @@ oobabooga <oobabooga4@gmail.com>
|
||||
opparco <parco.opaai@gmail.com>
|
||||
Ori Pekelman <ori@pekelman.com>
|
||||
Oscar Barenys <rtfss1@gmail.com>
|
||||
Oscar M. Herrera <oscar@kss-inc.com>
|
||||
OSecret <135510162+OLSecret@users.noreply.github.com>
|
||||
ostix360 <55257054+ostix360@users.noreply.github.com>
|
||||
Ouadie EL FAROUKI <ouadie.elfarouki@codeplay.com>
|
||||
@@ -1611,6 +1649,7 @@ Raj Hammeer Singh Hada <hammeerraj@gmail.com>
|
||||
Rajendra Matcha <matcraje@qti.qualcomm.com>
|
||||
Ralph Soika <ralph.soika@imixs.com>
|
||||
Raman Shinde <raman.shinde15@gmail.com>
|
||||
Ramzi <146260554+rmz-24@users.noreply.github.com>
|
||||
Rand Xie <randxiexyy29@gmail.com>
|
||||
Randall Fitzgerald <randall@dasaku.net>
|
||||
Random Fly <renfei8@live.cn>
|
||||
@@ -1651,6 +1690,7 @@ Rich Dougherty <rich@rd.nz>
|
||||
Richard <r-burton@hotmail.co.uk>
|
||||
Richard Davison <richard.davison1@gmail.com>
|
||||
Richard Kiss <him@richardkiss.com>
|
||||
Richard Palethorpe <io@richiejp.com>
|
||||
Richard Roberson <richardr1126@gmail.com>
|
||||
RichardScottOZ <rnmscott@netspace.net.au>
|
||||
Rick G <26732651+TheFlipbook@users.noreply.github.com>
|
||||
@@ -1695,6 +1735,7 @@ Roni <sulpher@gmx.net>
|
||||
Ronny Brendel <ronnyb@nvidia.com>
|
||||
Ronny Brendel <ronnybrendel@gmail.com>
|
||||
Ronsor <ronsor@ronsor.pw>
|
||||
Roshan Kumar <52444089+roshankumarg529@users.noreply.github.com>
|
||||
Rotem Dan <rotemdan@gmail.com>
|
||||
Rowan Hart <rowanbhart@gmail.com>
|
||||
rspOverflow <217881046+rspOverflow@users.noreply.github.com>
|
||||
@@ -1727,9 +1768,11 @@ Saba Fallah <10401143+sfallah@users.noreply.github.com>
|
||||
Saba Fallah <sabafallah@gmail.com>
|
||||
Sachin Desai <smdesai@gmail.com>
|
||||
Sachin Sharma <sachin@zettabolt.com>
|
||||
Saejin Kim-Heinert <saejinmh@gmail.com>
|
||||
Safi Ullah <safiullah.3915@gmail.com>
|
||||
safranowith <bsh155762@gmail.com>
|
||||
Said Taghadouini <84044788+staghado@users.noreply.github.com>
|
||||
SaidSultani <62264302+SaidSultani@users.noreply.github.com>
|
||||
Sait Furkan Teke <35101659+stfurkan@users.noreply.github.com>
|
||||
SakuraUmi <yukinon244@gmail.com>
|
||||
Salvador E. Tropea <stropea@inti.gob.ar>
|
||||
@@ -1754,6 +1797,7 @@ Sang-Kil Park <sang.park@42dot.ai>
|
||||
Sanjay Ahari <sanjayahari1704@gmail.com>
|
||||
Sarah Wu <sarahwu185@gmail.com>
|
||||
Sarah Wu <sarwu@nvidia.com>
|
||||
sargentmajord <jasonlather13@gmail.com>
|
||||
Sascha Rogmann <59577610+srogmann@users.noreply.github.com>
|
||||
sasha0552 <admin@sasha0552.org>
|
||||
Satinder Grewal <grewal.satinder@gmail.com>
|
||||
@@ -1786,6 +1830,7 @@ Sergiu <8598216+mzsergiu@users.noreply.github.com>
|
||||
serhii-nakon <57632032+serhii-nakon@users.noreply.github.com>
|
||||
Sertaç Özercan <852750+sozercan@users.noreply.github.com>
|
||||
seryogakovalyov <seryogakovalyov@gmail.com>
|
||||
Seungjun Bae <linkedlatch@gmail.com>
|
||||
Seungmin Kim <8457324+ehfd@users.noreply.github.com>
|
||||
SeungWon Jeong <65549245+redlion0929@users.noreply.github.com>
|
||||
Seyoung Jeong <seyoungjeong@gmail.com>
|
||||
@@ -1871,6 +1916,7 @@ SmartestWashingMachine <ottobizness@gmail.com>
|
||||
smugman-dot <wbsmoke101@gmail.com>
|
||||
SnA1lGo <44647694+skrandy@users.noreply.github.com>
|
||||
snadampal <87143774+snadampal@users.noreply.github.com>
|
||||
Sofian Elmotiem <sofianelmotiem@gmail.com>
|
||||
SoftwareRenderer <138734813+SoftwareRenderer@users.noreply.github.com>
|
||||
Someone <sergei.kozlukov@aalto.fi>
|
||||
Someone Serge <sergei.kozlukov@aalto.fi>
|
||||
@@ -1886,6 +1932,7 @@ sprayandwipe <spray.and.wipe@gmail.com>
|
||||
Sreeram <sreeram.sivasubramony@amd.com>
|
||||
SRHMorris <69468379+SRHMorris@users.noreply.github.com>
|
||||
Srihari-mcw <96763064+Srihari-mcw@users.noreply.github.com>
|
||||
Srijan Keshri <212402043+arcusbuilds@users.noreply.github.com>
|
||||
Srinivas Billa <nivibilla@gmail.com>
|
||||
srkizer <github.soreepeong.prqovzj1d@srkizer.net>
|
||||
ssweens <1149151+ssweens@users.noreply.github.com>
|
||||
@@ -1919,6 +1966,7 @@ Sutou Kouhei <kou@cozmixng.org>
|
||||
Svetlozar Georgiev <55534064+sgeor255@users.noreply.github.com>
|
||||
Swigler <124839156+Swigler@users.noreply.github.com>
|
||||
swittk <switt1995@gmail.com>
|
||||
sxq <18856096915@163.com>
|
||||
SXX <song_xiaoxi@126.com>
|
||||
SXX <sxx1136965276@gmail.com>
|
||||
Tai Duc Nguyen <taiducnguyen.drexel@gmail.com>
|
||||
@@ -2044,6 +2092,7 @@ Ulrich Drepper <drepper@gmail.com>
|
||||
unbounded <haakon@likedan.net>
|
||||
Unbug Lee <tidelgl@gmail.com>
|
||||
unraido <127105806+unraido@users.noreply.github.com>
|
||||
usrlocalben <benjamin@rqdq.com>
|
||||
uvos <carl@uvos.xyz>
|
||||
uvos <devnull@uvos.xyz>
|
||||
uvos <philipp@uvos.xyz>
|
||||
@@ -2084,6 +2133,7 @@ vk <89937361+itsvedantkumar@users.noreply.github.com>
|
||||
Vlad <spitfireage@gmail.com>
|
||||
Vladimir <bogdad@gmail.com>
|
||||
Vladimir Malyutin <first-leon@yandex.ru>
|
||||
Vladimir Sotnikov <bubert9@gmail.com>
|
||||
Vladimir Vuksanovic <109677816+vvuksanovic@users.noreply.github.com>
|
||||
Vladimir Zorin <vladimir@deviant.guru>
|
||||
Vladislav <vladplotnikov34@gmail.com>
|
||||
@@ -2139,6 +2189,7 @@ Wroclaw <wroclaw223@outlook.com>
|
||||
wsbagnsv1 <sclumpfpapa36@gmail.com>
|
||||
Wu Jian Ping <wujjpp@hotmail.com>
|
||||
Wu Jian Ping <wujp@greatld.com>
|
||||
WuXin <apuravmanhas123@gmail.com>
|
||||
wwoodsTM <104587230+wwoodsTM@users.noreply.github.com>
|
||||
Wyatt Caldwell <218154709+Detensable@users.noreply.github.com>
|
||||
wzy <32936898+Freed-Wu@users.noreply.github.com>
|
||||
@@ -2162,6 +2213,7 @@ Xinpeng Dou <15529241576@163.com>
|
||||
Xinpeng Dou <81913537+Dou-Git@users.noreply.github.com>
|
||||
xloem <0xloem@gmail.com>
|
||||
xris99 <79798089+xris99@users.noreply.github.com>
|
||||
Xuan Son Nguyen <son@huggingface.co>
|
||||
Xuan Son Nguyen <thichthat@gmail.com>
|
||||
Xuan-Son Nguyen <son@huggingface.co>
|
||||
Xuan-Son Nguyen <thichthat@gmail.com>
|
||||
@@ -2247,6 +2299,7 @@ zhangrunda <zhangrunda1234@outlook.com>
|
||||
zhangtao2-1 <478679312@qq.com>
|
||||
ZHAOKAI WANG <sanxianwei@163.com>
|
||||
Zhaolun Yin <129580161+ZhaolunYin@users.noreply.github.com>
|
||||
zhaoyshine <30611065+zhaoyshine@users.noreply.github.com>
|
||||
Zheng.Deng <32841220+dengzheng-cloud@users.noreply.github.com>
|
||||
zhentaoyu <zhentao.yu@intel.com>
|
||||
Zhenwei Jin <109658203+kylo5aby@users.noreply.github.com>
|
||||
|
||||
+1
-1
@@ -134,7 +134,7 @@ option(LLAMA_BUILD_TOOLS "llama: build tools"
|
||||
option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_SERVER "llama: build server example" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_APP "llama: build the unified binary" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_UI "llama: build the embedded Web UI for server" OFF)
|
||||
option(LLAMA_BUILD_UI "llama: build Llama Chat, the embedded chat app, for server" OFF)
|
||||
option(LLAMA_USE_PREBUILT_UI "llama: use prebuilt UI from HF Bucket when available" ON)
|
||||
option(LLAMA_CONNECT "llama: include the llama-connect tool (downloads a prebuilt binary)" OFF)
|
||||
|
||||
|
||||
+1
-1
@@ -102,7 +102,7 @@
|
||||
/tools/rpc/ @ggml-org/ggml-rpc
|
||||
/tools/server/* @ggml-org/llama-server # no subdir
|
||||
/tools/server/tests/ @ggml-org/llama-server
|
||||
/tools/ui/ @ggml-org/llama-ui
|
||||
/ui/ @ggml-org/llama-ui
|
||||
/tools/tokenize/ @ggerganov
|
||||
/tools/tts/ @ggerganov
|
||||
/vendor/ @ggerganov
|
||||
|
||||
@@ -16,6 +16,8 @@ The project differentiates between 3 levels of contributors:
|
||||
>
|
||||
> Detailed information regarding permissible and restricted uses of AI can be found in the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
### Requirements
|
||||
|
||||
If AI is used to generate any portion of the code, contributors must adhere to the following requirements:
|
||||
|
||||
1. Explicitly disclose the manner in which AI was employed.
|
||||
@@ -24,6 +26,16 @@ If AI is used to generate any portion of the code, contributors must adhere to t
|
||||
4. Be prepared to explain every line of code you submit when asked about it by a maintainer.
|
||||
5. It is strictly prohibited to use AI to write your posts for you (bug reports, feature requests, pull request descriptions, Github discussions, responding to humans, ...).
|
||||
|
||||
### Checklist
|
||||
|
||||
> [!IMPORTANT]
|
||||
>
|
||||
> **If you use AI, do all of the following:**
|
||||
>
|
||||
> 1. **Make sure [AGENTS.md](AGENTS.md) is loaded into your agent's context.** Skipping this step is one of the most common reasons PRs get rejected.
|
||||
> 2. **Search for related discussions, PRs and issues** to get the broader context before writing any code.
|
||||
> 3. **Run a local review pass before pushing anything**, using the [`code-review`](skills/code-review/SKILL.md) skill, once the change is implemented. Fix what it finds.
|
||||
|
||||
For more info, please refer to the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
# Pull requests (for contributors & collaborators)
|
||||
|
||||
@@ -13,10 +13,16 @@
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
|
||||
|
||||
[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
[v1.0.0](https://huggingface.co/blog/ggml-org/llama-cpp-v1) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
|
||||
</div>
|
||||
|
||||
## Description
|
||||
|
||||
The main goal of `llama.cpp` is to enable LLM (and MLLM) inference with minimal setup and state-of-the-art performance on a wide range of hardware - locally and in the cloud.
|
||||
|
||||
The `llama.cpp` project is built on top of the [ggml](https://github.com/ggml-org/ggml) library.
|
||||
|
||||
## Quick start
|
||||
|
||||
A few options to get `llama.cpp` installed on your machine:
|
||||
@@ -51,50 +57,12 @@ llama serve -hf ggml-org/Qwen3.5-0.8B-GGUF
|
||||
<i>VLM session with <b>llama cli</b></i>
|
||||
</td>
|
||||
<td align="center">
|
||||
<img width="1392" height="958" alt="Built-in web UI against `llama serve` running Qwen 3.6" src="https://github.com/user-attachments/assets/b402f972-2e32-4def-8771-8d849f08cf2e" />
|
||||
<i>Built-in web UI against <b>llama serve</b></i>
|
||||
<img width="1392" height="958" alt="Llama Chat, the built-in chat app, against `llama serve` running Qwen 3.6" src="https://github.com/user-attachments/assets/b402f972-2e32-4def-8771-8d849f08cf2e" />
|
||||
<i>Llama Chat, the built-in chat app, against <b>llama serve</b></i>
|
||||
</td>
|
||||
</tr>
|
||||
<table>
|
||||
|
||||
## Description
|
||||
|
||||
The main goal of `llama.cpp` is to enable LLM (and VLM) inference with minimal setup and state-of-the-art performance on
|
||||
a wide range of hardware - locally and in the cloud.
|
||||
|
||||
- Plain C/C++ implementation without any dependencies
|
||||
- Apple silicon is a first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks
|
||||
- AVX, AVX2, AVX512 and AMX support for x86 architectures
|
||||
- RVV, ZVFH, ZFH, ZICBOP and ZIHINTPAUSE support for RISC-V architectures
|
||||
- 1.5-bit, 2-bit, 3-bit, 4-bit, 5-bit, 6-bit, and 8-bit integer quantization for faster inference and reduced memory use
|
||||
- Custom CUDA kernels for running LLMs on NVIDIA GPUs (support for AMD GPUs via HIP and Moore Threads GPUs via MUSA)
|
||||
- Vulkan and SYCL backend support
|
||||
- CPU+GPU hybrid inference to partially accelerate models larger than the total VRAM capacity
|
||||
|
||||
The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-org/ggml) library.
|
||||
|
||||
## Supported backends
|
||||
|
||||
| Backend | Target devices |
|
||||
| --- | --- |
|
||||
| [BLAS](docs/build.md#blas-build) | All |
|
||||
| [BLIS](docs/backend/BLIS.md) | All |
|
||||
| [CANN](docs/build.md#cann) | Huawei Ascend NPU |
|
||||
| [CUDA](docs/build.md#cuda) | Nvidia GPU |
|
||||
| [HIP](docs/build.md#hip) | AMD GPU |
|
||||
| [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |
|
||||
| [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
|
||||
| [MUSA](docs/build.md#musa) | Moore Threads GPU |
|
||||
| [Metal](docs/build.md#metal-build) | Apple Silicon |
|
||||
| [OpenCL](docs/backend/OPENCL.md) | Adreno GPU |
|
||||
| [OpenVINO [In Progress]](docs/backend/OPENVINO.md) | Intel CPUs, GPUs, and NPUs |
|
||||
| [RPC](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) | All |
|
||||
| [SYCL](docs/backend/SYCL.md) | Intel GPU |
|
||||
| [VirtGPU](docs/backend/VirtGPU.md) | VirtGPU APIR |
|
||||
| [Vulkan](docs/build.md#vulkan) | GPU |
|
||||
| [WebGPU](docs/build.md#webgpu) | All |
|
||||
| [ZenDNN](docs/build.md#zendnn) | AMD CPU |
|
||||
|
||||
## Documentation
|
||||
|
||||
#### Tools
|
||||
@@ -102,6 +70,7 @@ The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-or
|
||||
- [cli](tools/cli/README.md)
|
||||
- [completion](tools/completion/README.md)
|
||||
- [server](tools/server/README.md)
|
||||
- [Text-to-speech (TTS) models](docs/multimodal/tts.md)
|
||||
- [GBNF grammars](grammars/README.md)
|
||||
|
||||
#### Development
|
||||
|
||||
+1
-1
@@ -46,7 +46,7 @@ Only vulnerabilities that fall within these parts of the project are considered
|
||||
- `ggml/**/*`
|
||||
- `gguf-py/**/*`
|
||||
- `tools/server/*`, **excluding** the following topics:
|
||||
- Web UI
|
||||
- Llama Chat, the built-in chat app (`ui/apps/chat`)
|
||||
- Features marked as experimental
|
||||
- Features not recommended for use in untrusted environments (e.g., router, MCP)
|
||||
- Bugs that can lead to Denial-of-Service attack
|
||||
|
||||
@@ -122,6 +122,8 @@ if [ ! -z ${GG_BUILD_VULKAN} ]; then
|
||||
MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION="${MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION}/SPIRV-Headers/SPIRV-HeadersConfig.cmake"
|
||||
if [[ -f "${MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION}" || -h "${MACOS_RUNNER_CUSTOM_SPIRV_HEADERS_LOCATION}" ]]; then
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DSPIRV-Headers_DIR=${MACOS_RUNNER_CUSTOM_VULKAN_CMAKE_LOCATION}/SPIRV-Headers"
|
||||
# SPIRV-Headers walks up three directories and appends /include, yielding /usr/local/lib/include; the extra vulkan/ level likely breaks its prefix calculation.
|
||||
CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_VULKAN_SPIRV_INCLUDE_DIR=/usr/local/include"
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
+14
-6
@@ -2829,7 +2829,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
const std::regex regex{ R"([,/]+)" };
|
||||
std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
|
||||
std::vector<std::string> split_arg{ it, {} };
|
||||
if (split_arg.size() >= llama_max_devices()) {
|
||||
if (split_arg.size() > llama_max_devices()) {
|
||||
throw std::invalid_argument(
|
||||
string_format("got %zu input configs, but system only has %zu devices", split_arg.size(), llama_max_devices())
|
||||
);
|
||||
@@ -2895,7 +2895,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
const std::regex regex{ R"([,/]+)" };
|
||||
std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
|
||||
std::vector<std::string> split_arg{ it, {} };
|
||||
if (split_arg.size() >= llama_max_devices()) {
|
||||
if (split_arg.size() > llama_max_devices()) {
|
||||
throw std::invalid_argument(
|
||||
string_format("got %zu input configs, but system only has %zu devices", split_arg.size(), llama_max_devices())
|
||||
);
|
||||
@@ -3416,7 +3416,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui-mcp-proxy", "--webui-mcp-proxy"},
|
||||
{"--no-ui-mcp-proxy", "--no-webui-mcp-proxy"},
|
||||
"experimental: whether to enable MCP CORS proxy, for the Web UI to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)\n"
|
||||
"experimental: whether to enable MCP CORS proxy, for Llama Chat to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)\n"
|
||||
"note: for security reasons, this will limit --cors-origins to localhost by default",
|
||||
[](common_params & params, bool value) {
|
||||
params.ui_mcp_proxy = value;
|
||||
@@ -3485,7 +3485,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui", "--webui"},
|
||||
{"--no-ui", "--no-webui"},
|
||||
string_format("whether to enable the Web UI (default: %s)", params.ui ? "enabled" : "disabled"),
|
||||
string_format("whether to enable Llama Chat, the built-in chat app (default: %s)", params.ui ? "enabled" : "disabled"),
|
||||
[](common_params & params, bool value) {
|
||||
params.ui = value;
|
||||
}
|
||||
@@ -4502,7 +4502,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--tts-lang"}, "FNAME",
|
||||
"language (ISO 639-1) for audio generation\n"
|
||||
"see tts/README.md for per-model usage notes",
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_lang = value;
|
||||
}
|
||||
@@ -4510,7 +4510,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--tts-voice"}, "VOICE",
|
||||
"voice for audio generation (model-dependent)\n"
|
||||
"see tts/README.md for per-model usage notes",
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_voice = value;
|
||||
}
|
||||
@@ -4522,6 +4522,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.tts_speaker_file = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-speaker-text"}, "TEXT",
|
||||
"transcript of --tts-speaker-file, for models that need it to clone the voice\n"
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_speaker_text = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
|
||||
//
|
||||
// diffusion params
|
||||
|
||||
@@ -562,7 +562,8 @@ common_peg_parser common_chat_peg_builder::python_or_json_value() {
|
||||
common_peg_parser common_chat_peg_builder::python_style_tool_calls(
|
||||
const ordered_json & tools,
|
||||
bool parallel_tool_calls,
|
||||
bool allow_json_literals) {
|
||||
bool allow_json_literals,
|
||||
bool allow_undeclared) {
|
||||
if (!tools.is_array() || tools.empty()) {
|
||||
return eps();
|
||||
}
|
||||
@@ -620,6 +621,18 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls(
|
||||
tool_choices |= rule("tool-" + std::to_string(i), tool_parser);
|
||||
}
|
||||
|
||||
// last, so declared tools win when parsing
|
||||
if (allow_undeclared) {
|
||||
auto ident = rule("tool-undeclared-ident", chars("[a-zA-Z_]", 1, 1) + chars("[a-zA-Z0-9_.]", 0, -1));
|
||||
auto value = tool_arg_value(allow_json_literals ? python_or_json_value() : python_value());
|
||||
auto arg = tool_arg(tool_arg_open(tool_arg_name(ident) + literal("=")) + value + tool_arg_close(eps()));
|
||||
auto args = optional(arg + zero_or_more("," + space() + arg));
|
||||
|
||||
tool_choices |= rule("tool-undeclared", tool(tool_open(tool_name(ident) + literal("(")) +
|
||||
space() + tool_args(args) + space() + tool_close(literal(")"))
|
||||
));
|
||||
}
|
||||
|
||||
if (parallel_tool_calls) {
|
||||
return "[" + space() + tool_choices + zero_or_more("," + space() + tool_choices) + space() + "]";
|
||||
}
|
||||
|
||||
@@ -161,9 +161,11 @@ class common_chat_peg_builder : public common_peg_parser_builder {
|
||||
|
||||
// Helper for Python-style function call format: name(arg1="value1", arg2=123)
|
||||
// Used by LFM2 and similar templates
|
||||
// allow_undeclared: also accept calls to names not in tools (any identifier, any arguments)
|
||||
common_peg_parser python_style_tool_calls(const common_json & tools,
|
||||
bool parallel_tool_calls,
|
||||
bool allow_json_literals);
|
||||
bool allow_json_literals,
|
||||
bool allow_undeclared = false);
|
||||
|
||||
private:
|
||||
// Python values plus JSON true/false/null.
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
#include "jinja/runtime.h"
|
||||
#include "jinja/value.h"
|
||||
#include "json.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <climits>
|
||||
@@ -857,8 +858,35 @@ static value tts_norm_builtin_sub(const func_args & args) {
|
||||
return mk_val<value_string>(string(res, text_val.all_parts_are_input()));
|
||||
}
|
||||
|
||||
void common_chat_tts_norm_add_builtins(context & ctx) {
|
||||
bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key, std::string & value) {
|
||||
const auto it = std::lower_bound(dict.keys.begin(), dict.keys.end(), key);
|
||||
if (it == dict.keys.end() || *it != key) {
|
||||
return false;
|
||||
}
|
||||
const size_t idx = it - dict.keys.begin();
|
||||
if (idx >= dict.values.size()) {
|
||||
throw std::runtime_error("tts_dict: malformed dictionary, a key has no value");
|
||||
}
|
||||
value = dict.values[idx];
|
||||
return true;
|
||||
}
|
||||
|
||||
void common_chat_tts_norm_add_builtins(context & ctx, std::shared_ptr<const common_tts_dicts> dicts) {
|
||||
ctx.set_val("tts_norm_scan", mk_val<value_func>("tts_norm_scan", tts_norm_builtin_scan));
|
||||
ctx.set_val("tts_norm_sub", mk_val<value_func>("tts_norm_sub", tts_norm_builtin_sub));
|
||||
ctx.set_val("tts_dict", mk_val<value_func>("tts_dict", [dicts](const func_args & args) -> value {
|
||||
args.ensure_count(2, 2);
|
||||
args.ensure_vals<value_string, value_string>();
|
||||
const std::string name = args.get_pos(0)->as_string().str();
|
||||
const auto it = dicts ? dicts->find(name) : common_tts_dicts::const_iterator();
|
||||
if (!dicts || it == dicts->end()) {
|
||||
throw std::runtime_error("tts_dict: unknown dictionary '" + name + "'");
|
||||
}
|
||||
std::string doc;
|
||||
if (!common_tts_dict_find(it->second, args.get_pos(1)->as_string().str(), doc)) {
|
||||
return mk_val<value_none>();
|
||||
}
|
||||
return value_from_json(common_json::parse(doc), false);
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
+16
-3
@@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
@@ -11,9 +12,10 @@
|
||||
// it is NOT a regex engine: the semantics are defined by llama.cpp (see docs/tts-norm-patterns.md), so that this
|
||||
// implementation and the python reference (scripts/chat_tts_norm.py) give the same result for every input
|
||||
//
|
||||
// two functions are exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
|
||||
// functions exposed to task templates (e.g. "tts") via common_chat_tts_norm_add_builtins():
|
||||
// tts_norm_scan(text, rules) -> list of segments {rule, text, groups}
|
||||
// tts_norm_sub(text, pattern, repl) -> string
|
||||
// tts_dict(name, key) -> value of key in a dictionary of the model, or none
|
||||
|
||||
namespace jinja {
|
||||
struct context;
|
||||
@@ -57,6 +59,17 @@ std::vector<common_tts_norm_segment> common_tts_norm_scan(const std::string & te
|
||||
// repl references: %0 (whole match), %1..%9 (groups), %% (literal percent)
|
||||
std::string common_tts_norm_sub(const std::string & text, const common_tts_norm_pattern & pattern, const std::string & repl);
|
||||
|
||||
// add tts_norm_scan() and tts_norm_sub() to the context of a task template
|
||||
// a dictionary of the model, e.g. a pronunciation lexicon
|
||||
struct common_tts_dict {
|
||||
std::vector<std::string> keys; // sorted by their bytes
|
||||
std::vector<std::string> values; // the JSON document of each key
|
||||
};
|
||||
|
||||
using common_tts_dicts = std::map<std::string, common_tts_dict>;
|
||||
|
||||
// binary search of key in a dictionary; on success, value is set to the JSON document
|
||||
bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key, std::string & value);
|
||||
|
||||
// add tts_norm_scan(), tts_norm_sub() and tts_dict() to the context of a task template
|
||||
// they are not jinja global builtins on purpose: chat templates are not affected
|
||||
void common_chat_tts_norm_add_builtins(jinja::context & ctx);
|
||||
void common_chat_tts_norm_add_builtins(jinja::context & ctx, std::shared_ptr<const common_tts_dicts> dicts = nullptr);
|
||||
|
||||
+49
-5
@@ -318,6 +318,7 @@ struct common_chat_templates {
|
||||
std::unique_ptr<common_chat_template> template_default; // always set (defaults to chatml)
|
||||
std::unique_ptr<common_chat_template> template_tool_use;
|
||||
std::unique_ptr<common_chat_template> template_tts; // the prompt of a text to speak
|
||||
std::shared_ptr<common_tts_dicts> tts_dicts; // dictionaries of the model for tts_dict(), by name
|
||||
};
|
||||
|
||||
common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {
|
||||
@@ -806,10 +807,10 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
|
||||
}
|
||||
|
||||
// renders a task template (e.g. "tts") with the given variables
|
||||
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) {
|
||||
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars, const std::shared_ptr<common_tts_dicts> & tts_dicts) {
|
||||
jinja::context ctx(tmpl.source());
|
||||
// text normalization helpers: tts_norm_scan() and tts_norm_sub(), see docs/tts-norm-patterns.md
|
||||
common_chat_tts_norm_add_builtins(ctx);
|
||||
// text normalization helpers: tts_norm_scan(), tts_norm_sub() and tts_dict(), see docs/tts-norm-patterns.md
|
||||
common_chat_tts_norm_add_builtins(ctx, tts_dicts);
|
||||
jinja::global_from_json(ctx, vars, false);
|
||||
ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
|
||||
ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
|
||||
@@ -827,6 +828,11 @@ static std::string string_lower(std::string s) {
|
||||
return s;
|
||||
}
|
||||
|
||||
// VibeVoice-ASR: default system prompt of the template written by the converter
|
||||
static bool is_vibevoice_asr_template(const std::string & src) {
|
||||
return src.find("transcribes audio input into text output in JSON format") != std::string::npos;
|
||||
}
|
||||
|
||||
// LFM3-Audio: language code -> name
|
||||
static std::string lfm3_language_name(const std::string & language) {
|
||||
static const std::map<std::string, std::string> names = {
|
||||
@@ -847,7 +853,8 @@ static std::string lfm3_asr_system_prompt(const std::string & language) {
|
||||
return "Perform ASR in " + lfm3_language_name(language) + ".";
|
||||
}
|
||||
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language,
|
||||
double audio_length, const std::string & context) {
|
||||
common_chat_prompt_preset asr_preset;
|
||||
asr_preset.system = "";
|
||||
asr_preset.user = "Transcribe audio to text";
|
||||
@@ -861,6 +868,17 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates
|
||||
asr_preset.system = lfm3_asr_system_prompt(language);
|
||||
asr_preset.user = "";
|
||||
asr_preset.has_language = true;
|
||||
} else if (is_vibevoice_asr_template(src)) {
|
||||
// without the audio length the model does not know where the audio ends and loops
|
||||
const char * keys = "transcribe it with these keys: Start time, End time, Speaker ID, Content";
|
||||
if (context.empty()) {
|
||||
asr_preset.user = string_format("\nThis is a %.2f seconds audio, please %s", audio_length, keys);
|
||||
} else {
|
||||
asr_preset.user = string_format("\nThis is a %.2f seconds audio, with extra info: %s\n\nPlease %s", audio_length, context.c_str(), keys);
|
||||
asr_preset.has_context = true;
|
||||
}
|
||||
asr_preset.media_first = true;
|
||||
asr_preset.needs_audio_length = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -876,7 +894,7 @@ std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templa
|
||||
{"messages", json::array({{{"role", "user"}, {"content", text}}})},
|
||||
{"language", language},
|
||||
{"voice", voice},
|
||||
});
|
||||
}, chat_templates->tts_dicts);
|
||||
}
|
||||
|
||||
std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
|
||||
@@ -1007,6 +1025,25 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
|
||||
LOG_ERR("%s: failed to parse the tts chat template (ignoring it): %s\n", __func__, e.what());
|
||||
}
|
||||
}
|
||||
// dictionaries for tts_dict(), listed by name in tokenizer.tts_dicts
|
||||
tmpls->tts_dicts = std::make_shared<common_tts_dicts>();
|
||||
for (int32_t i = 0; i < llama_model_meta_arr_n(model, "tokenizer.tts_dicts"); i++) {
|
||||
const std::string name = llama_model_meta_arr_str(model, "tokenizer.tts_dicts", i);
|
||||
const std::string key = "tokenizer.tts_dict." + name + ".keys";
|
||||
const std::string val_key = "tokenizer.tts_dict." + name + ".values";
|
||||
const int32_t n = llama_model_meta_arr_n(model, key.c_str());
|
||||
if (n < 0 || llama_model_meta_arr_n(model, val_key.c_str()) != n) {
|
||||
LOG_ERR("%s: the tts dictionary '%s' has no value for each of its keys (ignoring it)\n", __func__, name.c_str());
|
||||
continue;
|
||||
}
|
||||
auto & dict = (*tmpls->tts_dicts)[name];
|
||||
dict.keys.reserve(n);
|
||||
dict.values.reserve(n);
|
||||
for (int32_t j = 0; j < n; j++) {
|
||||
dict.keys.emplace_back(llama_model_meta_arr_str(model, key.c_str(), j));
|
||||
dict.values.emplace_back(llama_model_meta_arr_str(model, val_key.c_str(), j));
|
||||
}
|
||||
}
|
||||
}
|
||||
return tmpls;
|
||||
}
|
||||
@@ -1427,6 +1464,12 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
return common_chat_params_init_qwen3_coder;
|
||||
}
|
||||
|
||||
// Inkling 1 and 2
|
||||
if (is_inkling_template(src)) {
|
||||
LOG_DBG("Using specialized template: Inkling\n");
|
||||
return common_chat_params_init_inkling;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
@@ -1959,5 +2002,6 @@ common_json common_chat_templates_get_caps(const common_chat_templates * chat_te
|
||||
: chat_templates->template_default->caps;
|
||||
common_json result = caps.to_map();
|
||||
result["reasoning_effort_levels"] = caps.reasoning_effort_levels;
|
||||
result["reasoning_effort_rejected"] = caps.reasoning_effort_rejected;
|
||||
return result;
|
||||
}
|
||||
|
||||
+8
-2
@@ -491,10 +491,16 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
|
||||
struct common_chat_prompt_preset {
|
||||
std::string system;
|
||||
std::string user;
|
||||
bool has_language = false; // the preset already names the requested language
|
||||
bool has_language = false; // the preset already names the requested language
|
||||
bool has_context = false; // the user text already holds the context
|
||||
bool media_first = false; // the media goes before the user text
|
||||
bool needs_audio_length = false; // the user text depends on the audio length, call again with it
|
||||
};
|
||||
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "");
|
||||
// audio_length: in seconds, 0 if unknown
|
||||
// context: extra info from the user (e.g. names, hotwords), only used by some presets, see has_context
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "",
|
||||
double audio_length = 0.0, const std::string & context = "");
|
||||
|
||||
// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
|
||||
// language and voice are optional, the template defines their defaults
|
||||
|
||||
@@ -796,6 +796,7 @@ struct common_params {
|
||||
std::string tts_lang = "";
|
||||
std::string tts_voice = "";
|
||||
std::string tts_speaker_file = "";
|
||||
std::string tts_speaker_text = "";
|
||||
|
||||
bool is_gen_docs = false; // whether we are running inside llama-gen-docs
|
||||
};
|
||||
|
||||
+11
-1
@@ -627,9 +627,14 @@ caps caps_get(jinja::program & prog) {
|
||||
// levels sharing a render are aliases and the highest one is kept
|
||||
const auto unknown = render("REASONING_EFFORT_UNKNOWN");
|
||||
std::vector<std::string> renders;
|
||||
std::vector<std::string> rejected;
|
||||
for (const auto & level : candidates) {
|
||||
const auto rendered = render(level);
|
||||
if (!rendered || rendered == unknown) {
|
||||
if (!rendered) {
|
||||
rejected.push_back(level);
|
||||
continue;
|
||||
}
|
||||
if (rendered == unknown) {
|
||||
continue;
|
||||
}
|
||||
const auto it = std::find(renders.begin(), renders.end(), *rendered);
|
||||
@@ -640,6 +645,11 @@ caps caps_get(jinja::program & prog) {
|
||||
result.reasoning_effort_levels.push_back(level);
|
||||
}
|
||||
}
|
||||
|
||||
// a template that renders no level fails the probe for another reason, so its failures say nothing about the levels
|
||||
if (!result.reasoning_effort_levels.empty()) {
|
||||
result.reasoning_effort_rejected = rejected;
|
||||
}
|
||||
}
|
||||
|
||||
JJ_DEBUG("%s\n", result.to_string().c_str());
|
||||
|
||||
@@ -23,6 +23,9 @@ struct caps {
|
||||
// reasoning effort levels the template renders distinctly, from lowest to highest
|
||||
std::vector<std::string> reasoning_effort_levels;
|
||||
|
||||
// reasoning effort levels the template raises on
|
||||
std::vector<std::string> reasoning_effort_rejected;
|
||||
|
||||
// one of the 2 content capabilities must be true
|
||||
bool supports_string_content = true;
|
||||
bool supports_typed_content = false;
|
||||
|
||||
@@ -263,41 +263,6 @@ lexer_result lexer::tokenize(const std::string & source) {
|
||||
|
||||
char ch = src[pos];
|
||||
|
||||
bool is_closing_block = ch == '-' && next_pos_is( {'%', '}'} );
|
||||
|
||||
// Check for unary operators
|
||||
if (!is_closing_block && (ch == '-' || ch == '+')) {
|
||||
start_pos = pos;
|
||||
token::type last_token_type = tokens.empty() ? token::eof : tokens.back().t;
|
||||
if (last_token_type == token::text || last_token_type == token::eof) {
|
||||
throw lexer_exception(std::string("unexpected character: ") + ch, source, pos);
|
||||
}
|
||||
switch (last_token_type) {
|
||||
case token::identifier:
|
||||
case token::numeric_literal:
|
||||
case token::string_literal:
|
||||
case token::close_paren:
|
||||
case token::close_square_bracket:
|
||||
// Part of a binary operator
|
||||
// a - 1, 1 - 1, true - 1, "apple" - 1, (1) - 1, a[1] - 1
|
||||
// Continue parsing normally
|
||||
break;
|
||||
default: {
|
||||
// Is part of a unary operator
|
||||
// (-1), [-1], (1 + -1), not -1, -apple
|
||||
++pos; // Consume the operator
|
||||
|
||||
// Check for numbers following the unary operator
|
||||
std::string num = consume_numeric();
|
||||
std::string value = std::string(1, ch) + num;
|
||||
token::type t = num.empty() ? token::unary_operator : token::numeric_literal;
|
||||
// JJ_DEBUG("consumed unary operator or numeric literal: '%s'", value.c_str());
|
||||
tokens.push_back({t, value, start_pos});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try to match one of the tokens in the mapping table
|
||||
bool matched = false;
|
||||
for (const auto & [seq, typ] : ordered_mapping_table) {
|
||||
|
||||
@@ -39,7 +39,6 @@ struct token {
|
||||
multiplicative_binary_operator, // * / %
|
||||
power_operator, // **
|
||||
comparison_binary_operator, // < > <= >= == !=
|
||||
unary_operator, // ! - +
|
||||
comment, // {# ... #}
|
||||
};
|
||||
type t;
|
||||
@@ -74,7 +73,6 @@ static std::string type_to_string(token::type t) {
|
||||
case token::multiplicative_binary_operator: return "multiplicative_binary_operator";
|
||||
case token::power_operator: return "power_operator";
|
||||
case token::comparison_binary_operator: return "comparison_binary_operator";
|
||||
case token::unary_operator: return "unary_operator";
|
||||
case token::comment: return "comment";
|
||||
default: return "unknown";
|
||||
}
|
||||
@@ -141,6 +139,7 @@ struct lexer {
|
||||
{"~", token::additive_binary_operator},
|
||||
{"**", token::power_operator},
|
||||
{"*", token::multiplicative_binary_operator},
|
||||
{"//", token::multiplicative_binary_operator},
|
||||
{"/", token::multiplicative_binary_operator},
|
||||
{"%", token::multiplicative_binary_operator},
|
||||
// Assignment operator
|
||||
|
||||
@@ -482,7 +482,8 @@ private:
|
||||
}
|
||||
|
||||
statement_ptr parse_unary_expression() {
|
||||
if (is(token::unary_operator)) {
|
||||
// a sign where an operand is expected is unary: -x, 1 + -x, x if c else -1
|
||||
if (is(token::additive_binary_operator) && peek().value != "~") {
|
||||
depth_guard guard(*this);
|
||||
size_t start_pos = current;
|
||||
auto op = next();
|
||||
|
||||
+35
-10
@@ -191,11 +191,30 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
|
||||
if (is_float) {
|
||||
return mk_val<value_float>(res);
|
||||
} else {
|
||||
return mk_val<value_int>(static_cast<int64_t>(res));
|
||||
return mk_val<value_int>(float_to_int(res));
|
||||
}
|
||||
} else if (op.value == "/") {
|
||||
JJ_DEBUG("Division operation: %f / %f", a, b);
|
||||
return mk_val<value_float>(a / b);
|
||||
} else if (op.value == "//") {
|
||||
// python semantics: round toward -inf
|
||||
if (is_val<value_int>(left_val) && is_val<value_int>(right_val)) {
|
||||
const int64_t x = left_val->as_int();
|
||||
const int64_t y = right_val->as_int();
|
||||
if (y == 0) {
|
||||
throw std::runtime_error("integer division by zero");
|
||||
}
|
||||
if (y == -1) {
|
||||
// INT64_MIN / -1 is UB, wrap around instead
|
||||
return mk_val<value_int>((int64_t) (0 - (uint64_t) x));
|
||||
}
|
||||
int64_t q = x / y;
|
||||
if (x % y != 0 && (x < 0) != (y < 0)) {
|
||||
q -= 1;
|
||||
}
|
||||
return mk_val<value_int>(q);
|
||||
}
|
||||
return mk_val<value_float>(std::floor(a / b));
|
||||
} else if (op.value == "%") {
|
||||
double rem = std::fmod(a, b);
|
||||
JJ_DEBUG("Modulo operation: %f %% %f = %f", a, b, rem);
|
||||
@@ -203,7 +222,7 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
|
||||
if (is_float) {
|
||||
return mk_val<value_float>(rem);
|
||||
} else {
|
||||
return mk_val<value_int>(static_cast<int64_t>(rem));
|
||||
return mk_val<value_int>(float_to_int(rem));
|
||||
}
|
||||
} else if (op.value == "<") {
|
||||
JJ_DEBUG("Comparison operation: %f < %f is %d", a, b, a < b);
|
||||
@@ -274,12 +293,17 @@ static value exec_binary_op(context & ctx, const token & op, value left_val, val
|
||||
// String membership
|
||||
if (is_val<value_string>(left_val) && is_val<value_string>(right_val)) {
|
||||
// case: "a" in "abc"
|
||||
bool member = test_is_in();
|
||||
if (op.value == "in") {
|
||||
return mk_val<value_bool>(member);
|
||||
} else if (op.value == "not in") {
|
||||
return mk_val<value_bool>(!member);
|
||||
if (op.value == "in" || op.value == "not in") {
|
||||
bool member = test_is_in();
|
||||
return mk_val<value_bool>(op.value == "in" ? member : !member);
|
||||
}
|
||||
// case: "a" < "b"; UTF-8 byte order is the same as codepoint order, like python
|
||||
const std::string a = left_val->as_string().str();
|
||||
const std::string b = right_val->as_string().str();
|
||||
if (op.value == "<") { return mk_val<value_bool>(a < b); }
|
||||
if (op.value == ">") { return mk_val<value_bool>(a > b); }
|
||||
if (op.value == "<=") { return mk_val<value_bool>(a <= b); }
|
||||
if (op.value == ">=") { return mk_val<value_bool>(a >= b); }
|
||||
}
|
||||
|
||||
// Value key in object
|
||||
@@ -787,13 +811,14 @@ value call_statement::execute_impl(context & ctx) const {
|
||||
}
|
||||
auto * callee_func = cast_val<value_func>(callee_val);
|
||||
|
||||
context caller_ctx(ctx); // new scope for caller execution
|
||||
// caller() can be saved and called after this scope ended, it owns a snapshot of the variables
|
||||
auto caller_ctx = std::make_shared<context>(ctx.snapshot());
|
||||
|
||||
const func_handler func = [this, caller_ctx = std::move(caller_ctx)](const func_args & args) -> value {
|
||||
const func_handler func = [this, caller_ctx](const func_args & args) -> value {
|
||||
if (args.ctx.macro_depth >= JINJA_RECURSIVE_LIMIT) {
|
||||
throw std::runtime_error("macro calls are nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
|
||||
}
|
||||
context block_ctx(caller_ctx); // new scope for block execution
|
||||
context block_ctx(*caller_ctx); // new scope for block execution
|
||||
block_ctx.macro_depth = args.ctx.macro_depth + 1;
|
||||
|
||||
bind_parameters("caller", this->caller_args, args, block_ctx);
|
||||
|
||||
+30
-3
@@ -81,12 +81,38 @@ struct context {
|
||||
current_time(parent.current_time),
|
||||
is_get_stats(parent.is_get_stats),
|
||||
macro_depth(parent.macro_depth),
|
||||
// inherit variables (for example, when entering a new scope)
|
||||
env(mk_val<value_object>(*parent.env)) {}
|
||||
// child scope: own variables only, lookups fall back to the parent (which must outlive this scope)
|
||||
parent(&parent),
|
||||
env(mk_val<value_object>()) {
|
||||
env->has_builtins = false;
|
||||
}
|
||||
|
||||
context(context &&) = default;
|
||||
|
||||
// copy with all visible variables flattened in, for a scope that can outlive its parents (caller() of a call block)
|
||||
context snapshot() const {
|
||||
std::vector<const context *> chain;
|
||||
for (const context * c = this; c; c = c->parent) {
|
||||
chain.push_back(c);
|
||||
}
|
||||
context res(*this);
|
||||
res.parent = nullptr;
|
||||
for (auto it = chain.rbegin(); it != chain.rend(); ++it) {
|
||||
for (const auto & kv : (*it)->env->as_ordered_object()) {
|
||||
res.env->insert(kv.first, kv.second);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
// returns nullptr if not found
|
||||
value find_val(const value & key) {
|
||||
return env->find(key);
|
||||
for (const context * c = this; c; c = c->parent) {
|
||||
if (value v = c->env->find(key)) {
|
||||
return v;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
value get_val(const std::string & name) {
|
||||
@@ -107,6 +133,7 @@ struct context {
|
||||
}
|
||||
|
||||
private:
|
||||
const context * parent = nullptr;
|
||||
value_object env;
|
||||
};
|
||||
|
||||
|
||||
+49
-18
@@ -281,14 +281,12 @@ static value tojson(const func_args & args) {
|
||||
if (is_val<value_int>(val_indent)) {
|
||||
indent = static_cast<int>(val_indent->as_int());
|
||||
}
|
||||
if (val_sort->as_bool()) { // undefined == false
|
||||
throw not_implemented_exception("tojson sort_keys=true not implemented");
|
||||
}
|
||||
const bool sort_keys = val_sort->as_bool(); // undefined == false
|
||||
const bool ensure_ascii = val_ascii->as_bool(); // undefined == false
|
||||
auto separators = (is_val<value_array>(val_separators) ? val_separators : mk_val<value_array>())->as_array();
|
||||
std::string item_sep = separators.size() > 0 ? separators[0]->as_string().str() : (indent < 0 ? ", " : ",");
|
||||
std::string key_sep = separators.size() > 1 ? separators[1]->as_string().str() : ": ";
|
||||
std::string json_str = value_to_json(args.get_pos(0), indent, item_sep, key_sep);
|
||||
std::string json_str = value_to_json(args.get_pos(0), indent, item_sep, key_sep, sort_keys);
|
||||
if (ensure_ascii) {
|
||||
json_str = json_ensure_ascii_preserving_format(json_str);
|
||||
}
|
||||
@@ -713,7 +711,7 @@ const func_builtins & value_float_t::get_builtins() const {
|
||||
}},
|
||||
{"int", [](const func_args & args) -> value {
|
||||
args.ensure_vals<value_float>();
|
||||
int64_t val = static_cast<int64_t>(args.get_pos(0)->as_float());
|
||||
int64_t val = float_to_int(args.get_pos(0)->as_float());
|
||||
return mk_val<value_int>(val);
|
||||
}},
|
||||
{"float", [](const func_args & args) -> value {
|
||||
@@ -1101,8 +1099,28 @@ const func_builtins & value_bool_t::get_builtins() const {
|
||||
return builtins;
|
||||
}
|
||||
|
||||
[[noreturn]] static value array_sum_not_implemented(const func_args &) {
|
||||
throw not_implemented_exception("Array sum builtin not implemented");
|
||||
static value array_sum(const func_args & args) {
|
||||
args.ensure_kwargs({"attribute", "start"});
|
||||
args.ensure_count(1, 3);
|
||||
args.ensure_vals<value_array>();
|
||||
value undef = mk_val<value_undefined>();
|
||||
value attribute = args.get_kwarg_or_pos("attribute", 1);
|
||||
value start = args.get_kwarg_or_pos("start", 2);
|
||||
// same as the + operator: compute in double, int result if all operands are int
|
||||
bool is_float = is_val<value_float>(start);
|
||||
double sum = start->is_undefined() ? 0.0 : start->as_float();
|
||||
for (const auto & item : args.get_pos(0)->as_array()) {
|
||||
value v = attribute->is_undefined() ? item : get_attribute(item, attribute, undef);
|
||||
if (v->is_undefined() && !attribute->is_undefined()) {
|
||||
throw raised_exception("sum: " + item->type() + " has no attribute '" + attribute->as_string().str() + "'");
|
||||
}
|
||||
if (!is_val<value_int>(v) && !is_val<value_float>(v)) {
|
||||
throw raised_exception("sum: unsupported operand type " + v->type());
|
||||
}
|
||||
is_float = is_float || is_val<value_float>(v);
|
||||
sum += v->as_float();
|
||||
}
|
||||
return is_float ? value(mk_val<value_float>(sum)) : value(mk_val<value_int>(float_to_int(sum)));
|
||||
}
|
||||
|
||||
const func_builtins & value_array_t::get_builtins() const {
|
||||
@@ -1171,7 +1189,7 @@ const func_builtins & value_array_t::get_builtins() const {
|
||||
{"select", select_or_reject<false, false>},
|
||||
{"rejectattr", select_or_reject<true, true>},
|
||||
{"reject", select_or_reject<true, false>},
|
||||
{"sum", array_sum_not_implemented},
|
||||
{"sum", array_sum},
|
||||
{"join", [](const func_args & args) -> value {
|
||||
args.ensure_kwargs({"d", "attribute"});
|
||||
args.ensure_count(1, 3);
|
||||
@@ -1656,6 +1674,11 @@ bool value_compare(const value & a, const value & b, value_compare_op op) {
|
||||
return result;
|
||||
}
|
||||
|
||||
template<>
|
||||
value value_from_json(const common_json & json_val, bool mark_input) {
|
||||
return from_json(json_val, mark_input);
|
||||
}
|
||||
|
||||
template<>
|
||||
void global_from_json(context & ctx, const common_json & json_obj, bool mark_input) {
|
||||
// printf("global_from_json: %s\n" , json_obj.dump(2).c_str());
|
||||
@@ -1741,7 +1764,7 @@ static std::string float_json(double val) {
|
||||
|
||||
// recursively convert value to JSON string
|
||||
// TODO: avoid circular references
|
||||
static void value_to_json_internal(std::ostringstream & oss, const value & val, int curr_lvl, int indent, const std::string_view item_sep, const std::string_view key_sep) {
|
||||
static void value_to_json_internal(std::ostringstream & oss, const value & val, int curr_lvl, int indent, const std::string_view item_sep, const std::string_view key_sep, bool sort_keys) {
|
||||
auto indent_str = [indent, curr_lvl]() -> std::string {
|
||||
return (indent > 0) ? std::string(curr_lvl * indent, ' ') : "";
|
||||
};
|
||||
@@ -1788,7 +1811,7 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
|
||||
oss << newline();
|
||||
for (size_t i = 0; i < arr.size(); ++i) {
|
||||
oss << indent_str() << (indent > 0 ? std::string(indent, ' ') : "");
|
||||
value_to_json_internal(oss, arr[i], curr_lvl + 1, indent, item_sep, key_sep);
|
||||
value_to_json_internal(oss, arr[i], curr_lvl + 1, indent, item_sep, key_sep, sort_keys);
|
||||
if (i < arr.size() - 1) {
|
||||
oss << item_sep;
|
||||
}
|
||||
@@ -1799,17 +1822,25 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
|
||||
oss << "]";
|
||||
} else if (is_val<value_object>(val)) {
|
||||
const auto & obj = val->as_ordered_object(); // IMPORTANT: need to keep exact order
|
||||
std::vector<std::pair<std::string, value>> items;
|
||||
items.reserve(obj.size());
|
||||
for (const auto & pair : obj) {
|
||||
items.emplace_back(is_val<value_float>(pair.first) ? float_json(pair.first->as_float()) : pair.first->as_string().str(), pair.second);
|
||||
}
|
||||
if (sort_keys) {
|
||||
// same order as python json.dumps(sort_keys=True): utf-8 byte order is code point order
|
||||
std::stable_sort(items.begin(), items.end(), [](const auto & a, const auto & b) { return a.first < b.first; });
|
||||
}
|
||||
oss << "{";
|
||||
if (!obj.empty()) {
|
||||
if (!items.empty()) {
|
||||
oss << newline();
|
||||
size_t i = 0;
|
||||
for (const auto & pair : obj) {
|
||||
for (const auto & [key, item] : items) {
|
||||
oss << indent_str() << (indent > 0 ? std::string(indent, ' ') : "");
|
||||
const std::string key = is_val<value_float>(pair.first) ? float_json(pair.first->as_float()) : pair.first->as_string().str();
|
||||
value_to_json_internal(oss, mk_val<value_string>(key), curr_lvl + 1, indent, item_sep, key_sep);
|
||||
value_to_json_internal(oss, mk_val<value_string>(key), curr_lvl + 1, indent, item_sep, key_sep, sort_keys);
|
||||
oss << key_sep;
|
||||
value_to_json_internal(oss, pair.second, curr_lvl + 1, indent, item_sep, key_sep);
|
||||
if (i < obj.size() - 1) {
|
||||
value_to_json_internal(oss, item, curr_lvl + 1, indent, item_sep, key_sep, sort_keys);
|
||||
if (i < items.size() - 1) {
|
||||
oss << item_sep;
|
||||
}
|
||||
oss << newline();
|
||||
@@ -1823,9 +1854,9 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
|
||||
}
|
||||
}
|
||||
|
||||
std::string value_to_json(const value & val, int indent, const std::string_view item_sep, const std::string_view key_sep) {
|
||||
std::string value_to_json(const value & val, int indent, const std::string_view item_sep, const std::string_view key_sep, bool sort_keys) {
|
||||
std::ostringstream oss;
|
||||
value_to_json_internal(oss, val, 0, indent, item_sep, key_sep);
|
||||
value_to_json_internal(oss, val, 0, indent, item_sep, key_sep, sort_keys);
|
||||
JJ_DEBUG("value_to_json: result=%s", oss.str().c_str());
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
+21
-3
@@ -105,6 +105,10 @@ struct context; // forward declaration
|
||||
template<typename T_JSON>
|
||||
void global_from_json(context & ctx, const T_JSON & json_obj, bool mark_input);
|
||||
|
||||
// convert a single JSON value, same rules as global_from_json()
|
||||
template<typename T_JSON>
|
||||
value value_from_json(const T_JSON & json_val, bool mark_input);
|
||||
|
||||
//
|
||||
// base value type
|
||||
//
|
||||
@@ -199,7 +203,7 @@ value find_builtin_func(context & ctx, const std::string & name, value & input);
|
||||
value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false);
|
||||
value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args);
|
||||
|
||||
std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": ");
|
||||
std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": ", bool sort_keys = false);
|
||||
|
||||
// Note: only used for debugging purposes
|
||||
std::string value_to_string_repr(const value & val);
|
||||
@@ -233,6 +237,20 @@ struct value_equality {
|
||||
// primitive value types
|
||||
//
|
||||
|
||||
// double to int64, saturating and with nan as 0; a plain cast is undefined outside the int64 range
|
||||
inline int64_t float_to_int(double v) {
|
||||
if (std::isnan(v)) {
|
||||
return 0;
|
||||
}
|
||||
if (v >= 9223372036854775808.0) {
|
||||
return INT64_MAX;
|
||||
}
|
||||
if (v <= -9223372036854775808.0) {
|
||||
return INT64_MIN;
|
||||
}
|
||||
return static_cast<int64_t>(v);
|
||||
}
|
||||
|
||||
struct value_int_t : public value_t {
|
||||
static constexpr auto kind_id = value_kind::INTEGER;
|
||||
int64_t val_int;
|
||||
@@ -240,7 +258,7 @@ struct value_int_t : public value_t {
|
||||
value_int_t(int64_t v) : value_t(kind_id) {
|
||||
val_int = v;
|
||||
val_flt = static_cast<double>(v);
|
||||
if (static_cast<int64_t>(val_flt) != v) {
|
||||
if (float_to_int(val_flt) != v) {
|
||||
val_flt = v < 0 ? -INFINITY : INFINITY;
|
||||
}
|
||||
}
|
||||
@@ -276,7 +294,7 @@ struct value_float_t : public value_t {
|
||||
double val_flt;
|
||||
value_float_t(double v) : value_t(kind_id) {
|
||||
val_flt = v;
|
||||
val_int = std::isfinite(v) ? static_cast<int64_t>(v) : 0;
|
||||
val_int = std::isfinite(v) ? float_to_int(v) : 0;
|
||||
}
|
||||
virtual std::string type() const override { return "Float"; }
|
||||
virtual double as_float() const override { return val_flt; }
|
||||
|
||||
@@ -303,7 +303,8 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4;
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = "<|channel>thought";
|
||||
data.thinking_end_tags = {"<channel|>"};
|
||||
// a tool call may start before the thought channel is closed; <channel|> stays first, the budget sampler forces it
|
||||
data.thinking_end_tags = {"<channel|>", "<|tool_call>"};
|
||||
|
||||
data.preserved_tokens = {
|
||||
"<|channel>",
|
||||
@@ -334,9 +335,10 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &
|
||||
auto start = p.rule("start", p.optional(p.literal("<|turn>model\n")));
|
||||
|
||||
if (extract_reasoning) {
|
||||
p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until("<channel|>")) + p.literal("<channel|>"));
|
||||
// the model may start a tool call before closing the thought channel
|
||||
p.rule("thought", p.literal("<|channel>thought") + p.space() + p.reasoning(p.until_one_of({"<channel|>", "<|tool_call>"})) + (p.literal("<channel|>") | p.peek(p.literal("<|tool_call>"))));
|
||||
} else {
|
||||
p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until("<channel|>") + p.literal("<channel|>")));
|
||||
p.rule("thought", p.content(p.literal("<|channel>thought") + p.space() + p.until_one_of({"<channel|>", "<|tool_call>"}) + (p.literal("<channel|>") | p.peek(p.literal("<|tool_call>")))));
|
||||
}
|
||||
|
||||
auto consume_empty_channels = p.gbnf(p.zero_or_more(p.literal("<|channel>") + p.negate(p.literal("thought"))), "");
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
#include "parsers.h"
|
||||
|
||||
bool is_inkling_template(const std::string & src) {
|
||||
return src.find("<|message_user|>") != std::string::npos &&
|
||||
src.find("<|content_image|>") != std::string::npos;
|
||||
}
|
||||
|
||||
// the template makes one message block per content part, but a media marker comes inside the text of a message
|
||||
// <header><|content_text|>{media}text<|end_message|> --> <header>{media}<|end_message|><header><|content_text|>text<|end_message|>
|
||||
static void inkling_split_media(common_chat_prompt & prompt) {
|
||||
const std::string TEXT = "<|content_text|>";
|
||||
const std::string END = "<|end_message|>";
|
||||
const std::string MEDIA_BEG = "<__media_";
|
||||
const std::string MEDIA_END = "__>";
|
||||
|
||||
std::vector<common_chat_prompt_part> out;
|
||||
for (size_t i = 0; i < prompt.parts.size(); i++) {
|
||||
const auto & part = prompt.parts[i];
|
||||
const bool in_text_block = part.is_input && part.text.find(MEDIA_BEG) != std::string::npos &&
|
||||
!out.empty() && !out.back().is_input && string_ends_with(out.back().text, TEXT) &&
|
||||
i + 1 < prompt.parts.size() && !prompt.parts[i + 1].is_input && string_starts_with(prompt.parts[i + 1].text, END);
|
||||
const size_t header_pos = in_text_block ? out.back().text.rfind("<|message_") : std::string::npos;
|
||||
if (header_pos == std::string::npos) {
|
||||
out.push_back(part);
|
||||
continue;
|
||||
}
|
||||
const std::string header = out.back().text.substr(header_pos, out.back().text.size() - TEXT.size() - header_pos);
|
||||
|
||||
std::string rest = part.text;
|
||||
for (bool first = true; !rest.empty(); first = false) {
|
||||
const size_t beg = rest.find(MEDIA_BEG);
|
||||
const size_t end = beg == std::string::npos ? beg : rest.find(MEDIA_END, beg);
|
||||
const bool is_media = beg == 0 && end != std::string::npos;
|
||||
const size_t len = is_media ? end + MEDIA_END.size() : (end == std::string::npos ? rest.size() : beg);
|
||||
if (!first) {
|
||||
out.push_back({ END + header + (is_media ? "" : TEXT), false });
|
||||
} else if (is_media) {
|
||||
// mtmd opens the media block
|
||||
out.back().text.resize(out.back().text.size() - TEXT.size());
|
||||
}
|
||||
out.push_back({ rest.substr(0, len), true });
|
||||
rest.erase(0, len);
|
||||
}
|
||||
}
|
||||
prompt.parts = std::move(out);
|
||||
}
|
||||
|
||||
// Inkling 1 and 2 - the assistant turn is a list of blocks, all closed by <|end_message|>:
|
||||
// <|message_model|><|content_thinking|>...<|end_message|>
|
||||
// <|message_model|><|content_text|>...<|end_message|>
|
||||
// <|message_model|>NAME<|content_invoke_tool_json|>{"name":"NAME","args":{...}}<|end_message|>
|
||||
common_chat_params common_chat_params_init_inkling(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
inkling_split_media(data.prompt);
|
||||
|
||||
const std::string HEADER = "<|message_model|>";
|
||||
const std::string THINK = "<|content_thinking|>";
|
||||
const std::string TEXT = "<|content_text|>";
|
||||
const std::string TOOL_JSON = "<|content_invoke_tool_json|>";
|
||||
const std::string END = "<|end_message|>";
|
||||
|
||||
data.preserved_tokens = { HEADER, THINK, TEXT, TOOL_JSON, END };
|
||||
|
||||
data.thinking_start_tag = THINK;
|
||||
data.thinking_end_tags = { END };
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, HEADER },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|message_user|>" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|message_system|>" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|message_tool|>" },
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt.clear();
|
||||
if (!msg.reasoning_content.empty()) {
|
||||
data.generation_prompt = HEADER + THINK + msg.reasoning_content;
|
||||
}
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
if (!data.generation_prompt.empty()) {
|
||||
data.generation_prompt += END;
|
||||
}
|
||||
data.generation_prompt += HEADER + TEXT + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto use_tools = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto header = p.literal(HEADER);
|
||||
auto end = p.literal(END);
|
||||
|
||||
auto think = p.literal(THINK);
|
||||
auto thinking = p.rule("thinking", extract_reasoning ? think + p.reasoning(p.until(END)) + end :
|
||||
p.atomic(p.content(think)) + p.content(p.until(END) + end));
|
||||
auto text = p.rule("text", p.literal(TEXT) + p.content(p.until(END)) + end);
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_format = p.rule("response-format", p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
|
||||
return p.zero_or_more(header + thinking) + header + p.literal(TEXT) + response_format + end;
|
||||
}
|
||||
|
||||
auto blocks = p.zero_or_more(header + (thinking | text));
|
||||
|
||||
if (!use_tools) {
|
||||
return blocks;
|
||||
}
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
|
||||
auto open = p.tool_name(p.literal(name)) + p.literal(TOOL_JSON) + p.literal("{\"name\":" + json(name).dump() + ",\"args\":");
|
||||
auto args = p.tool_args(p.json());
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), p.tool(p.tool_open(open) + args + p.tool_close(p.literal("}")) + end));
|
||||
});
|
||||
|
||||
// tool calls are the last blocks of the turn
|
||||
auto calls = inputs.parallel_tool_calls ? tool_choice + p.zero_or_more(header + tool_choice) : tool_choice;
|
||||
auto tool_calls = header + p.trigger_rule("tool-calls", calls);
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return blocks + tool_calls;
|
||||
}
|
||||
|
||||
return blocks + p.optional(tool_calls);
|
||||
});
|
||||
|
||||
if (has_response_format || use_tools) {
|
||||
data.grammar_lazy = !has_response_format && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
if (data.grammar_lazy) {
|
||||
// a header followed by text and not by a content token is a tool call, the grammar starts at the name
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|message_model\\|>([^<])" },
|
||||
};
|
||||
// Inkling 1: the first header is in the generation prompt, and a lazy grammar does not get the prompt.
|
||||
// an output that starts with no content token is a tool call. not for a continuation, it starts with text
|
||||
if (!inputs.has_continuation()) {
|
||||
data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL, "[^<]+" });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -83,7 +83,8 @@ common_chat_params common_chat_params_init_lfm2(const common_chat_template &
|
||||
auto tool_calls = p.rule("tool-calls",
|
||||
p.trigger_rule("tool-call",
|
||||
p.literal(TOOL_CALL_START) +
|
||||
p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls, /* allow_json_literals = */ true) +
|
||||
p.python_style_tool_calls(inputs.tools, inputs.parallel_tool_calls, /* allow_json_literals = */ true,
|
||||
/* allow_undeclared = */ true) +
|
||||
p.literal(TOOL_CALL_END)
|
||||
)
|
||||
);
|
||||
|
||||
@@ -50,7 +50,7 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat
|
||||
|
||||
data.supports_thinking = true;
|
||||
data.thinking_start_tag = "[THINK]";
|
||||
data.thinking_end_tags = {"[/THINK]"};
|
||||
data.thinking_end_tags = {"[/THINK]", "[TOOL_CALLS]"};
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override = */ adjusted_messages);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
@@ -75,7 +75,7 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templat
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto generation_prompt = p.eps();
|
||||
auto reasoning =
|
||||
extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();
|
||||
extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until_one_of({"[/THINK]", "[TOOL_CALLS]"})) + p.choice({p.literal("[/THINK]"), p.peek(p.literal("[TOOL_CALLS]"))})) : p.eps();
|
||||
|
||||
// Response format parser
|
||||
if (has_response_format) {
|
||||
|
||||
@@ -41,6 +41,8 @@ std::string common_chat_template_generation_prompt_impl(
|
||||
|
||||
bool is_lfm2_template(const std::string & src);
|
||||
|
||||
bool is_inkling_template(const std::string & src);
|
||||
|
||||
common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
@@ -55,6 +57,8 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
|
||||
|
||||
common_chat_params common_chat_params_init_harmony_xml(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_inkling(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_k2_horizon(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
@@ -10,6 +10,7 @@ set(LLAMA_CHAT_PARSERS_SOURCES
|
||||
${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/harmony-xml.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/inkling.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/k2-horizon.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp
|
||||
|
||||
@@ -1003,9 +1003,9 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id
|
||||
} else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {
|
||||
return "Ref(" + p.name + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_tag_parser>) {
|
||||
return "Tag(" + p.tag + ", " + dump(p.child) + ")";
|
||||
return "Tag(" + p.tag + ", " + dump_impl(p.child, visited) + ")";
|
||||
} else if constexpr (std::is_same_v<T, common_peg_atomic_parser>) {
|
||||
return "Atomic(" + dump(p.child) + ")";
|
||||
return "Atomic(" + dump_impl(p.child, visited) + ")";
|
||||
} else {
|
||||
return "Unknown";
|
||||
}
|
||||
|
||||
+2
-1
@@ -255,7 +255,8 @@ struct common_sampler * common_sampler_init(
|
||||
break;
|
||||
}
|
||||
default:
|
||||
GGML_ASSERT(false && "unknown trigger type");
|
||||
throw std::runtime_error(
|
||||
"unknown grammar trigger type: " + std::to_string(trigger.type));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -138,6 +138,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Grok1ForCausalLM": "grok",
|
||||
"GrokForCausalLM": "grok",
|
||||
"GroveMoeForCausalLM": "grovemoe",
|
||||
"HiggsAudio3Model": "higgs_audio",
|
||||
"HunYuanDenseV1ForCausalLM": "hunyuan",
|
||||
"HunYuanMoEV1ForCausalLM": "hunyuan",
|
||||
"HunYuanVLForConditionalGeneration": "hunyuan",
|
||||
@@ -145,6 +146,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"HYV3ForCausalLM": "hunyuan",
|
||||
"HYV4ForCausalLM": "hy_v4",
|
||||
"IQuestCoderForCausalLM": "llama",
|
||||
"Inkling2ForCausalLM": "inkling2",
|
||||
"Inkling2ForConditionalGeneration": "inkling2",
|
||||
"InternLM2ForCausalLM": "internlm",
|
||||
"InternLM3ForCausalLM": "internlm",
|
||||
"JAISLMHeadModel": "jais",
|
||||
@@ -218,6 +221,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"MiniMaxM2ForCausalLM": "minimax",
|
||||
"MiniMaxM3SparseForCausalLM": "minimax",
|
||||
"MiniMaxM3SparseForConditionalGeneration": "minimax",
|
||||
"DSparkMiniMaxDraftModel": "minimax",
|
||||
"Ministral3ForCausalLM": "mistral3",
|
||||
"Ministral3Model": "mistral3",
|
||||
"Mistral3ForConditionalGeneration": "mistral3",
|
||||
@@ -279,6 +283,9 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Qwen3NextForCausalLM": "qwen",
|
||||
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
|
||||
"PocketTTSModel": "pockettts",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
|
||||
@@ -321,6 +328,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"UMT5Model": "t5",
|
||||
"UltravoxModel": "ultravox",
|
||||
"UnlimitedOCRForCausalLM": "deepseek",
|
||||
"VibeVoiceAsrForConditionalGeneration": "vibevoice_asr",
|
||||
"VLlama3ForCausalLM": "llama",
|
||||
"VllmDecision2Model": "vllm_decision2",
|
||||
"VoxtralForConditionalGeneration": "llama",
|
||||
@@ -330,6 +338,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"XverseForCausalLM": "xverse",
|
||||
"YoutuForCausalLM": "deepseek",
|
||||
"YoutuVLForConditionalGeneration": "deepseek",
|
||||
"fish_qwen3_omni": "fish_s2",
|
||||
"modeling_grove_moe.GroveMoeForCausalLM": "grovemoe",
|
||||
"modeling_sarvam_moe.SarvamMoEForCausalLM": "bailingmoe",
|
||||
}
|
||||
@@ -353,6 +362,9 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"EmbeddingGemma2Model": "gemma",
|
||||
"Exaone4_5_ForConditionalGeneration": "exaone",
|
||||
"FishOmniModel": "drama3",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Gemma3ForConditionalGeneration": "gemma",
|
||||
"Gemma3nForConditionalGeneration": "gemma",
|
||||
"Gemma4ForConditionalGeneration": "gemma",
|
||||
@@ -367,8 +379,10 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"Granite4VisionForConditionalGeneration": "granite",
|
||||
"GraniteSpeechForConditionalGeneration": "granite",
|
||||
"GraniteSpeechPlusForConditionalGeneration": "granite",
|
||||
"HiggsAudio3Model": "higgs_audio",
|
||||
"HunYuanVLForConditionalGeneration": "hunyuan",
|
||||
"Idefics3ForConditionalGeneration": "smolvlm",
|
||||
"Inkling2ForConditionalGeneration": "inkling2",
|
||||
"InternVisionModel": "internvl",
|
||||
"JanusForConditionalGeneration": "januspro",
|
||||
"KimiK25ForConditionalGeneration": "kimivl",
|
||||
@@ -425,9 +439,11 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"Step3p7ForConditionalGeneration": "step3",
|
||||
"UltravoxModel": "ultravox",
|
||||
"UnlimitedOCRForCausalLM": "deepseek",
|
||||
"VibeVoiceAsrForConditionalGeneration": "vibevoice_asr",
|
||||
"VoxtralForConditionalGeneration": "ultravox",
|
||||
"VoxtralRealtimeForConditionalGeneration": "ultravox",
|
||||
"YoutuVLForConditionalGeneration": "youtuvl",
|
||||
"fish_qwen3_omni": "fish_s2",
|
||||
}
|
||||
|
||||
|
||||
|
||||
+23
-4
@@ -398,6 +398,9 @@ class ModelBase:
|
||||
return data / scale.float()
|
||||
|
||||
def dequant_simple(weight: Tensor, scale: Tensor, block_size: Sequence[int] | None = None) -> Tensor:
|
||||
if scale.dtype == torch.uint8:
|
||||
# E8M0 exponent stored as raw bytes (MXFP8)
|
||||
scale = (scale.float() - 127.0).exp2()
|
||||
scale = scale.float()
|
||||
|
||||
# ModelOpt block scales are pre-shaped for broadcasting, e.g. [out/128, 1, in/128, 1]
|
||||
@@ -538,7 +541,7 @@ class ModelBase:
|
||||
s = self.model_tensors[name]
|
||||
self.model_tensors[weight_name] = lambda w=w, s=s: dequant_bitnet(w(), s())
|
||||
tensors_to_remove.append(name)
|
||||
elif quant_method in ("fp8", "bsgemm-moe"): # bsgemm-moe: drama-3, same block layout
|
||||
elif quant_method in ("fp8", "bsgemm-moe", "mxfp8"): # bsgemm-moe: drama-3, same block layout
|
||||
block_size = quant_config.get("weight_block_size")
|
||||
for name in self.model_tensors.keys():
|
||||
if name.endswith("_scale_inv"):
|
||||
@@ -1357,10 +1360,10 @@ class ModelBase:
|
||||
consumed.append(input_scale_name)
|
||||
|
||||
# Check if this is a per-expert tensor
|
||||
m = re.search(r'\.experts\.(\d+)\.(gate_proj|up_proj|down_proj)\.weight$', name)
|
||||
m = re.search(r'\.experts\.(\d+)\.(gate_proj|up_proj|down_proj|w1|w2|w3)\.weight$', name)
|
||||
if m:
|
||||
expert_id = int(m.group(1))
|
||||
proj_type = m.group(2)
|
||||
proj_type = {"w1": "gate_proj", "w2": "down_proj", "w3": "up_proj"}.get(m.group(2), m.group(2))
|
||||
bid_m = re.search(r'\.layers\.(\d+)\.', name)
|
||||
bid = int(bid_m.group(1)) if bid_m else 0
|
||||
key = (bid, proj_type)
|
||||
@@ -1451,6 +1454,8 @@ class ModelBase:
|
||||
quant_layers = quant_config.get("quantized_layers", quant_layers) or {}
|
||||
|
||||
nvfp4_compressed_tensors = self._is_nvfp4_compressed_tensors(quant_method, quant_format, quant_groups)
|
||||
# e.g. MXFP8 dense weights with compressed-tensors NVFP4 experts
|
||||
nvfp4_compressed_tensors |= quantization_config.get("moe_quant_format") == "nvfp4-pack-quantized"
|
||||
|
||||
self._nvfp4_global_algo = quant_algo
|
||||
|
||||
@@ -1979,6 +1984,14 @@ class TextModel(ModelBase):
|
||||
if (local_rope_theta := self.rope_parameters.get("sliding_attention", {}).get("rope_theta")) is not None:
|
||||
self.gguf_writer.add_rope_freq_base_swa(local_rope_theta)
|
||||
logger.info(f"gguf: rope theta swa = {local_rope_theta}")
|
||||
# Add rope_pattern from no_rope_layers
|
||||
if no_rope_layers := self.hparams.get("no_rope_layers"):
|
||||
if len(no_rope_layers) == self.block_count: # "no_rope_layers": [0, 0, 0, 1]
|
||||
rope_pattern = [bool(x) for x in no_rope_layers]
|
||||
else: # "no_rope_layers": [3]
|
||||
rope_pattern = [i not in no_rope_layers for i in range(self.block_count)]
|
||||
self.gguf_writer.add_rope_pattern(rope_pattern)
|
||||
logger.info(f"gguf: rope_pattern = {sum(rope_pattern)} RoPE layers / {len(rope_pattern)} total")
|
||||
if (f_rms_eps := self.find_hparam(["rms_norm_eps", "norm_eps"], optional=True)) is not None:
|
||||
self.gguf_writer.add_layer_norm_rms_eps(f_rms_eps)
|
||||
logger.info(f"gguf: rms norm epsilon = {f_rms_eps}")
|
||||
@@ -2432,6 +2445,9 @@ class TextModel(ModelBase):
|
||||
if chkhsh == "a9af07a84191f55098b248ae6f3dfe9e32d3190bebe8eafd91c1ddec9bc3449f":
|
||||
# ref: https://huggingface.co/IFM/K2-Horizon-36B
|
||||
res = "k2-horizon"
|
||||
if chkhsh == "5635e1c8d621dff7b4825f41f2f97b6ceb39bcf2d5c541db28577080d6efd5e5":
|
||||
# ref: https://huggingface.co/thinkingmachines/Inkling-2-Small
|
||||
res = "inkling2"
|
||||
|
||||
if res is None:
|
||||
logger.warning("\n")
|
||||
@@ -3461,7 +3477,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
|
||||
# Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older
|
||||
# Kimi-Linear-48B architecture and cannot load K3 (no attention residuals,
|
||||
# latent MoE, situ, ...). Route on the top-level architecture instead.
|
||||
if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration"):
|
||||
if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration", "HiggsAudio3Model"):
|
||||
return arch
|
||||
|
||||
# if "architectures" is found in the sub-config, use that instead
|
||||
@@ -3472,6 +3488,9 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
|
||||
# some models (llava_onevision2) only declare the architecture inside text_config
|
||||
if arch is None and text_config.get("architectures") is not None:
|
||||
arch = text_config["architectures"][0]
|
||||
# some configs only have a model_type (fishaudio/s2-pro)
|
||||
if arch is None:
|
||||
arch = hparams.get("model_type")
|
||||
if arch is None:
|
||||
raise ValueError("Failed to detect model architecture")
|
||||
return arch
|
||||
|
||||
+32
-19
@@ -18,6 +18,30 @@ _TOWER_SHARED = "audio_decoder_mtp_shared.layers."
|
||||
_TOWER_RESIDUAL = "audio_decoder_mtp.layers."
|
||||
|
||||
|
||||
def load_codec(dir_model) -> dict[str, Tensor]:
|
||||
sd = torch.load(dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
|
||||
return sd.get("state_dict", sd)
|
||||
|
||||
|
||||
def fold_conv(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# fold weight norm
|
||||
if prefix + ".conv.weight" in sd:
|
||||
return sd[prefix + ".conv.weight"].float()
|
||||
g = sd[prefix + ".conv.parametrizations.weight.original0"].float()
|
||||
v = sd[prefix + ".conv.parametrizations.weight.original1"].float()
|
||||
return g * v / v.norm(dim=(1, 2), keepdim=True)
|
||||
|
||||
|
||||
def fold_wn(g: Tensor, v: Tensor) -> Tensor:
|
||||
# fold old-style weight norm of a 1x1 conv, returns [out, in]
|
||||
return (g.float() * v.float() / v.float().norm(dim=(1, 2), keepdim=True)).squeeze(-1)
|
||||
|
||||
|
||||
def fold_snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
|
||||
alpha = alpha.float().flatten()
|
||||
return alpha, 1.0 / (alpha + 1e-9)
|
||||
|
||||
|
||||
@ModelBase.register("FishOmniModel")
|
||||
@ModelBase.example("fishaudio/drama-3")
|
||||
class Drama3TextModel(Qwen3_5MoeTextModel):
|
||||
@@ -36,6 +60,7 @@ class Drama3TextModel(Qwen3_5MoeTextModel):
|
||||
class Drama3MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
gen_projector_type = gguf.VisionProjectorType.DRAMA3_GEN
|
||||
|
||||
_LAYER_MAP = {
|
||||
"enorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ENORM, True),
|
||||
@@ -69,7 +94,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
cfg = self.hparams
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.DRAMA3_GEN)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(self.gen_projector_type)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(cfg["dim"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(cfg["intermediate_size"])
|
||||
@@ -153,32 +178,20 @@ class Drama3MmprojModel(MmprojModel):
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# only the DAC decoder, the encoder is for voice cloning (not supported yet)
|
||||
sd = torch.load(self.dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
|
||||
sd = sd.get("state_dict", sd)
|
||||
sd = load_codec(self.dir_model)
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
def get(name: str) -> Tensor:
|
||||
return sd[name].float()
|
||||
|
||||
def conv(prefix: str) -> Tensor:
|
||||
# fold weight norm
|
||||
if prefix + ".conv.weight" in sd:
|
||||
return get(prefix + ".conv.weight")
|
||||
g = get(prefix + ".conv.parametrizations.weight.original0")
|
||||
v = get(prefix + ".conv.parametrizations.weight.original1")
|
||||
return g * v / v.norm(dim=(1, 2), keepdim=True)
|
||||
return fold_conv(sd, prefix)
|
||||
|
||||
def quant_table(prefix: str) -> Tensor:
|
||||
# fold out_proj into the codebook, so decoding is a row lookup
|
||||
g = get(prefix + ".out_proj.weight_g")
|
||||
v = get(prefix + ".out_proj.weight_v")
|
||||
w = (g * v / v.norm(dim=(1, 2), keepdim=True)).squeeze(-1)
|
||||
w = fold_wn(sd[prefix + ".out_proj.weight_g"], sd[prefix + ".out_proj.weight_v"])
|
||||
return get(prefix + ".codebook.weight") @ w.T + get(prefix + ".out_proj.bias")
|
||||
|
||||
def snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
|
||||
alpha = alpha.flatten()
|
||||
return alpha, 1.0 / (alpha + 1e-9)
|
||||
|
||||
q = "quantizer."
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), quant_table(q + "semantic_quantizer.quantizers.0"))
|
||||
n_rest = sum(1 for k in sd if k.startswith(q + "quantizer.quantizers.") and k.endswith(".codebook.weight"))
|
||||
@@ -228,7 +241,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get(d + "0.conv.bias"))
|
||||
for bid in range(4):
|
||||
p = f"{d}{bid + 1}.block."
|
||||
a, b = snake(get(p + "0.alpha"))
|
||||
a, b = fold_snake(get(p + "0.alpha"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), conv(p + "1"))
|
||||
@@ -236,7 +249,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
for xid in range(3):
|
||||
r = f"{p}{xid + 2}.block."
|
||||
for act, tensor in (("0", T.A_GEN_WAV_DAC_RES_ACT1), ("2", T.A_GEN_WAV_DAC_RES_ACT2)):
|
||||
a, b = snake(get(r + act + ".alpha"))
|
||||
a, b = fold_snake(get(r + act + ".alpha"))
|
||||
name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
|
||||
yield (name + ".alpha", a)
|
||||
yield (name + ".beta", b)
|
||||
@@ -244,7 +257,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
|
||||
yield (name + ".weight", conv(r + cv))
|
||||
yield (name + ".bias", get(r + cv + ".conv.bias"))
|
||||
a, b = snake(get(d + "5.alpha"))
|
||||
a, b = fold_snake(get(d + "5.alpha"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6"))
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, gguf
|
||||
from .drama3 import Drama3MmprojModel, fold_conv, fold_snake, fold_wn, load_codec
|
||||
from .qwen import Qwen3Model
|
||||
|
||||
# Fish Audio S2 Pro (dual-AR)
|
||||
# text model: Qwen3 backbone (slow AR), samples the semantic code of each frame
|
||||
# mmproj: fast AR code predictor (other codebooks) and Fish DAC decoder (codec.pth, same as drama3)
|
||||
|
||||
# residual codebook size of codec.pth, the fast AR head is cut to it
|
||||
_N_RESIDUAL_CODES = 1024
|
||||
|
||||
# codec encoder strides (encoder_rates of the codec config), the quantizer then downsamples 2x2
|
||||
_CODEC_ENC_STRIDES = (2, 4, 8, 8)
|
||||
|
||||
# codec encoder transformers, both 1024 wide with 16 heads of 64
|
||||
# generic audio layers a.blk.0..3: last encoder block, a.blk.4..11: quantizer pre_module
|
||||
_N_ENC_TFM_LAYERS = 4
|
||||
_N_PRE_TFM_LAYERS = 8
|
||||
_CODEC_HEAD_DIM = 64
|
||||
|
||||
|
||||
def _load_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
|
||||
cfg = hparams["text_config"]
|
||||
hparams["text_config"] = {
|
||||
"hidden_size": cfg["dim"],
|
||||
"num_hidden_layers": cfg["n_layer"],
|
||||
"num_attention_heads": cfg["n_head"],
|
||||
"num_key_value_heads": cfg["n_local_heads"],
|
||||
"head_dim": cfg["head_dim"],
|
||||
"intermediate_size": cfg["intermediate_size"],
|
||||
"rms_norm_eps": cfg["norm_eps"],
|
||||
"rope_theta": cfg["rope_base"],
|
||||
"max_position_embeddings": cfg["max_seq_len"],
|
||||
"vocab_size": cfg["vocab_size"],
|
||||
"tie_word_embeddings": cfg["tie_word_embeddings"],
|
||||
}
|
||||
return hparams
|
||||
|
||||
|
||||
def _neox_rows(w: Tensor, head_dim: int) -> Tensor:
|
||||
# checkpoint uses interleaved RoPE, reorder the rows of each head so NEOX RoPE gives the same result
|
||||
order = torch.cat([torch.arange(0, head_dim, 2), torch.arange(1, head_dim, 2)])
|
||||
return w.reshape(-1, head_dim, *w.shape[1:])[:, order].reshape(w.shape)
|
||||
|
||||
|
||||
@ModelBase.register("fish_qwen3_omni")
|
||||
@ModelBase.example("fishaudio/s2-pro")
|
||||
class FishS2TextModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# the rest is mapped by tensor_mapping.py
|
||||
name = name.removeprefix("text_model.")
|
||||
T = gguf.MODEL_TENSOR
|
||||
head_dim = self.hparams["head_dim"]
|
||||
if name.endswith(".attention.wqkv.weight"):
|
||||
n_q = self.hparams["num_attention_heads"] * head_dim
|
||||
n_kv = self.hparams["num_key_value_heads"] * head_dim
|
||||
q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
|
||||
yield (self.format_tensor_name(T.ATTN_Q, bid), _neox_rows(q, head_dim))
|
||||
yield (self.format_tensor_name(T.ATTN_K, bid), _neox_rows(k, head_dim))
|
||||
yield (self.format_tensor_name(T.ATTN_V, bid), v)
|
||||
return
|
||||
if name.endswith(".attention.q_norm.weight"):
|
||||
yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), _neox_rows(data_torch, head_dim))
|
||||
return
|
||||
if name.endswith(".attention.k_norm.weight"):
|
||||
yield (self.format_tensor_name(T.ATTN_K_NORM, bid), _neox_rows(data_torch, head_dim))
|
||||
return
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("fish_qwen3_omni")
|
||||
@ModelBase.example("fishaudio/s2-pro")
|
||||
class FishS2MmprojModel(Drama3MmprojModel):
|
||||
has_audio_encoder = True # codec encoder, for voice cloning
|
||||
gen_projector_type = gguf.VisionProjectorType.FISH_S2_GEN
|
||||
|
||||
_FAST_AR_MAP = {
|
||||
"attention_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
|
||||
"attention.wo.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
|
||||
"ffn_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
|
||||
"feed_forward.w1.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
|
||||
"feed_forward.w3.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP,
|
||||
"feed_forward.w2.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
|
||||
}
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
cfg = dict(self.global_config["audio_decoder_config"])
|
||||
cfg["num_hidden_layers"] = cfg["n_layer"]
|
||||
return cfg
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
# codec encoder: raw 44.1 kHz waveform -> backbone input rows, other hparams are hardcoded in tools/mtmd
|
||||
self.gguf_writer.add_clip_has_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.FISH_S2_SPKENC)
|
||||
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_audio_num_mel_bins(1)
|
||||
self.gguf_writer.add_audio_block_count(_N_ENC_TFM_LAYERS + _N_PRE_TFM_LAYERS)
|
||||
self.gguf_writer.add_audio_embedding_length(1024)
|
||||
self.gguf_writer.add_audio_feed_forward_length(3072)
|
||||
self.gguf_writer.add_audio_head_count(1024 // _CODEC_HEAD_DIM)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# the VQ picks the nearest codebook entry, small errors in the encoder change the codes
|
||||
if new_name.startswith("a.codec."):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# codec transformers: F16 gives codes closer to the reference than BF16
|
||||
if new_name.startswith("a.blk.") and n_dims == 2:
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name = item[0]
|
||||
if name.startswith("audio_decoder.") or name == "text_model.model.embeddings.weight":
|
||||
return item
|
||||
return None
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
T = gguf.MODEL_TENSOR
|
||||
cfg = self.hparams
|
||||
|
||||
# already named by generate_extra_tensors()
|
||||
if name.startswith("a."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
if name == "text_model.model.embeddings.weight":
|
||||
# semantic band of the backbone embeddings
|
||||
begin = self.global_config["semantic_start_token_id"]
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch[begin:begin + cfg["vocab_size"]])
|
||||
return
|
||||
|
||||
if name == "audio_decoder.codebook_embeddings.weight":
|
||||
# backbone input of a speech row, one table per codebook
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), data_torch.reshape(cfg["num_codebooks"], -1, data_torch.shape[-1]))
|
||||
return
|
||||
|
||||
if name == "audio_decoder.embeddings.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_TOK_EMBD), data_torch)
|
||||
return
|
||||
|
||||
if name == "audio_decoder.output.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch[:_N_RESIDUAL_CODES])
|
||||
return
|
||||
|
||||
if name == "audio_decoder.norm.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_OUTPUT_NORM), data_torch)
|
||||
return
|
||||
|
||||
assert bid is not None and name.startswith(f"audio_decoder.layers.{bid}.")
|
||||
key = name.removeprefix(f"audio_decoder.layers.{bid}.")
|
||||
if key == "attention.wqkv.weight":
|
||||
head_dim = cfg["head_dim"]
|
||||
n_q = cfg["n_head"] * head_dim
|
||||
n_kv = cfg["n_local_heads"] * head_dim
|
||||
q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, bid), _neox_rows(q, head_dim))
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, bid), _neox_rows(k, head_dim))
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, bid), v)
|
||||
return
|
||||
yield (self.format_tensor_name(self._FAST_AR_MAP[key], bid), data_torch)
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
yield from super().generate_extra_tensors()
|
||||
|
||||
sd = load_codec(self.dir_model)
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
def get(name: str) -> Tensor:
|
||||
return sd[name].float()
|
||||
|
||||
def tn(tensor: gguf.MODEL_TENSOR, bid: int | None = None, xid: int | None = None) -> str:
|
||||
return gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
|
||||
|
||||
def conv_wb(tensor: gguf.MODEL_TENSOR, prefix: str, bid: int | None = None, xid: int | None = None, stride: int = 1):
|
||||
# taps-major [K / stride, OC, stride * IC], see conv() in tools/mtmd/models/fish-s2-spkenc.cpp
|
||||
w = fold_conv(sd, prefix)
|
||||
n_out, n_in, k = w.shape
|
||||
w = w.reshape(n_out, n_in, k // stride, stride).permute(2, 0, 3, 1).reshape(k // stride, n_out, stride * n_in)
|
||||
yield (tn(tensor, bid, xid) + ".weight", w)
|
||||
yield (tn(tensor, bid, xid) + ".bias", get(prefix + ".conv.bias"))
|
||||
|
||||
def snake_ab(tensor: gguf.MODEL_TENSOR, name: str, bid: int | None = None, xid: int | None = None):
|
||||
a, b = fold_snake(get(name))
|
||||
yield (tn(tensor, bid, xid) + ".alpha", a)
|
||||
yield (tn(tensor, bid, xid) + ".beta", b)
|
||||
|
||||
def tfm_layers(prefix: str, n_layer: int, offset: int):
|
||||
for il in range(n_layer):
|
||||
p = f"{prefix}layers.{il}."
|
||||
bid = offset + il
|
||||
wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0)
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_Q, bid), _neox_rows(wq, _CODEC_HEAD_DIM))
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_K, bid), _neox_rows(wk, _CODEC_HEAD_DIM))
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_V, bid), wv)
|
||||
yield (self.format_tensor_name(T.A_ENC_OUTPUT, bid), get(p + "attention.wo.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_INPUT_NORM, bid), get(p + "attention_norm.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_SCALE, bid), get(p + "attention_layer_scale.gamma"))
|
||||
yield (self.format_tensor_name(T.A_ENC_OUTPUT_NORM, bid), get(p + "ffn_norm.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_GATE, bid), get(p + "feed_forward.w1.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_UP, bid), get(p + "feed_forward.w3.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_DOWN, bid), get(p + "feed_forward.w2.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_SCALE_LS, bid), get(p + "ffn_layer_scale.gamma"))
|
||||
|
||||
# encoder: entry conv, 4x (3 residual units, Snake, strided conv), Snake, post conv
|
||||
e = "encoder.block."
|
||||
yield from conv_wb(T.A_CODEC_ENC_ENTRY, e + "0")
|
||||
for bid, stride in enumerate(_CODEC_ENC_STRIDES):
|
||||
p = f"{e}{bid + 1}.block."
|
||||
for xid in range(3):
|
||||
r = f"{p}{xid}.block."
|
||||
yield from snake_ab(T.A_CODEC_ENC_RES_ACT1, r + "0.alpha", bid, xid)
|
||||
yield from conv_wb(T.A_CODEC_ENC_RES_CONV1, r + "1", bid, xid)
|
||||
yield from snake_ab(T.A_CODEC_ENC_RES_ACT2, r + "2.alpha", bid, xid)
|
||||
yield from conv_wb(T.A_CODEC_ENC_RES_CONV2, r + "3", bid, xid)
|
||||
yield from snake_ab(T.A_CODEC_ENC_SNAKE, p + "3.alpha", bid)
|
||||
yield from conv_wb(T.A_CODEC_ENC_CONV, p + "4", bid, stride=stride)
|
||||
# the last block ends with a windowed transformer
|
||||
yield from tfm_layers(f"{e}4.block.5.", _N_ENC_TFM_LAYERS, 0)
|
||||
yield (self.format_tensor_name(T.A_POST_NORM), get(f"{e}4.block.5.norm.weight"))
|
||||
yield from snake_ab(T.A_CODEC_ENC_POST_SNAKE, e + "5.alpha")
|
||||
yield from conv_wb(T.A_CODEC_ENC_POST_CONV, e + "6")
|
||||
|
||||
# quantizer: 2x (strided conv + ConvNeXt), windowed transformer, then semantic + residual VQ
|
||||
q = "quantizer."
|
||||
for bid in range(2):
|
||||
p = f"{q}downsample.{bid}."
|
||||
yield from conv_wb(T.A_CODEC_DOWN_CONV, p + "0", bid, stride=2)
|
||||
# depthwise: one [C] row per tap
|
||||
yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".weight", fold_conv(sd, p + "1.dwconv").squeeze(1).T.contiguous())
|
||||
yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".bias", get(p + "1.dwconv.conv.bias"))
|
||||
for key, tensor in (("norm", T.A_CODEC_DOWN_NORM), ("pwconv1", T.A_CODEC_DOWN_PW1), ("pwconv2", T.A_CODEC_DOWN_PW2)):
|
||||
yield (tn(tensor, bid) + ".weight", get(f"{p}1.{key}.weight"))
|
||||
yield (tn(tensor, bid) + ".bias", get(f"{p}1.{key}.bias"))
|
||||
yield (tn(T.A_CODEC_DOWN_GAMMA, bid), get(p + "1.gamma"))
|
||||
yield from tfm_layers(q + "pre_module.", _N_PRE_TFM_LAYERS, _N_ENC_TFM_LAYERS)
|
||||
yield (self.format_tensor_name(T.A_CODEC_PRE_NORM), get(q + "pre_module.norm.weight"))
|
||||
|
||||
# nearest codebook entry by cosine similarity, see VectorQuantize.decode_latents()
|
||||
def vq(prefix: str) -> tuple[Tensor, Tensor, Tensor]:
|
||||
w = fold_wn(sd[prefix + ".in_proj.weight_g"], sd[prefix + ".in_proj.weight_v"])
|
||||
cb = torch.nn.functional.normalize(get(prefix + ".codebook.weight"), dim=-1)
|
||||
return w, get(prefix + ".in_proj.bias"), cb
|
||||
|
||||
w, b, cb = vq(q + "semantic_quantizer.quantizers.0")
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".weight"), w)
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".bias"), b)
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_CB), cb)
|
||||
n_rest = self.hparams["num_codebooks"] - 1
|
||||
rest = [vq(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)]
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".weight"), torch.stack([r[0] for r in rest]))
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".bias"), torch.stack([r[1] for r in rest]))
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_CB), torch.stack([r[2] for r in rest]))
|
||||
@@ -112,14 +112,6 @@ class GraniteSWAModel(GraniteModel):
|
||||
logger.info("gguf: (granite_swa) sliding_window_pattern (inferred) = %d SWA layers / %d total",
|
||||
sum(is_swa), n_layers)
|
||||
|
||||
# Add rope_pattern from no_rope_layers
|
||||
if no_rope_layers := self.hparams.get("no_rope_layers"):
|
||||
# Convert 1/0 to bool (1 = use RoPE, 0 = NoPE)
|
||||
rope_pattern = [bool(x) for x in no_rope_layers]
|
||||
self.gguf_writer.add_rope_pattern(rope_pattern)
|
||||
logger.info("gguf: (granite_swa) rope_pattern = %d RoPE layers / %d total",
|
||||
sum(rope_pattern), len(rope_pattern))
|
||||
|
||||
|
||||
@ModelBase.register("GraniteMoeSWAForCausalLM")
|
||||
class GraniteMoeSWAModel(GraniteSWAModel):
|
||||
@@ -560,7 +552,9 @@ class GraniteHybridModel(Mamba2Model, GraniteMoeModel):
|
||||
or not self._ssm_layers
|
||||
)
|
||||
self.gguf_writer.add_rope_scaling_finetuned(use_rope)
|
||||
if not use_rope:
|
||||
# Set context length to an arbitrary large value if rope is not used AND no previous value is present.
|
||||
# https://huggingface.co/ibm-granite/granite-4.0-h-micro-GGUF/discussions/1
|
||||
if not use_rope and "max_position_embeddings" not in self.hparams:
|
||||
self.gguf_writer.add_context_length(2**20)
|
||||
|
||||
## Validation ##
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, gguf
|
||||
from .qwen import Qwen3Model
|
||||
from .ultravox import WhisperEncoderModel
|
||||
|
||||
|
||||
# plain ChatML as built by transcribe.py upstream, the Qwen3 template renders past assistant turns differently from the last one
|
||||
HIGGS_AUDIO_CHAT_TEMPLATE = (
|
||||
"{%- for message in messages -%}"
|
||||
"{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}"
|
||||
"{%- endfor -%}"
|
||||
"{%- if add_generation_prompt -%}"
|
||||
"{{- '<|im_start|>assistant\\n' -}}"
|
||||
"{%- if enable_thinking is defined and enable_thinking is false -%}"
|
||||
"{{- '<think>\\n\\n</think>\\n\\n' -}}"
|
||||
"{%- endif -%}"
|
||||
"{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
@ModelBase.register("HiggsAudio3Model")
|
||||
@ModelBase.example("bosonai/higgs-audio-v3-stt")
|
||||
class HiggsAudioModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
def set_vocab(self):
|
||||
super().set_vocab()
|
||||
self.gguf_writer.add_chat_template(HIGGS_AUDIO_CHAT_TEMPLATE)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
# the audio_* tensors are dropped by TextModel, rename the text head before that
|
||||
if name == "audio_decoder_proj.text_lm_head.weight":
|
||||
name = "lm_head.weight"
|
||||
else:
|
||||
name = "model." + name
|
||||
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
|
||||
@ModelBase.register("HiggsAudio3Model")
|
||||
class HiggsAudioWhisperEncoderModel(WhisperEncoderModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
return self.global_config["audio_encoder_config"]
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# depthwise conv kernel, same as the whisper convs
|
||||
if name == "audio_encoder_proj.temporal.weight":
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
assert self.global_config["projector_type"] == "mlp" and self.global_config["projector_temporal_downsample"] == 2
|
||||
assert self.global_config["audio_adapter_type"] == "stack" and not self.global_config.get("duplicate_audio_triplet", False)
|
||||
super().set_gguf_parameters()
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.HIGGS_AUDIO)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not name.startswith(("audio_tower.", "audio_encoder_proj.")):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
@@ -0,0 +1,331 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from functools import partial
|
||||
from typing import Callable, Iterable, TYPE_CHECKING, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import LazyTorchTensor, ModelBase, MmprojModel, TextModel, gguf, logger
|
||||
|
||||
|
||||
def _e4m3_table() -> np.ndarray:
|
||||
codes = np.arange(256, dtype=np.uint8)
|
||||
exponent = ((codes & 0x7F) >> 3).astype(np.int32)
|
||||
mantissa = (codes & 7).astype(np.float32)
|
||||
values = np.where(exponent == 0, np.ldexp(mantissa, -9), np.ldexp(1 + mantissa / 8, exponent - 7))
|
||||
values = np.where(codes & 0x80, -values, values)
|
||||
values[(codes & 0x7F) == 0x7F] = np.nan
|
||||
return values.astype(np.float32)
|
||||
|
||||
|
||||
_E4M3 = _e4m3_table()
|
||||
_E2M1 = np.array([0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6], dtype=np.float32)
|
||||
_EXPERT_CHUNK_ROWS = 128
|
||||
|
||||
|
||||
def _dequant_expert_chunk(weight: Callable[[], Tensor], scale: Callable[[], Tensor], amax: Callable[[], Tensor] | None, denominator: Callable[[], Tensor] | None, name: str, expert: int, rows: slice) -> np.ndarray:
|
||||
# Fresh lazy tensors release their mappings after each chunk instead of caching the layer.
|
||||
packed = LazyTorchTensor.to_eager(weight().view(torch.uint8)[expert, rows]).numpy()
|
||||
scales = LazyTorchTensor.to_eager(scale().view(torch.uint8)[expert, rows]).numpy()
|
||||
if amax is not None:
|
||||
block_scale = _E4M3[scales]
|
||||
if np.isnan(block_scale).any():
|
||||
raise ValueError(f"Inkling2: {name}_scale expert {expert}: E4M3FN NaN")
|
||||
values = np.empty((packed.shape[0], packed.shape[1] * 2), dtype=np.float32)
|
||||
values[:, 0::2] = _E2M1[packed & 15]
|
||||
values[:, 1::2] = _E2M1[packed >> 4]
|
||||
values.reshape(values.shape[0], -1, 16)[:] *= block_scale[:, :, None]
|
||||
assert denominator is not None
|
||||
a = LazyTorchTensor.to_eager(amax()[expert]).item()
|
||||
d = LazyTorchTensor.to_eager(denominator()).item()
|
||||
if not np.isfinite(a) or a < 0:
|
||||
raise ValueError(f"Inkling2: {name}_amax expert {expert}: expected finite nonnegative value")
|
||||
if not np.isfinite(d) or d <= 0:
|
||||
raise ValueError(f"Inkling2: {name}_outer_scale_denominator expert {expert}: expected finite positive value")
|
||||
with np.errstate(over="ignore", invalid="ignore"):
|
||||
values *= np.float32(a) / np.float32(d)
|
||||
else:
|
||||
if np.any(scales == 255):
|
||||
raise ValueError(f"Inkling2: {name}_scale expert {expert}: UE8M0 byte 255 is NaN")
|
||||
values = _E4M3[packed]
|
||||
if np.isnan(values).any():
|
||||
raise ValueError(f"Inkling2: {name} expert {expert}: E4M3FN NaN")
|
||||
block_scale = np.ldexp(np.ones(scales.shape, dtype=np.float32), scales.astype(np.int32) - 127)
|
||||
with np.errstate(over="ignore", invalid="ignore"):
|
||||
values.reshape(values.shape[0], -1, 32)[:] *= block_scale[:, :, None]
|
||||
if not np.isfinite(values).all():
|
||||
raise ValueError(f"Inkling2: {name} expert {expert}: non-finite dequantized weight")
|
||||
return values
|
||||
|
||||
|
||||
def _is_inkling2(dir_model: Path) -> bool:
|
||||
if not (dir_model / "config.json").is_file():
|
||||
return False
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
return json.load(f).get("model_type") in ("inkling2", "inkling2_text")
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_inkling2)
|
||||
def _load_inkling2_hparams(dir_model: Path):
|
||||
# AutoConfig injects MXFP8 defaults and can replace the original layer schedule.
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@ModelBase.register("Inkling2ForConditionalGeneration", "Inkling2ForCausalLM")
|
||||
@ModelBase.example("thinkingmachines/Inkling-2-Small")
|
||||
class Inkling2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.INKLING2
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
self._expert_quant: dict[str, dict] = {}
|
||||
self._packed_experts: dict[str, gguf.LazyChunkedTensor] = {}
|
||||
quant_path = dir_model / "hf_quant_config.json"
|
||||
if quant_path.is_file():
|
||||
with open(quant_path, encoding="utf-8") as f:
|
||||
groups = json.load(f).get("model", {}).get("parameter_groups", [])
|
||||
for group in groups:
|
||||
for name in group.get("tensors", []):
|
||||
item = self.filter_tensors((name, lambda: torch.empty(0)))
|
||||
if item is not None and item[0].endswith((".w13_weight", ".w2_weight")):
|
||||
name = item[0]
|
||||
if name in self._expert_quant:
|
||||
raise ValueError(f"Inkling2: duplicate hf_quant_config.json scheme for {name}")
|
||||
self._expert_quant[name] = group
|
||||
super().__init__(dir_model, *args, **kwargs)
|
||||
logger.warning("Inkling2: converting the text backbone only; vision and MTP tensors are skipped")
|
||||
hp = self.hparams
|
||||
for name, default in (("q_bias", False), ("o_bias", False), ("n_shared_experts", 0), ("use_value_norm_no_gamma", True), ("attn_res_normalize_values", True), ("scoring_func", "sigmoid")):
|
||||
if hp.get(name, default) != default:
|
||||
raise ValueError(f"Inkling2: unsupported {name}={hp[name]!r}")
|
||||
types = hp.get("layer_types")
|
||||
if types is None:
|
||||
local = hp["local_layer_ids"]
|
||||
types = ["hybrid_sliding" if i in local else "hybrid" for i in range(self.block_count)]
|
||||
if len(types) != self.block_count or set(types) - {"hybrid", "hybrid_sliding"}:
|
||||
raise ValueError("Inkling2: invalid layer_types")
|
||||
self.is_swa = [t == "hybrid_sliding" for t in types]
|
||||
|
||||
def dequant_model(self):
|
||||
for name in list(self.model_tensors):
|
||||
if not name.endswith((".w13_weight", ".w2_weight")):
|
||||
continue
|
||||
weight = self.model_tensors[name]
|
||||
tensor = weight()
|
||||
companions = [name + suffix for suffix in ("_scale", "_amax", "_outer_scale_denominator")]
|
||||
if tensor.dtype.is_floating_point and tensor.element_size() > 1:
|
||||
if any(n in self.model_tensors for n in companions):
|
||||
raise ValueError(f"Inkling2: floating-point {name} has packed quantization companions")
|
||||
continue
|
||||
config = self._expert_quant.get(name, {})
|
||||
scheme = config.get("weight_transform")
|
||||
if scheme not in ("nvfp4_nearest", "mxfp8_nearest"):
|
||||
raise ValueError(f"Inkling2: {name}: missing or unsupported hf_quant_config.json scheme {scheme!r}")
|
||||
nvfp4 = scheme == "nvfp4_nearest"
|
||||
if nvfp4 and config.get("weight_scale_format") != "amax_denominator":
|
||||
raise ValueError(f"Inkling2: {name}: expected NVFP4 amax_denominator scale format")
|
||||
required = companions if nvfp4 else companions[:1]
|
||||
for companion in required:
|
||||
if companion not in self.model_tensors:
|
||||
raise ValueError(f"Inkling2: {name}: missing companion {companion}")
|
||||
if not nvfp4 and any(n in self.model_tensors for n in companions[1:]):
|
||||
raise ValueError(f"Inkling2: {name}: MXFP8 must not have amax or outer_scale_denominator")
|
||||
scale = self.model_tensors[companions[0]]
|
||||
scale_tensor = scale()
|
||||
weight_dtype, scale_dtype = (torch.uint8, torch.float8_e4m3fn) if nvfp4 else (torch.float8_e4m3fn, torch.uint8)
|
||||
if tensor.dtype != weight_dtype or scale_tensor.dtype != scale_dtype:
|
||||
raise ValueError(f"Inkling2: {name}: expected {weight_dtype} weights and {scale_dtype} scales, got {tensor.dtype} and {scale_tensor.dtype}")
|
||||
experts = self.hparams["n_routed_experts"]
|
||||
hidden = self.hparams["hidden_size"]
|
||||
inter = self.find_hparam(["moe_intermediate_size", "intermediate_size"])
|
||||
w13 = name.endswith(".w13_weight")
|
||||
rows, cols = (2 * inter, hidden) if w13 else (hidden, inter)
|
||||
shape = (experts, rows, cols)
|
||||
if min(shape) <= 0 or cols % (16 if nvfp4 else 32) or tuple(tensor.shape) != (experts, rows, cols // (2 if nvfp4 else 1)):
|
||||
raise ValueError(f"Inkling2: {name}: invalid packed shape {tuple(tensor.shape)} for logical shape {shape}")
|
||||
ss = tuple(scale_tensor.shape)
|
||||
if len(ss) != 3 or ss[0] != experts or min(ss) <= 0 or rows % ss[1] or cols % ss[2]:
|
||||
raise ValueError(f"Inkling2: {name}_scale: shape {ss} does not divide logical shape {shape}")
|
||||
block = (rows // ss[1], cols // ss[2])
|
||||
if block != (1, 16 if nvfp4 else 32):
|
||||
raise ValueError(f"Inkling2: {name}_scale: unsupported stored block shape {block}")
|
||||
if list(block) != config.get("weight_block_shape"):
|
||||
logger.warning("Inkling2: %s: hf_quant_config.json weight_block_shape %s conflicts with stored weight/scale shapes %s/%s; using %s", name, config.get("weight_block_shape"), tuple(tensor.shape), ss, list(block))
|
||||
amax = denominator = None
|
||||
if nvfp4:
|
||||
amax, denominator = (self.model_tensors[n] for n in companions[1:])
|
||||
for companion, gen, expected in ((companions[1], amax, (experts,)), (companions[2], denominator, (1,))):
|
||||
t = gen()
|
||||
if t.dtype != torch.float32 or tuple(t.shape) != expected:
|
||||
raise ValueError(f"Inkling2: {companion}: expected F32 {expected}, got {t.dtype} {tuple(t.shape)}")
|
||||
chunks: list[Callable[[], np.ndarray]] = []
|
||||
for expert in range(experts):
|
||||
for half in range(2 if w13 else 1):
|
||||
for start in range(0, rows // 2 if w13 else rows, _EXPERT_CHUNK_ROWS):
|
||||
stop = min(start + _EXPERT_CHUNK_ROWS, rows // 2 if w13 else rows)
|
||||
# The same packed row slice selects the weight and its scale groups.
|
||||
row_slice = slice(2 * start + half, 2 * stop + half, 2) if w13 else slice(start, stop)
|
||||
chunks.append(partial(_dequant_expert_chunk, weight, scale, amax, denominator, name, expert, row_slice))
|
||||
self._packed_experts[name] = gguf.LazyChunkedTensor(chunks, shape, np.float32)
|
||||
self.model_tensors[name] = lambda shape=shape: torch.empty(shape, dtype=torch.float32, device="meta")
|
||||
for companion in required:
|
||||
del self.model_tensors[companion]
|
||||
for name, gen in self.model_tensors.items():
|
||||
t = gen()
|
||||
if not t.dtype.is_floating_point or t.element_size() == 1 or name.endswith(("_weight_scale", "_weight_amax", "_weight_outer_scale_denominator")):
|
||||
raise ValueError(f"Inkling2: unexpected packed tensor or orphan companion {name}")
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
if any(p in name for p in ("visual.", "vision_tower.", "mtp.", "markov_head", "nextn_predict")):
|
||||
return None
|
||||
name = name.replace("model.language_model.", "model.").replace("model.llm.", "model.")
|
||||
# the mapping table cannot shift a block index: attn_queries.{i} feeds block i + 1
|
||||
name = re.sub(r"attn_res\.attn_queries\.(\d+)", lambda m: f"layers.{int(m[1]) + 1}.attention_input.query", name)
|
||||
# the table maps a module, not its bias: gate.weight is the router, gate.bias the score correction
|
||||
name = name.replace("mlp.gate.bias", "mlp.gate.e_score_correction_bias")
|
||||
return name, gen
|
||||
|
||||
def get_vocab_base(self):
|
||||
tokens, types, pre = super().get_vocab_base()
|
||||
n = self.hparams["unpadded_vocab_size"]
|
||||
return tokens[:n], types[:n], pre
|
||||
|
||||
def set_vocab(self):
|
||||
tokens, types, pre = self.get_vocab_base()
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre(pre)
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(types)
|
||||
special = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
eos = self.hparams["eos_token_id"]
|
||||
special.special_token_ids["eos"] = eos
|
||||
special.special_token_ids["eot"] = eos
|
||||
special.add_special_token["bos"] = False
|
||||
special.add_to_gguf(self.gguf_writer)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
hp["num_attention_heads"] = [hp.get("swa_num_attention_heads", hp["num_attention_heads"]) if s else hp["num_attention_heads"] for s in self.is_swa]
|
||||
hp["num_key_value_heads"] = [hp.get("swa_num_key_value_heads", hp["num_key_value_heads"]) if s else hp["num_key_value_heads"] for s in self.is_swa]
|
||||
for name in ("n_routed_experts", "num_experts_per_tok"):
|
||||
if name not in hp:
|
||||
raise KeyError(f"Inkling2: missing {name}")
|
||||
super().set_gguf_parameters()
|
||||
w = self.gguf_writer
|
||||
w.add_vocab_size(hp["vocab_size"])
|
||||
w.add_vocab_size_unpadded(hp["unpadded_vocab_size"])
|
||||
w.add_expert_feed_forward_length(self.find_hparam(["moe_intermediate_size", "intermediate_size"]))
|
||||
w.add_key_length_swa(hp.get("swa_head_dim", hp["head_dim"]))
|
||||
w.add_value_length_swa(hp.get("swa_head_dim", hp["head_dim"]))
|
||||
w.add_rope_dimension_count(0)
|
||||
w.add_sliding_window(hp["sliding_window_size"])
|
||||
w.add_sliding_window_pattern(self.is_swa)
|
||||
w.add_shortconv_l_cache(self.find_hparam(["conv_kernel_size", "sconv_kernel_size"]))
|
||||
w.add_expert_weights_scale(hp["route_scale"])
|
||||
w.add_activation_situ_beta(hp["mlp_gate_softcap"])
|
||||
w.add_activation_situ_linear_beta(hp["mlp_up_softcap"])
|
||||
w.add_attn_res_block_size(hp["n_layers_per_attn_res_block"])
|
||||
w.add_final_logit_softcapping(hp["final_logit_softcapping"])
|
||||
w.add_relative_length(hp["d_rel"])
|
||||
w.add_relative_extent(hp["rel_extent"])
|
||||
w.add_qk_clip(hp["qk_l2_clip_threshold"])
|
||||
w.add_relative_clip(hp["rel_l2_clip_threshold"])
|
||||
w.add_log_scale_floor(hp["log_scaling_n_floor"])
|
||||
w.add_log_scale_alpha(hp["log_scaling_alpha"])
|
||||
|
||||
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
|
||||
if new_name.endswith(("_conv.weight", "_clip.weight", "_res_query.weight", "attn_rel_bias.weight")):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name in self._packed_experts:
|
||||
target = name.replace(".w13_weight", ".gate_up_proj").replace(".w2_weight", ".down_proj")
|
||||
yield self.map_tensor_name(target, try_suffixes=()) + ".weight", cast("Tensor", self._packed_experts[name])
|
||||
return
|
||||
if name.endswith(".w13_weight"):
|
||||
shape = data_torch.shape
|
||||
data_torch = data_torch.reshape(shape[0], shape[1] // 2, 2, shape[2]).transpose(1, 2).reshape(shape)
|
||||
name = name.replace(".w13_weight", ".gate_up_proj")
|
||||
name = name.replace(".w2_weight", ".down_proj")
|
||||
for static, conv in ((".static_weight", ".conv1d.weight"), (".static_w", ".weight")):
|
||||
if name.endswith(static):
|
||||
data_torch = data_torch.flip(0).T.contiguous()
|
||||
name = name.removesuffix(static) + conv
|
||||
if name.endswith((".conv1d.weight", "sconv.weight")):
|
||||
data_torch = data_torch.squeeze(1) if data_torch.ndim == 3 else data_torch
|
||||
elif name.endswith(".rel_logits_proj.proj"):
|
||||
data_torch = data_torch.T.contiguous()
|
||||
# a name the table maps without a suffix is a bare parameter
|
||||
new_name = self.tensor_map.get_name(name)
|
||||
if new_name is not None:
|
||||
suffix = ".bias" if name.endswith(".e_score_correction_bias") else ".weight"
|
||||
yield new_name + suffix, data_torch
|
||||
else:
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("Inkling2ForConditionalGeneration")
|
||||
class Inkling2VisionModel(MmprojModel):
|
||||
def get_vision_config(self):
|
||||
config = self.global_config.get("vision_config")
|
||||
if not config:
|
||||
return None
|
||||
config = dict(config)
|
||||
config["patch_size"] = config.get("native_patch_size", config.get("patch_size", 20))
|
||||
config["spatial_merge_size"] = config.get("adapter_merge_size", config.get("spatial_merge_size", 2))
|
||||
config["intermediate_size"] = config.get("intermediate_size") or int(config["hidden_size"] * config.get("mlp_ratio", 4.0))
|
||||
config["image_size"] = config["posemb_grid_size"] * config["patch_size"]
|
||||
if config.get("num_channels", 3) != 3 or config.get("hidden_act", "gelu") != "gelu":
|
||||
raise ValueError("Inkling2 vision requires RGB patches and exact GELU")
|
||||
return config
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
self.preprocessor_config.setdefault("image_mean", [0.5, 0.5, 0.5])
|
||||
self.preprocessor_config.setdefault("image_std", [0.5, 0.5, 0.5])
|
||||
super().set_gguf_parameters()
|
||||
config = self.hparams
|
||||
rope = config.get("rope_parameters") or {"rope_type": "axial", "rope_theta": 10000.0}
|
||||
if rope["rope_type"] != "axial" or rope["rope_theta"] != 10000.0:
|
||||
raise ValueError("Inkling2 vision requires axial RoPE with theta 10000")
|
||||
w = self.gguf_writer
|
||||
w.add_clip_vision_projector_type(gguf.VisionProjectorType.INKLING2V)
|
||||
w.add_vision_attention_layernorm_eps(config["layer_norm_eps"])
|
||||
w.add_vision_spatial_merge_size(config["spatial_merge_size"])
|
||||
|
||||
def dequant_model(self):
|
||||
# vision tensors are BF16, only the text experts are quantized
|
||||
return
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
name, gen = item
|
||||
if not name.startswith(("model.visual.", "model.vision_tower.")):
|
||||
return None
|
||||
return name, gen
|
||||
|
||||
# mm.{bid} takes the projector layer index, the mapping table cannot derive it from these names
|
||||
_projector_layers = {"model.visual.proj_in": 1, "model.visual.proj_out": 2, "model.vision_tower.merger.linear_1": 1, "model.vision_tower.merger.linear_2": 2}
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
module, _, suffix = name.rpartition(".")
|
||||
if module in self._projector_layers:
|
||||
yield self.format_tensor_name(gguf.MODEL_TENSOR.V_MMPROJ, self._projector_layers[module], "." + suffix), data_torch
|
||||
return
|
||||
# lay out the linear patch embedding (HWC patches) as a conv2d for build_inp()
|
||||
if name.endswith(".patch_embedding.weight"):
|
||||
ps = self.hparams["patch_size"]
|
||||
data_torch = data_torch.reshape(data_torch.shape[0], ps, ps, 3).permute(0, 3, 1, 2)
|
||||
# these are bare parameters, they have no .weight
|
||||
if name.endswith(("position_embedding", ".lambda_1", ".lambda_2", ".gamma")):
|
||||
name = name.removesuffix(".gamma") + ".weight"
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
+54
-1
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from typing import Iterable, Sequence, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
@@ -8,6 +10,7 @@ if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, MmprojModel, gguf, logger
|
||||
from .qwen import DSparkModel
|
||||
|
||||
|
||||
@ModelBase.register("MiniMaxText01ForCausalLM")
|
||||
@@ -170,6 +173,15 @@ class MiniMaxM2Model(TextModel):
|
||||
class MiniMaxM3Model(MiniMaxM2Model):
|
||||
model_arch = gguf.MODEL_ARCH.MINIMAXM3
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
# transformers 5.x folds sparse_attention_config and moe_layer_freq into other keys
|
||||
with open(self.dir_model / "config.json", "r", encoding="utf-8") as f:
|
||||
raw_hparams = json.load(f)
|
||||
for key, value in raw_hparams.get("text_config", raw_hparams).items():
|
||||
self.hparams.setdefault(key, value)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
if ".indexer." in new_name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
@@ -188,6 +200,7 @@ class MiniMaxM3Model(MiniMaxM2Model):
|
||||
self.gguf_writer.add_indexer_top_k(sac["sparse_topk_blocks"])
|
||||
self.gguf_writer.add_indexer_block_size(sac["sparse_block_size"])
|
||||
self.gguf_writer.add_indexer_local_blocks(sac["sparse_local_block"])
|
||||
self.gguf_writer.add_indexer_types([bool(v) for v in sac["sparse_attention_freq"]])
|
||||
|
||||
moe_layer_freq = self.find_hparam(["moe_layer_freq"])
|
||||
n_dense = 0
|
||||
@@ -201,7 +214,47 @@ class MiniMaxM3Model(MiniMaxM2Model):
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
|
||||
# Gemma-style (1 + w) RMSNorm: bake the +1 in so llama.cpp can use plain RMSNorm
|
||||
if name.endswith("norm.weight"):
|
||||
data_torch = data_torch + 1.0
|
||||
data_torch = data_torch.float() + 1.0
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("DSparkMiniMaxDraftModel")
|
||||
class MiniMaxM3DSparkModel(DSparkModel):
|
||||
model_arch = gguf.MODEL_ARCH.DFLASH
|
||||
|
||||
def __init__(self, dir_model, *args, **kwargs):
|
||||
# transformers drops the draft keys when it parses the minimax_m3_vl config
|
||||
with open(dir_model / "config.json", "r", encoding="utf-8") as f:
|
||||
kwargs["hparams"] = json.load(f)
|
||||
super().__init__(dir_model, *args, **kwargs)
|
||||
|
||||
# draft layers use a dense FFN, the MoE keys are copied from the target config
|
||||
self.hparams["intermediate_size"] = self.hparams["dspark_ffn_hidden_size"]
|
||||
for key in ("num_local_experts", "num_experts_per_tok", "scoring_func"):
|
||||
self.hparams.pop(key, None)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
|
||||
self.gguf_writer.add_hidden_act(self.hparams["hidden_act"])
|
||||
|
||||
layer_types = self.hparams["layer_types"]
|
||||
self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
|
||||
self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in layer_types])
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
name, gen = item
|
||||
name = name.removeprefix("language_model.model.dspark.").replace(".decoder_layer.", ".")
|
||||
if name == "final_norm.weight":
|
||||
name = "norm.weight"
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None):
|
||||
# Gemma-style (1 + w) RMSNorm, same as the target
|
||||
if name.endswith("norm.weight"):
|
||||
data_torch = data_torch.float() + 1.0
|
||||
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@@ -0,0 +1,285 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, MmprojModel, gguf
|
||||
from .drama3 import fold_snake
|
||||
from .qwen import Qwen3Model
|
||||
|
||||
# MiniMax Music 3, diffusers layout: one checkpoint per component
|
||||
# text model: the global LLM (language_model/), samples the semantic code of each frame
|
||||
# mmproj: RVQ depth decoder, condition encoder, flow-matching DiT and DAC decoder (vocoder)
|
||||
|
||||
LM_DIR = "language_model"
|
||||
TOK_DIR = "tokenizer"
|
||||
MMPROJ_DIRS = ("rvq_depth_decoder", "condition_encoder", "transformer", "vocoder")
|
||||
|
||||
# token ids of the checkpoint, see encoders.py of the diffusers pipeline
|
||||
AUDIO_START = 151669
|
||||
AUDIO_END = 151670
|
||||
AUDIO_CODE_OFFSET = 151675
|
||||
N_SEMANTIC = 16384
|
||||
|
||||
|
||||
def _load_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
return ModelBase.load_hparams(dir_model / LM_DIR, is_mistral_format=False)
|
||||
|
||||
|
||||
def _index_dirs(model: ModelBase, dirs: Iterable[str], keep: Callable[[str], bool]) -> dict[str, Callable[[], Tensor]]:
|
||||
# the names get the component dir as prefix
|
||||
root = model.dir_model
|
||||
tensors: dict[str, Callable[[], Tensor]] = {}
|
||||
try:
|
||||
for d in dirs:
|
||||
model.dir_model = root / d
|
||||
prefix = "model" if d == LM_DIR else "diffusion_pytorch_model"
|
||||
for name, gen in model.index_local_tensors(prefix).items():
|
||||
if keep(f"{d}.{name}"):
|
||||
tensors[f"{d}.{name}"] = gen
|
||||
finally:
|
||||
model.dir_model = root
|
||||
return tensors
|
||||
|
||||
|
||||
def _fold_wn(g: Tensor, v: Tensor) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = g.float(), v.float()
|
||||
return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
|
||||
|
||||
|
||||
@ModelBase.register("MiniMaxMusic3ForConditionalGeneration")
|
||||
class MiniMaxMusic3TextModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
tensors = _index_dirs(self, [LM_DIR], lambda _: True)
|
||||
return {name.removeprefix(LM_DIR + "."): gen for name, gen in tensors.items()}
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
# the codes are sampled with classifier-free guidance, the uncond prompt runs as a second sequence
|
||||
self.gguf_writer.add_classifier_free_guidance(True)
|
||||
|
||||
def set_vocab(self):
|
||||
root = self.dir_model
|
||||
self.dir_model = root / TOK_DIR
|
||||
try:
|
||||
tokens, toktypes, tokpre = self.get_vocab_base()
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
finally:
|
||||
self.dir_model = root
|
||||
|
||||
# the semantic codes are not in the tokenizer, name them for debugging
|
||||
for i in range(N_SEMANTIC):
|
||||
tokens[AUDIO_CODE_OFFSET + i] = f"<|audio_code_{i}|>"
|
||||
toktypes[AUDIO_CODE_OFFSET + i] = gguf.TokenType.CONTROL
|
||||
|
||||
# the generation ends with <|audio_end|>, the chat template of the tokenizer is not used
|
||||
template_path = Path(__file__).parent.parent / "models" / "templates" / "MiniMaxAI-MiniMax-Music3-tts.jinja"
|
||||
with open(template_path, encoding="utf-8") as f:
|
||||
special_vocab.chat_template = [{"name": "tts", "template": f.read()}]
|
||||
special_vocab.special_token_ids["eos"] = AUDIO_END
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre(tokpre)
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
|
||||
@ModelBase.register("MiniMaxMusic3ForConditionalGeneration")
|
||||
class MiniMaxMusic3MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
|
||||
hparams["text_config"] = _load_hparams(dir_model)
|
||||
for d in MMPROJ_DIRS:
|
||||
hparams[d] = ModelBase.load_hparams(dir_model / d, is_mistral_format=False)
|
||||
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
# the generic gen layers are the depth decoder
|
||||
return self.global_config["rvq_depth_decoder"]
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
# from the backbone, only the embeddings of the semantic codes
|
||||
return _index_dirs(self, (LM_DIR, *MMPROJ_DIRS),
|
||||
lambda name: not name.startswith(LM_DIR + ".") or name == f"{LM_DIR}.model.embed_tokens.weight")
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
return item
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.MINIMAX_MUSIC3_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(hp["hidden_size"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(hp["intermediate_size"])
|
||||
self.gguf_writer.add_gen_audio_block_count(hp["num_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(hp["num_attention_heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-6)
|
||||
# the DiT, the condition encoder and the vocoder hparams come from the tensor shapes or are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# only the transformer layers follow --outtype, the convs and the small projections stay F32
|
||||
if n_dims >= 2 and new_name.startswith(("a.gen.code.blk.", "a.gen.flow.blk.")):
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
if new_name.startswith(("a.gen.code.head", "a.gen.code.embd", "a.gen.code.out_embd", "a.gen.code.proj_in")):
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name.startswith("a."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
comp, key = name.split(".", 1)
|
||||
if comp == LM_DIR:
|
||||
yield ("a.gen.code.out_embd.weight", data_torch[AUDIO_CODE_OFFSET:AUDIO_CODE_OFFSET + N_SEMANTIC])
|
||||
elif comp == "rvq_depth_decoder":
|
||||
yield from self._depth(data_torch, key)
|
||||
elif comp == "condition_encoder":
|
||||
yield from self._cond(data_torch, key)
|
||||
elif comp == "transformer":
|
||||
yield from self._dit(data_torch, key)
|
||||
# the vocoder is folded in generate_extra_tensors()
|
||||
|
||||
def _depth(self, t: Tensor, key: str) -> Iterable[tuple[str, Tensor]]:
|
||||
p = "a.gen.code"
|
||||
n_cb = self.hparams["num_codebooks"] - 1
|
||||
if key == "audio_embeddings.weight":
|
||||
# one table per residual codebook
|
||||
yield (f"{p}.embd.weight", t.reshape(n_cb, -1, t.shape[-1]))
|
||||
elif key.startswith("audio_heads."):
|
||||
if key == "audio_heads.0.weight":
|
||||
heads = [self._tensor(f"rvq_depth_decoder.audio_heads.{i}.weight") for i in range(n_cb)]
|
||||
yield (f"{p}.head.weight", torch.stack(heads))
|
||||
elif key == "projection.weight":
|
||||
yield (f"{p}.proj_in.weight", t)
|
||||
elif key == "pos_embedding.weight":
|
||||
yield (f"{p}.position_embd.weight", t)
|
||||
elif key == "norm.weight":
|
||||
yield (f"{p}.post_ln.weight", t)
|
||||
else:
|
||||
_, il, rest = key.split(".", 2)
|
||||
layer = {
|
||||
"input_layernorm.weight": "ln1.weight",
|
||||
"attn.to_q.weight": "attn_q.weight",
|
||||
"attn.to_k.weight": "attn_k.weight",
|
||||
"attn.to_v.weight": "attn_v.weight",
|
||||
"attn.to_out.weight": "attn_out.weight",
|
||||
"post_attention_layernorm.weight": "ln2.weight",
|
||||
"gate_proj.weight": "ffn_gate.weight",
|
||||
"up_proj.weight": "ffn_up.weight",
|
||||
"down_proj.weight": "ffn_down.weight",
|
||||
}
|
||||
yield (f"{p}.blk.{il}.{layer[rest]}", t)
|
||||
|
||||
def _cond(self, t: Tensor, key: str) -> Iterable[tuple[str, Tensor]]:
|
||||
p = "a.gen.cond"
|
||||
if key == "layer_weight_logits":
|
||||
# softmax mix of the 8 hidden states of a frame, with the scale folded in
|
||||
scale = self._tensor("condition_encoder.layer_scale").float()
|
||||
yield (f"{p}.layer_w", torch.softmax(t.float(), dim=0) * scale)
|
||||
elif key == "proj.weight":
|
||||
yield (f"{p}.proj.weight", t)
|
||||
elif key == "proj.bias":
|
||||
yield (f"{p}.proj.bias", t)
|
||||
|
||||
def _dit(self, t: Tensor, key: str) -> Iterable[tuple[str, Tensor]]:
|
||||
p = "a.gen.flow"
|
||||
if key.startswith("transformer_blocks."):
|
||||
_, il, rest = key.split(".", 2)
|
||||
q = f"{p}.blk.{il}"
|
||||
if rest.startswith("ff_in."):
|
||||
# value half first, then the silu gate half
|
||||
suffix = rest.removeprefix("ff_in.")
|
||||
up, gate = t.chunk(2, dim=0)
|
||||
yield (f"{q}.ffn_up.{suffix}", up)
|
||||
yield (f"{q}.ffn_gate.{suffix}", gate)
|
||||
return
|
||||
layer = {
|
||||
"norm1.weight": "ln1.weight",
|
||||
"norm1.bias": "ln1.bias",
|
||||
"attn.to_q.weight": "attn_q.weight",
|
||||
"attn.to_k.weight": "attn_k.weight",
|
||||
"attn.to_v.weight": "attn_v.weight",
|
||||
"attn.to_out.0.weight": "attn_out.weight",
|
||||
"norm2.weight": "ln2.weight",
|
||||
"norm2.bias": "ln2.bias",
|
||||
"ff_out.weight": "ffn_down.weight",
|
||||
"ff_out.bias": "ffn_down.bias",
|
||||
}
|
||||
yield (f"{q}.{layer[rest]}", t)
|
||||
return
|
||||
if key == "time_proj.weight":
|
||||
# random Fourier features, 2 * pi folded in
|
||||
yield (f"{p}.time_freq", t.float().flatten() * 2.0 * torch.pi)
|
||||
return
|
||||
out = {
|
||||
"time_embed.linear_1.weight": "time_in.weight",
|
||||
"time_embed.linear_1.bias": "time_in.bias",
|
||||
"time_embed.linear_2.weight": "time_out.weight",
|
||||
"time_embed.linear_2.bias": "time_out.bias",
|
||||
"preprocess_conv.weight": "pre.weight",
|
||||
"proj_in.weight": "proj_in.weight",
|
||||
"proj_out.weight": "proj_out.weight",
|
||||
"postprocess_conv.weight": "post.weight",
|
||||
}[key]
|
||||
yield (f"{p}.{out}", t.squeeze(-1) if key.endswith("conv.weight") else t)
|
||||
|
||||
def _tensor(self, name: str) -> Tensor:
|
||||
# read directly, without the dtype handling of prepare_tensors()
|
||||
return self.model_tensors[name]().float()
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# DAC decoder, same layout as the yue2 VAE (a.gen.vae.*), with the snake and the weight norm folded
|
||||
def get(key: str) -> Tensor:
|
||||
return self._tensor(f"vocoder.{key}")
|
||||
|
||||
def wn(key: str) -> Tensor:
|
||||
return _fold_wn(get(key + ".weight_g"), get(key + ".weight_v"))
|
||||
|
||||
def snake(dst: str, key: str) -> Iterable[tuple[str, Tensor]]:
|
||||
a, b = fold_snake(get(key + ".alpha"))
|
||||
yield (f"{dst}.alpha", a)
|
||||
yield (f"{dst}.beta", b)
|
||||
|
||||
p = "a.gen.vae"
|
||||
yield (f"{p}.in_proj.weight", get("dec_in_proj.weight"))
|
||||
yield (f"{p}.in_proj.bias", get("dec_in_proj.bias"))
|
||||
yield (f"{p}.conv_in.weight", wn("conv_in"))
|
||||
yield (f"{p}.conv_in.bias", get("conv_in.bias"))
|
||||
n_blk = len(self.global_config["vocoder"]["upsampling_ratios"])
|
||||
for b in range(n_blk):
|
||||
s = f"blocks.{b}"
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.snake1")
|
||||
yield (f"{p}.blk.{b}.up.weight", wn(f"{s}.conv_t1"))
|
||||
yield (f"{p}.blk.{b}.up.bias", get(f"{s}.conv_t1.bias"))
|
||||
for r in range(3):
|
||||
rs = f"{s}.res_unit{r + 1}"
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake1", f"{rs}.snake1")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.weight", wn(f"{rs}.conv1"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.bias", get(f"{rs}.conv1.bias"))
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake2", f"{rs}.snake2")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.weight", wn(f"{rs}.conv2"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.bias", get(f"{rs}.conv2.bias"))
|
||||
yield from snake(f"{p}.post_snake", "snake_out")
|
||||
yield (f"{p}.conv_out.weight", wn("conv_out"))
|
||||
yield (f"{p}.conv_out.bias", get("conv_out.bias"))
|
||||
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import MmprojModel, ModelBase, gguf
|
||||
from .qwen import Qwen2Model
|
||||
|
||||
|
||||
# the upstream chat_template.jinja is a processor template (list content, audio parts), use plain ChatML with its default system prompt
|
||||
VIBEVOICE_ASR_CHAT_TEMPLATE = (
|
||||
"{%- if messages[0]['role'] == 'system' -%}"
|
||||
"{{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' -}}"
|
||||
"{%- set loop_messages = messages[1:] -%}"
|
||||
"{%- else -%}"
|
||||
"{{- '<|im_start|>system\\nYou are a helpful assistant that transcribes audio input into text output in JSON format.<|im_end|>\\n' -}}"
|
||||
"{%- set loop_messages = messages -%}"
|
||||
"{%- endif -%}"
|
||||
"{%- for message in loop_messages -%}"
|
||||
"{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}"
|
||||
"{%- endfor -%}"
|
||||
"{%- if add_generation_prompt -%}"
|
||||
"{{- '<|im_start|>assistant\\n' -}}"
|
||||
"{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
@ModelBase.register("VibeVoiceAsrForConditionalGeneration")
|
||||
@ModelBase.example("microsoft/VibeVoice-ASR-HF")
|
||||
class VibeVoiceAsrModel(Qwen2Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN2
|
||||
|
||||
def set_vocab(self):
|
||||
super().set_vocab()
|
||||
self.gguf_writer.add_chat_template(VIBEVOICE_ASR_CHAT_TEMPLATE)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not name.startswith("language_model."):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
|
||||
@ModelBase.register("VibeVoiceAsrForConditionalGeneration")
|
||||
class VibeVoiceAsrAudioModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
|
||||
# values hard-coded in tools/mtmd/models/vibevoice-asr.cpp
|
||||
_encoder_consts = {"channels": 1, "kernel_size": 7, "num_filters": 32, "ffn_expansion": 4, "hidden_act": "gelu", "rms_norm_eps": 1e-5}
|
||||
_encoders = ("acoustic_tokenizer_encoder", "semantic_tokenizer_encoder")
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
c = self.global_config["acoustic_tokenizer_encoder_config"]
|
||||
return {
|
||||
**c,
|
||||
"intermediate_size": c["ffn_expansion"] * c["hidden_size"],
|
||||
"num_attention_heads": 1,
|
||||
"num_hidden_layers": sum(c["depths"]),
|
||||
}
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
for enc in self._encoders:
|
||||
c = self.global_config[f"{enc}_config"]
|
||||
for k, v in self._encoder_consts.items():
|
||||
assert c[k] == v, f"{enc}.{k} = {c[k]}, expected {v}"
|
||||
assert c["depths"] == self.hparams["depths"] and c["downsampling_ratios"] == self.hparams["downsampling_ratios"]
|
||||
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.VIBEVOICE_ASR)
|
||||
# the input is the raw waveform, one channel takes the place of the mel bins
|
||||
self.gguf_writer.add_audio_num_mel_bins(1)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams["rms_norm_eps"])
|
||||
self.gguf_writer.add_audio_conv_kernel_size(self.hparams["kernel_size"])
|
||||
# stride of each stage, the stem has none
|
||||
self.gguf_writer.add_audio_patch_sizes([1] + [int(r) for r in self.hparams["downsampling_ratios"]])
|
||||
self.gguf_writer.add_audio_depths([int(d) for d in self.hparams["depths"]])
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
if ".dw." in new_name:
|
||||
return gguf.GGMLQuantizationType.F32 # ggml_ssm_conv
|
||||
if ".conv." in new_name and new_name.endswith(".weight"):
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not name.startswith(cls._encoders + ("multi_modal_projector.",)):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
_block_tensors = {
|
||||
"norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_NORM,
|
||||
"mixer.conv.weight": gguf.MODEL_TENSOR.A_ENC_CNX_DW,
|
||||
"mixer.conv.bias": gguf.MODEL_TENSOR.A_ENC_CNX_DW,
|
||||
"gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS1,
|
||||
"ffn_norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_NORM,
|
||||
"ffn.linear1.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP,
|
||||
"ffn.linear1.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP,
|
||||
"ffn.linear2.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
|
||||
"ffn.linear2.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
|
||||
"ffn_gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS2,
|
||||
}
|
||||
|
||||
# mm.a.mlp.{0,1} + mm.a.mlp_norm.0 for the acoustic connector, {2,3} + 2 for the semantic one
|
||||
_proj_tensors = {
|
||||
"acoustic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 0),
|
||||
"acoustic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 0),
|
||||
"acoustic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 1),
|
||||
"semantic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 2),
|
||||
"semantic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 2),
|
||||
"semantic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 3),
|
||||
}
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# the layer scales (gamma) have no suffix upstream
|
||||
suffix = "." + name.rsplit(".", 1)[-1] if name.endswith((".weight", ".bias")) else ".weight"
|
||||
|
||||
if name.startswith("multi_modal_projector."):
|
||||
key, idx = self._proj_tensors[name.split(".")[1]]
|
||||
yield (self.format_tensor_name(key, idx, suffix=suffix), data_torch)
|
||||
return
|
||||
|
||||
enc, _, rest = name.partition(".")
|
||||
enc_id = self._encoders.index(enc)
|
||||
parts = rest.split(".")
|
||||
depths = [int(d) for d in self.hparams["depths"]]
|
||||
|
||||
# causal convs of the stages: 0 = stem, 1..6 = downsample, 7 = head
|
||||
if rest.startswith("stem.conv."):
|
||||
conv_id = 0
|
||||
elif parts[0] == "conv_layers" and parts[2] == "conv":
|
||||
conv_id = int(parts[1]) + 1
|
||||
elif parts[0] == "head":
|
||||
conv_id = len(depths)
|
||||
else:
|
||||
conv_id = None
|
||||
if conv_id is not None:
|
||||
yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_CNX_CONV].format(bid=enc_id, xid=conv_id) + suffix, data_torch)
|
||||
return
|
||||
|
||||
# ConvNeXt blocks are numbered per stage upstream and flat here
|
||||
if parts[0] == "stem":
|
||||
stage, blk, tail = 0, int(parts[2]), ".".join(parts[3:])
|
||||
else:
|
||||
assert parts[0] == "conv_layers" and parts[2] == "stage", name
|
||||
stage, blk, tail = int(parts[1]) + 1, int(parts[3]), ".".join(parts[4:])
|
||||
xid = sum(depths[:stage]) + blk
|
||||
|
||||
key = self._block_tensors[tail]
|
||||
if key == gguf.MODEL_TENSOR.A_ENC_CNX_DW and tail.endswith(".weight"):
|
||||
data_torch = data_torch.squeeze(1) # [C, 1, K] -> [C, K] for ggml_ssm_conv
|
||||
yield (gguf.TENSOR_NAMES[key].format(bid=enc_id, xid=xid) + suffix, data_torch)
|
||||
@@ -0,0 +1,234 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, MmprojModel, gguf, logger
|
||||
from .qwen import QwenModel, Qwen3Model
|
||||
|
||||
# token ids of the yue2_infer protocol (protocol.py)
|
||||
EOD = 151643
|
||||
ABC_START, ABC_END = 151847, 151848
|
||||
MUSIC_START, MUSIC_END = 151851, 151852
|
||||
CODEC_OFFSET, CODEC_SIZE = 151853, 32768
|
||||
LATENT_START = 184621
|
||||
|
||||
# the VAE is a separate repo (m-a-p/YuE2-Vae), downloaded into this subdir of the model dir
|
||||
VAE_DIR = "vae"
|
||||
|
||||
INSTRUCTIONS = {
|
||||
"off": "Generate music with codec tokens from the given conditions.",
|
||||
"melody": "Generate a melody-only ABC transcription without chord symbols, then generate music with codec tokens from the given conditions.",
|
||||
"full": "Generate a chord-annotated ABC transcription, then generate music with codec tokens from the given conditions.",
|
||||
}
|
||||
|
||||
# the text is "[Tags]\n<style>\n[Lyrics]\n<lyrics>", optionally followed by "\n[ABC]\n<score>" (cover mode)
|
||||
# voice picks the planning mode: full (default), melody or off
|
||||
TTS_TEMPLATE = (
|
||||
"{%- set text = messages[0]['content'] -%}"
|
||||
"{%- set mode = (voice | lower) if voice is defined and voice and (voice | lower) in ['full', 'melody', 'off'] else 'full' -%}"
|
||||
"{%- set parts = text.split('\\n[ABC]\\n') -%}"
|
||||
"{%- set instructions = {"
|
||||
+ ", ".join(f"'{k}': '{v}'" for k, v in INSTRUCTIONS.items())
|
||||
+ "} -%}"
|
||||
"{{- '<|endoftext|>' + instructions[mode] + '\\n' + parts[0].rstrip() + '\\n' -}}"
|
||||
"{%- if mode == 'off' -%}"
|
||||
"{{- '<abc></abc><|music_start|>' -}}"
|
||||
"{%- elif parts | length > 1 -%}"
|
||||
"{{- '<abc>' + parts[1] + '</abc><|music_start|>' -}}"
|
||||
"{%- else -%}"
|
||||
"{{- '<abc>' -}}"
|
||||
"{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
def _is_nar_tensor(name: str) -> bool:
|
||||
return ".nar_" in name or name.startswith(("llm2vae.", "vae2llm.", "time_embedder.", "latent_pos_embed."))
|
||||
|
||||
|
||||
@ModelBase.register("YuE2ForCausalLM")
|
||||
class YuE2TextModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
# the NAR path goes to the mmproj
|
||||
if _is_nar_tensor(item[0]):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
# the codes are sampled with classifier-free guidance, the uncond prompt runs as a second sequence
|
||||
self.gguf_writer.add_classifier_free_guidance(True)
|
||||
|
||||
def set_vocab(self):
|
||||
# checkpoint-native tiktoken file, see tokenization_yue2.py
|
||||
ranks: dict[bytes, int] = {}
|
||||
with open(self.dir_model / "qwen.tiktoken", "rb") as f:
|
||||
for line in f.read().splitlines():
|
||||
if line:
|
||||
tok, rank = line.split()
|
||||
ranks[base64.b64decode(tok)] = int(rank)
|
||||
if len(ranks) != EOD:
|
||||
raise ValueError(f"YuE2: expected {EOD} BPE tokens, got {len(ranks)}")
|
||||
|
||||
vocab_size = self.hparams["vocab_size"]
|
||||
tokens: list[str] = [""] * vocab_size
|
||||
toktypes: list[int] = [gguf.TokenType.UNUSED] * vocab_size
|
||||
merges: list[str] = []
|
||||
for tok, rank in ranks.items():
|
||||
tokens[rank] = QwenModel.token_bytes_to_string(tok)
|
||||
toktypes[rank] = gguf.TokenType.NORMAL
|
||||
if len(tok) > 1:
|
||||
merged = QwenModel.bpe(ranks, tok, max_rank=rank)
|
||||
assert len(merged) == 2
|
||||
merges.append(" ".join(map(QwenModel.token_bytes_to_string, merged)))
|
||||
|
||||
specials = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<R>", "<S>", "<X>", "<mask>", "<sep>"]
|
||||
specials += [f"<extra_{i}>" for i in range(200)]
|
||||
specials[204:206] = ["<abc>", "</abc>"]
|
||||
named = {EOD + i: s for i, s in enumerate(specials)}
|
||||
named[MUSIC_START] = "<|music_start|>"
|
||||
named[MUSIC_END] = "<|music_end|>"
|
||||
for i in range(CODEC_SIZE):
|
||||
named[CODEC_OFFSET + i] = f"<|codec_{i}|>"
|
||||
for i, s in enumerate(["<|latent_start|>", "<|latent_end|>", "<|latent_pad|>"]):
|
||||
named[LATENT_START + i] = s
|
||||
for i in range(EOD, vocab_size):
|
||||
tokens[i] = named.get(i, f"[PAD{i}]")
|
||||
toktypes[i] = gguf.TokenType.CONTROL if i in named else gguf.TokenType.UNUSED
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre("qwen2")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
self.gguf_writer.add_token_merges(merges)
|
||||
self.gguf_writer.add_bos_token_id(EOD)
|
||||
self.gguf_writer.add_eos_token_id(MUSIC_END)
|
||||
self.gguf_writer.add_pad_token_id(EOD)
|
||||
self.gguf_writer.add_add_bos_token(False)
|
||||
self.gguf_writer.add_chat_template([{"name": "tts", "template": TTS_TEMPLATE}])
|
||||
|
||||
|
||||
@ModelBase.register("YuE2ForCausalLM")
|
||||
class YuE2MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
# the NAR path: same shapes as the backbone layers
|
||||
return {k: self.global_config[k] for k in ("num_hidden_layers", "hidden_size", "intermediate_size",
|
||||
"num_attention_heads", "num_key_value_heads", "rms_norm_eps")}
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name = item[0]
|
||||
if name == "latent_pos_embed.pe":
|
||||
return None # sinusoidal, computed at runtime
|
||||
if _is_nar_tensor(name) or name == "model.norm.weight":
|
||||
return item
|
||||
return None
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.YUE2_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(hp["hidden_size"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(hp["intermediate_size"])
|
||||
self.gguf_writer.add_gen_audio_block_count(hp["num_hidden_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(hp["num_attention_heads"])
|
||||
self.gguf_writer.add_gen_audio_head_count_kv(hp["num_key_value_heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(hp["rms_norm_eps"])
|
||||
# rope theta, the ODE solver and the VAE hparams are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# read on the host, or small convs and projections of the flow / VAE graphs
|
||||
if not new_name.startswith("a.gen.nar.blk.") or n_dims < 2:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@staticmethod
|
||||
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
|
||||
return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
from safetensors.torch import load_file
|
||||
path = self.dir_model / VAE_DIR / "model.safetensors"
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"YuE2: VAE not found at {path}, download m-a-p/YuE2-Vae into it")
|
||||
logger.info(f"gguf: loading VAE decoder from '{path}'")
|
||||
sd = {k[len("decoder.layers."):]: v for k, v in load_file(str(path)).items() if k.startswith("decoder.layers.")}
|
||||
p = "a.gen.vae"
|
||||
|
||||
# SnakeBeta with log-scale params: x + 1/(exp(beta) + 1e-9) * sin(exp(alpha) * x)^2
|
||||
def snake(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
yield (f"{dst}.alpha", torch.exp(sd[src + ".alpha"].float()))
|
||||
yield (f"{dst}.beta", 1.0 / (torch.exp(sd[src + ".beta"].float()) + 1e-9))
|
||||
|
||||
yield (f"{p}.conv_in.weight", self._wn(sd, "0"))
|
||||
yield (f"{p}.conv_in.bias", sd["0.bias"])
|
||||
n_blk = sum(1 for k in sd if k.endswith(".layers.1.weight_g") and k.count(".") == 3)
|
||||
for b in range(n_blk):
|
||||
s = f"{b + 1}.layers"
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.0")
|
||||
yield (f"{p}.blk.{b}.up.weight", self._wn(sd, f"{s}.1"))
|
||||
yield (f"{p}.blk.{b}.up.bias", sd[f"{s}.1.bias"])
|
||||
for r in range(3):
|
||||
rs = f"{s}.{r + 2}.layers"
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake1", f"{rs}.0")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.weight", self._wn(sd, f"{rs}.1"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.bias", sd[f"{rs}.1.bias"])
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake2", f"{rs}.2")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.weight", self._wn(sd, f"{rs}.3"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.bias", sd[f"{rs}.3.bias"])
|
||||
yield from snake(f"{p}.post_snake", f"{n_blk + 1}")
|
||||
yield (f"{p}.conv_out.weight", self._wn(sd, f"{n_blk + 2}"))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name.startswith("a.gen.vae."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
p = "a.gen.nar"
|
||||
aux = {
|
||||
"model.norm.weight": f"{p}.output_norm.weight",
|
||||
"vae2llm.weight": f"{p}.inp.weight",
|
||||
"vae2llm.bias": f"{p}.inp.bias",
|
||||
"llm2vae.weight": f"{p}.out.weight",
|
||||
"llm2vae.bias": f"{p}.out.bias",
|
||||
"time_embedder.mlp.0.weight": f"{p}.time.0.weight",
|
||||
"time_embedder.mlp.0.bias": f"{p}.time.0.bias",
|
||||
"time_embedder.mlp.2.weight": f"{p}.time.2.weight",
|
||||
"time_embedder.mlp.2.bias": f"{p}.time.2.bias",
|
||||
}
|
||||
if name in aux:
|
||||
yield (aux[name], data_torch)
|
||||
return
|
||||
|
||||
# the NAR layers keep the libllama names, mtmd loads them as a text model
|
||||
assert bid is not None, name
|
||||
layer = {
|
||||
"nar_input_layernorm.weight": "attn_norm.weight",
|
||||
"nar_self_attn.q_proj.weight": "attn_q.weight",
|
||||
"nar_self_attn.k_proj.weight": "attn_k.weight",
|
||||
"nar_self_attn.v_proj.weight": "attn_v.weight",
|
||||
"nar_self_attn.o_proj.weight": "attn_output.weight",
|
||||
"nar_self_attn.q_norm.weight": "attn_q_norm.weight",
|
||||
"nar_self_attn.k_norm.weight": "attn_k_norm.weight",
|
||||
"nar_pre_mlp_layernorm.weight": "ffn_norm.weight",
|
||||
"nar_mlp.gate_proj.weight": "ffn_gate.weight",
|
||||
"nar_mlp.up_proj.weight": "ffn_up.weight",
|
||||
"nar_mlp.down_proj.weight": "ffn_down.weight",
|
||||
}
|
||||
key = name.split(".", 3)[3] # strip "model.layers.{bid}."
|
||||
yield (f"{p}.blk.{bid}.{layer[key]}", data_torch)
|
||||
@@ -0,0 +1,575 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, MmprojModel, LazyTorchTensor, gguf, logger
|
||||
|
||||
LM_DIR = Path("lm") / "zonos3_mini"
|
||||
|
||||
# codec decoder: quantizers and semantic head from the generator, the rest from the head checkpoint
|
||||
CODEC_GENERATOR = Path("autoencoder") / "generator.pt"
|
||||
CODEC_VARIANT = "keep2_plain"
|
||||
CODEC_HEAD = Path("autoencoder") / "heads" / CODEC_VARIANT / "step_0050000.pt"
|
||||
CODEC_STAGES = [0, 2, 4, 6, 8] # transformer stages of the decoder
|
||||
|
||||
DEFAULT_VOICE = "Savannah_en-American_F_1"
|
||||
|
||||
# speaker encoder (ownmect_s2), MECT_B2 blocks: (cnn depth, out channels, (stride f, stride t), expansion)
|
||||
SPK_CKPT = Path("speaker_encoders") / "ownmect_s2" / "ckpt_0037.pth"
|
||||
SPK_STAGES = [
|
||||
[(3, 32, (1, 1), 4)] * 2,
|
||||
[(3, 64, (2, 1), 2)] + [(3, 64, (1, 2), 2)] * 3,
|
||||
[(3, 128, (2, 1), 2)] + [(3, 128, (1, 2), 2)] * 4,
|
||||
[(3, 256, (2, 1), 1)] + [(2, 256, (1, 2), 1)] * 2,
|
||||
]
|
||||
VOICE_MAX_FRAMES = 900 # reference_max_frames of the reference
|
||||
|
||||
# GDN geometry, hard-coded in the reference (zonos3.reference.lm)
|
||||
GDN_HEADS, GDN_HEAD_DIM, GDN_KERNEL = 4, 128, 4
|
||||
|
||||
# the text table keeps the tokenizer ids shifted by this offset, ids 2 and 3 are BOS and EOS
|
||||
TEXT_OFFSET = 4
|
||||
TEXT_BOS, TEXT_EOS = 2, 3
|
||||
|
||||
|
||||
def _round_up(value: float, multiple: int) -> int:
|
||||
return multiple * ((int(value) + multiple - 1) // multiple)
|
||||
|
||||
|
||||
def _load_params(dir_model: Path) -> dict[str, Any] | None:
|
||||
path = dir_model / LM_DIR / "params.json"
|
||||
if not path.is_file():
|
||||
return None
|
||||
with open(path, encoding="utf-8") as f:
|
||||
params = json.load(f).get("model", {})
|
||||
return params if params.get("model_type") == "zonos2" else None
|
||||
|
||||
|
||||
def _is_zonos3_checkpoint(dir_model: Path) -> bool:
|
||||
return _load_params(dir_model) is not None
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_zonos3_checkpoint)
|
||||
def _load_zonos3_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected ZONOS3 checkpoint")
|
||||
params = _load_params(dir_model)
|
||||
assert params is not None
|
||||
for key, want in (("attention_pattern", "gdn3_global1"), ("residual_type", "full_attnres"),
|
||||
("moe_router_type", "linear"), ("moe_router_use_eda", False), ("use_xsa", False),
|
||||
("moe_end_from_layer", 0), ("speaker_enabled", True)):
|
||||
if params.get(key) != want:
|
||||
raise NotImplementedError(f"ZONOS3: unsupported {key} = {params.get(key)!r}, expected {want!r}")
|
||||
text_config = dict(params, hidden_size=params["dim"])
|
||||
return {"architectures": ["Zonos3Model"], "text_config": text_config, "audio_config": {"num_hidden_layers": 0}}
|
||||
|
||||
|
||||
def _index_lm_tensors(model: ModelBase) -> dict[str, Callable[[], Tensor]]:
|
||||
path = model.dir_model / LM_DIR / "model.pth"
|
||||
logger.info(f"gguf: indexing model part '{LM_DIR / 'model.pth'}'")
|
||||
state = torch.load(str(path), map_location="cpu", mmap=True, weights_only=True)
|
||||
tensors: dict[str, Callable[[], Tensor]] = {}
|
||||
for name, data in state.items():
|
||||
gen = (lambda data=data: LazyTorchTensor.from_eager(data)) if model.lazy else (lambda data=data: data)
|
||||
titem = model.filter_tensors((name, gen))
|
||||
model._source_tensor_names[name] = titem[0] if titem else None
|
||||
if titem:
|
||||
tensors[titem[0]] = titem[1]
|
||||
return tensors
|
||||
|
||||
|
||||
@ModelBase.register("Zonos3Model")
|
||||
class Zonos3TextModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.ZONOS3
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
hp = self.hparams
|
||||
self.n_embd = hp["dim"]
|
||||
self.n_codebooks = hp["n_codebooks"]
|
||||
self.audio_vocab = hp["codebook_size"] + 2 # codes, EOA, pad
|
||||
self.audio_pad_id = hp["audio_pad_id"]
|
||||
self.text_rows = hp["text_vocab"] + 1 # + the padding row, used by the generated rows
|
||||
self.cond_counts = [n + 1 for n in hp["condition_bucket_counts"]] # + the unknown bucket
|
||||
self.gdn_period = int(hp["attention_pattern"][len("gdn"):].split("_")[0]) + 1
|
||||
self.n_ff = _round_up(hp["ffn_dim_multiplier"] * hp["dim"], hp["multiple_of"])
|
||||
self.n_ff_exp = _round_up(hp["moe_ffn_dim_multiplier"] * hp["dim"], hp["moe_multiple_of"])
|
||||
self.n_latent = int(hp["dim"] * hp["moe_alpha"])
|
||||
self._embd: dict[str, Tensor] = {}
|
||||
self._router: dict[int, dict[str, Tensor]] = {}
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
return _index_lm_tensors(self)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
# audio heads and speaker projection go to the mmproj
|
||||
if name.startswith(("multi_output.", "speaker_projection.")):
|
||||
return None
|
||||
return item
|
||||
|
||||
def set_vocab(self):
|
||||
with open(self.dir_model / LM_DIR / "tokenizer" / "superbpe8192.json", encoding="utf-8") as f:
|
||||
tok = json.load(f)
|
||||
model = tok["model"]
|
||||
if model["type"] != "BPE" or model.get("byte_fallback") or tok.get("normalizer") is not None:
|
||||
raise NotImplementedError("ZONOS3: unexpected tokenizer type")
|
||||
|
||||
vocab: dict[str, int] = model["vocab"]
|
||||
n_bpe = len(vocab)
|
||||
if n_bpe + TEXT_OFFSET != self.hparams["text_vocab"]:
|
||||
raise ValueError(f"ZONOS3: tokenizer has {n_bpe} tokens, text_vocab is {self.hparams['text_vocab']}")
|
||||
added = {t["id"] for t in tok.get("added_tokens", [])}
|
||||
|
||||
tokens: list[str] = ["<|unused_0|>", "<|unused_1|>", "<|bos|>", "<|eos|>"]
|
||||
toktypes: list[int] = [gguf.TokenType.CONTROL] * TEXT_OFFSET
|
||||
inv = {i: t for t, i in vocab.items()}
|
||||
for i in range(n_bpe):
|
||||
tokens.append(inv[i])
|
||||
toktypes.append(gguf.TokenType.CONTROL if i in added else gguf.TokenType.NORMAL)
|
||||
|
||||
# padding row of the text table (the text column of every generated row)
|
||||
tokens.append("<|text_pad|>")
|
||||
toktypes.append(gguf.TokenType.CONTROL)
|
||||
|
||||
# condition rows, the last bucket of each table is "unknown"
|
||||
for cond, n in zip(self.hparams["condition_names"], self.cond_counts):
|
||||
tokens += [f"<|{cond}_{i}|>" for i in range(n - 1)] + [f"<|{cond}_unk|>"]
|
||||
toktypes += [gguf.TokenType.CONTROL] * n
|
||||
|
||||
merges = [" ".join(m) if isinstance(m, list) else m for m in model["merges"]]
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre("zonos3")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
self.gguf_writer.add_token_merges(merges)
|
||||
self.gguf_writer.add_bos_token_id(TEXT_BOS)
|
||||
self.gguf_writer.add_eos_token_id(TEXT_EOS)
|
||||
self.gguf_writer.add_pad_token_id(TEXT_OFFSET + 1) # <|pad|> of the tokenizer
|
||||
# the reference wraps the text as [BOS] + ids + [EOS]
|
||||
self.gguf_writer.add_add_bos_token(True)
|
||||
self.gguf_writer.add_add_eos_token(True)
|
||||
# the reference normalizes the text with NeMo, the template covers the English numbers
|
||||
template_path = Path(__file__).parent.parent / "models" / "templates" / "Zyphra-ZONOS3-Mini-tts.jinja"
|
||||
with open(template_path, "r", encoding="utf-8") as f:
|
||||
self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}])
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
n_layer = self.block_count
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_block_count(n_layer)
|
||||
self.gguf_writer.add_context_length(hp["max_seqlen"])
|
||||
self.gguf_writer.add_embedding_length(self.n_embd)
|
||||
self.gguf_writer.add_feed_forward_length(self.n_ff)
|
||||
self.gguf_writer.add_head_count(hp["n_heads"])
|
||||
self.gguf_writer.add_head_count_kv(hp["n_kv_heads"])
|
||||
self.gguf_writer.add_key_length(hp["head_dim"])
|
||||
self.gguf_writer.add_value_length(hp["head_dim"])
|
||||
self.gguf_writer.add_rope_dimension_count(hp["head_dim"])
|
||||
self.gguf_writer.add_rope_freq_base(hp["rope_theta"])
|
||||
self.gguf_writer.add_layer_norm_rms_eps(hp["norm_eps"])
|
||||
self.gguf_writer.add_full_attention_interval(self.gdn_period)
|
||||
|
||||
# gated delta net
|
||||
self.gguf_writer.add_ssm_conv_kernel(GDN_KERNEL)
|
||||
self.gguf_writer.add_ssm_inner_size(GDN_HEADS * GDN_HEAD_DIM)
|
||||
self.gguf_writer.add_ssm_state_size(GDN_HEAD_DIM)
|
||||
self.gguf_writer.add_ssm_time_step_rank(GDN_HEADS)
|
||||
self.gguf_writer.add_ssm_group_count(GDN_HEADS)
|
||||
|
||||
# latent MoE, the gating (softmax over the selected logits) is hard-coded in libllama
|
||||
self.gguf_writer.add_leading_dense_block_count(hp["moe_start_from_layer"])
|
||||
self.gguf_writer.add_expert_count(hp["moe_n_experts"])
|
||||
self.gguf_writer.add_expert_used_count(hp["moe_router_topk"])
|
||||
self.gguf_writer.add_expert_feed_forward_length(self.n_ff_exp)
|
||||
self.gguf_writer.add_moe_latent_size(self.n_latent)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# router logits pick the experts, keep them exact
|
||||
if new_name.endswith(("ffn_gate_inp.weight", "exp_probs_b.bias")):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def _token_embd(self) -> Tensor:
|
||||
n_cb = self.n_codebooks
|
||||
pad = sum(self._embd[f"multi_embedder.embedders.{cb}.weight"][self.audio_pad_id].float() for cb in range(n_cb))
|
||||
text = self._embd[f"multi_embedder.embedders.{n_cb}.weight"].float()
|
||||
if text.shape[0] != self.text_rows:
|
||||
raise ValueError(f"ZONOS3: text table has {text.shape[0]} rows, expected {self.text_rows}")
|
||||
conds = [self._embd[f"condition_embeddings.{i}.weight"].float() for i in range(len(self.cond_counts))]
|
||||
for t, n in zip(conds, self.cond_counts):
|
||||
assert t.shape[0] == n
|
||||
# a text row is [pad x n_codebooks, text_id], condition rows replace the summed row
|
||||
return torch.cat([text + pad] + conds, dim=0)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
T = gguf.MODEL_TENSOR
|
||||
hp = self.hparams
|
||||
n_layer = self.block_count
|
||||
|
||||
if name.startswith(("multi_embedder.", "condition_embeddings.")):
|
||||
self._embd[name] = data_torch
|
||||
if len(self._embd) == self.n_codebooks + 1 + len(self.cond_counts):
|
||||
yield (self.format_tensor_name(T.TOKEN_EMBD), self._token_embd())
|
||||
self._embd.clear()
|
||||
return
|
||||
|
||||
if name == "block_attnres_queries":
|
||||
# one query per routing point: 2*il before the mixer, 2*il+1 before the FFN, the last one before out_norm
|
||||
assert data_torch.shape[0] == 2 * n_layer + 1
|
||||
for il in range(n_layer):
|
||||
yield (self.format_tensor_name(T.ATTN_RES_SCORE, il), data_torch[2 * il])
|
||||
yield (self.format_tensor_name(T.FFN_RES_SCORE, il), data_torch[2 * il + 1])
|
||||
yield (self.format_tensor_name(T.OUTPUT_RES_SCORE), data_torch[2 * n_layer])
|
||||
return
|
||||
|
||||
if name == "out_norm.weight":
|
||||
yield (self.format_tensor_name(T.OUTPUT_NORM), data_torch)
|
||||
return
|
||||
|
||||
assert bid is not None, name
|
||||
key = name.split(".", 2)[2] # strip "layers.{bid}."
|
||||
|
||||
if key == "attention_norm.weight":
|
||||
yield (self.format_tensor_name(T.ATTN_NORM, bid), data_torch)
|
||||
return
|
||||
if key == "ffn_norm.weight":
|
||||
yield (self.format_tensor_name(T.FFN_NORM, bid), data_torch)
|
||||
return
|
||||
|
||||
# gated delta net
|
||||
if key == "attention.in_proj.weight":
|
||||
# split the fused [qkv | z | a | b] weight into the qwen35 GDN tensors
|
||||
# attn_gate holds z, per-head output gate in the attention layers
|
||||
inner = GDN_HEADS * GDN_HEAD_DIM
|
||||
qkv, z, a, b = data_torch.split([3 * inner, inner, GDN_HEADS, GDN_HEADS], dim=0)
|
||||
yield (self.format_tensor_name(T.ATTN_QKV, bid), qkv)
|
||||
yield (self.format_tensor_name(T.ATTN_GATE, bid), z)
|
||||
yield (self.format_tensor_name(T.SSM_ALPHA, bid), a)
|
||||
yield (self.format_tensor_name(T.SSM_BETA, bid), b)
|
||||
return
|
||||
if key == "attention.conv.weight":
|
||||
yield (self.format_tensor_name(T.SSM_CONV1D, bid), data_torch.squeeze(1))
|
||||
return
|
||||
if key == "attention.A_log":
|
||||
yield (self.format_tensor_name(T.SSM_A, bid, suffix=""), -torch.exp(data_torch.float()))
|
||||
return
|
||||
if key == "attention.dt_bias":
|
||||
yield (self.format_tensor_name(T.SSM_DT, bid, suffix=".bias"), data_torch)
|
||||
return
|
||||
if key == "attention.out_weight":
|
||||
yield (self.format_tensor_name(T.SSM_NORM, bid), data_torch)
|
||||
return
|
||||
if key == "attention.wo.weight" and (bid + 1) % self.gdn_period != 0:
|
||||
yield (self.format_tensor_name(T.SSM_OUT, bid), data_torch)
|
||||
return
|
||||
|
||||
# global attention
|
||||
if key == "attention.wq.weight":
|
||||
yield (self.format_tensor_name(T.ATTN_Q, bid), data_torch)
|
||||
return
|
||||
if key == "attention.wkv.weight":
|
||||
yield (self.format_tensor_name(T.ATTN_K, bid), data_torch[0])
|
||||
yield (self.format_tensor_name(T.ATTN_V, bid), data_torch[1])
|
||||
return
|
||||
if key == "attention.wo.weight":
|
||||
yield (self.format_tensor_name(T.ATTN_OUT, bid), data_torch)
|
||||
return
|
||||
if key == "attention.gater.weight":
|
||||
# one sigmoid gate per head, same tensor name as the GDN z gate
|
||||
yield (self.format_tensor_name(T.ATTN_GATE, bid), data_torch)
|
||||
return
|
||||
if key == "attention.temp":
|
||||
# Q = rms_norm(q) * |temp|, the per-head scale is stored as the Q norm weight
|
||||
temp = data_torch.float().abs().reshape(hp["n_heads"], 1).expand(hp["n_heads"], hp["head_dim"])
|
||||
yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), temp.contiguous())
|
||||
return
|
||||
|
||||
# dense FFN (layer 0): out = w_out(w_in[0](x) * silu(w_in[1](x)))
|
||||
if key == "feed_forward.w_in.weight":
|
||||
yield (self.format_tensor_name(T.FFN_UP, bid), data_torch[0])
|
||||
yield (self.format_tensor_name(T.FFN_GATE, bid), data_torch[1])
|
||||
return
|
||||
if key == "feed_forward.w_out.weight":
|
||||
yield (self.format_tensor_name(T.FFN_DOWN, bid), data_torch)
|
||||
return
|
||||
|
||||
# latent MoE
|
||||
if key in ("feed_forward.router.router_linear.weight", "feed_forward.router.rmsnorm_eda.weight"):
|
||||
# logits = router_linear(rms_norm(x) * w), the norm weight is folded into the router columns
|
||||
r = self._router.setdefault(bid, {})
|
||||
r[key] = data_torch
|
||||
if len(r) == 2:
|
||||
w = r["feed_forward.router.router_linear.weight"].float() * r["feed_forward.router.rmsnorm_eda.weight"].float()[None, :]
|
||||
yield (self.format_tensor_name(T.FFN_GATE_INP, bid), w)
|
||||
del self._router[bid]
|
||||
return
|
||||
if key == "feed_forward.router.balancing_biases":
|
||||
# the reference selects on logits - bias
|
||||
yield (self.format_tensor_name(T.FFN_EXP_PROBS_B, bid, suffix=".bias"), -data_torch.float())
|
||||
return
|
||||
if key == "feed_forward.latent_down.weight":
|
||||
yield (self.format_tensor_name(T.MOE_LATENT_DOWN, bid), data_torch)
|
||||
return
|
||||
if key == "feed_forward.latent_up.weight":
|
||||
yield (self.format_tensor_name(T.MOE_LATENT_UP, bid), data_torch)
|
||||
return
|
||||
if key == "feed_forward.experts.gate_up_proj":
|
||||
gate, up = data_torch.chunk(2, dim=1)
|
||||
yield (self.format_tensor_name(T.FFN_GATE_EXP, bid), gate)
|
||||
yield (self.format_tensor_name(T.FFN_UP_EXP, bid), up)
|
||||
return
|
||||
if key == "feed_forward.experts.down_proj":
|
||||
yield (self.format_tensor_name(T.FFN_DOWN_EXP, bid), data_torch)
|
||||
return
|
||||
|
||||
raise ValueError(f"ZONOS3: unexpected tensor {name}")
|
||||
|
||||
|
||||
@ModelBase.register("Zonos3Model")
|
||||
class Zonos3MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
hp = self.global_config["text_config"]
|
||||
self.n_codebooks = hp["n_codebooks"]
|
||||
self.text_vocab = hp["text_vocab"]
|
||||
self._embd: dict[str, Tensor] = {}
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
return _index_lm_tensors(self)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if name.startswith(("multi_output.", "multi_embedder.", "speaker_projection.")):
|
||||
return item
|
||||
return None
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
|
||||
# speaker encoder for voice cloning, the front-end hparams are hard-coded in tools/mtmd
|
||||
self.gguf_writer.add_clip_has_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.ZONOS3_SPKENC)
|
||||
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_audio_num_mel_bins(80)
|
||||
self.gguf_writer.add_audio_block_count(0)
|
||||
self.gguf_writer.add_audio_embedding_length(192)
|
||||
self.gguf_writer.add_audio_head_count(1)
|
||||
self.gguf_writer.add_audio_feed_forward_length(192)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
|
||||
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.ZONOS3_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(0)
|
||||
self.gguf_writer.add_gen_audio_block_count(0)
|
||||
self.gguf_writer.add_gen_audio_head_count(0)
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(self.global_config["text_config"]["norm_eps"])
|
||||
self.gguf_writer.add_gen_audio_model_variant(CODEC_VARIANT)
|
||||
if any(self.dir_model.glob(f"gguf/voices/*/{DEFAULT_VOICE}.codes.npy")):
|
||||
self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE)
|
||||
# softcap and the codec hparams are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# read on the host, or used by get_rows / ggml_ssm_conv / the conv ops of the speaker encoder
|
||||
if new_name.startswith(("voice.", "a.spk.")) or new_name.endswith((".table", ".dw.weight", ".mixer.weight", ".resample.weight")):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@staticmethod
|
||||
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
|
||||
return (g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))).squeeze(-1)
|
||||
|
||||
def _speaker_encoder(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# same layout as the reference ggml runtime (tools/convert_gguf.py speaker-encoder)
|
||||
sd = torch.load(self.dir_model / SPK_CKPT, map_location="cpu", weights_only=True)["modules"]["spk_model"]
|
||||
sd = {k[len("model."):]: v.double() for k, v in sd.items() if not k.endswith("num_batches_tracked")}
|
||||
p = "a.spk"
|
||||
|
||||
def bn(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
scale = sd[src + ".weight"] / torch.sqrt(sd[src + ".running_var"] + 1e-5)
|
||||
yield (f"{p}.{dst}.scale", scale)
|
||||
yield (f"{p}.{dst}.shift", sd[src + ".bias"] - sd[src + ".running_mean"] * scale)
|
||||
|
||||
def put(dst: str, t: Tensor) -> tuple[str, Tensor]:
|
||||
return (f"{p}.{dst}", t)
|
||||
|
||||
fe = "feature_extractor."
|
||||
yield put("fe.head.conv.weight", sd[fe + "head.0.weight"])
|
||||
yield put("fe.head.conv.bias", sd[fe + "head.0.bias"])
|
||||
yield from bn("fe.head.bn", fe + "head.1")
|
||||
c_in, f_in = 32, 80
|
||||
for si, stage in enumerate(SPK_STAGES):
|
||||
for k, (depth, out_ch, (s_f, s_t), expansion) in enumerate(stage):
|
||||
src, dst = f"{fe}mect_block{si + 1}.{k}.", f"fe.b{si + 1}.{k}."
|
||||
for r in range(depth):
|
||||
cs, cd = f"{src}cnn_block.{r}.", f"{dst}cnn.{r}."
|
||||
# grouped 3x3 convs, one depthwise kernel per input slot of a group
|
||||
for conv in ("conv1", "conv2"):
|
||||
w = sd[cs + conv + ".weight"]
|
||||
for j in range(w.shape[1]):
|
||||
yield put(f"{cd}{conv}.w{j}", w[:, j:j + 1])
|
||||
for i in (1, 2):
|
||||
yield put(f"{cd}pw{i}.weight", sd[f"{cs}conv_pw{i}.weight"])
|
||||
yield put(f"{cd}pw{i}.bias", sd[f"{cs}conv_pw{i}.bias"])
|
||||
yield from bn(f"{cd}bn{i}", f"{cs}bn{i}")
|
||||
if cs + "shortcut.0.weight" in sd:
|
||||
yield put(cd + "sc.weight", sd[cs + "shortcut.0.weight"])
|
||||
yield from bn(cd + "sc.bn", cs + "shortcut.1")
|
||||
if expansion > 1:
|
||||
yield put(dst + "reduce.weight", sd[src + "reduce_conv.0.weight"])
|
||||
yield put(dst + "reduce.bias", sd[src + "reduce_conv.0.bias"])
|
||||
yield from bn(dst + "reduce.bn", src + "reduce_conv.1")
|
||||
yield put(dst + "rproj.weight", sd[src + "reduce_project.0.weight"].squeeze(-1))
|
||||
yield put(dst + "rproj.bias", sd[src + "reduce_project.0.bias"])
|
||||
yield put(dst + "rproj.ln.weight", sd[src + "reduce_project.1.weight"])
|
||||
yield put(dst + "rproj.ln.bias", sd[src + "reduce_project.1.bias"])
|
||||
yield put(dst + "pos.dw.weight", sd[src + "pos_conv.pos_conv.0.weight"].squeeze(1))
|
||||
yield put(dst + "pos.dw.bias", sd[src + "pos_conv.pos_conv.0.bias"])
|
||||
yield from bn(dst + "pos.bn", src + "pos_conv.pos_conv.1")
|
||||
yield put(dst + "pos.pw.weight", sd[src + "pos_conv.pos_conv.3.weight"].squeeze(-1))
|
||||
yield put(dst + "pos.pw.bias", sd[src + "pos_conv.pos_conv.3.bias"])
|
||||
tr = src + "transformer."
|
||||
for a, b in (("q", "q_proj"), ("k", "k_proj"), ("v", "v_proj"), ("o", "out_proj")):
|
||||
yield put(f"{dst}attn.{a}.weight", sd[f"{tr}attention.{b}.weight"])
|
||||
yield put(f"{dst}attn.{a}.bias", sd[f"{tr}attention.{b}.bias"])
|
||||
yield put(dst + "ln1.weight", sd[tr + "layer_norm.weight"])
|
||||
yield put(dst + "ln1.bias", sd[tr + "layer_norm.bias"])
|
||||
moe = tr + "feed_forward.intermediate_dense.token_moe."
|
||||
yield put(dst + "moe.gate.weight", sd[moe + "gate.weight"])
|
||||
yield put(dst + "moe.experts.weight", sd[moe + "experts.weight"])
|
||||
yield put(dst + "ffn_out.weight", sd[tr + "feed_forward.output_dense.weight"])
|
||||
yield put(dst + "ffn_out.bias", sd[tr + "feed_forward.output_dense.bias"])
|
||||
yield put(dst + "ln2.weight", sd[tr + "final_layer_norm.weight"])
|
||||
yield put(dst + "ln2.bias", sd[tr + "final_layer_norm.bias"])
|
||||
yield put(dst + "eproj.weight", sd[src + "expansion_project.0.weight"].squeeze(-1))
|
||||
yield put(dst + "eproj.bias", sd[src + "expansion_project.0.bias"])
|
||||
c_in, f_in = c_in * s_f, f_in // s_f
|
||||
# the stage outputs are mixed by softmax weights
|
||||
yield put("fe.dense_w", torch.softmax(sd[fe + "dense3.inputs_weights"], dim=1)[0, :, :, 0])
|
||||
yield put("pool.l1.weight", sd["encoding_layers.linear1.weight"].squeeze(-1))
|
||||
yield put("pool.l1.bias", sd["encoding_layers.linear1.bias"])
|
||||
yield put("pool.l2.weight", sd["encoding_layers.linear2.weight"].squeeze(-1))
|
||||
yield put("pool.l2.bias", sd["encoding_layers.linear2.bias"])
|
||||
yield from bn("bn", "bn")
|
||||
yield put("linear.weight", sd["linear.weight"])
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
yield from self._speaker_encoder()
|
||||
logger.info(f"gguf: loading codec decoder from '{CODEC_GENERATOR}' and '{CODEC_HEAD}'")
|
||||
gen = torch.load(self.dir_model / CODEC_GENERATOR, map_location="cpu", mmap=True, weights_only=True)["generator"]
|
||||
head = torch.load(self.dir_model / CODEC_HEAD, map_location="cpu", mmap=True, weights_only=True)["trainable"]
|
||||
p = "a.gen.wav"
|
||||
|
||||
def convnext(dst: str, src: dict[str, Tensor], prefix: str) -> Iterable[tuple[str, Tensor]]:
|
||||
yield (f"{dst}.dw.weight", src[f"{prefix}.dwconv.weight"].squeeze(1))
|
||||
yield (f"{dst}.dw.bias", src[f"{prefix}.dwconv.bias"])
|
||||
yield (f"{dst}.norm.weight", src[f"{prefix}.norm.weight"])
|
||||
yield (f"{dst}.norm.bias", src[f"{prefix}.norm.bias"])
|
||||
yield (f"{dst}.pw1.weight", src[f"{prefix}.pwconv1.weight"])
|
||||
yield (f"{dst}.pw1.bias", src[f"{prefix}.pwconv1.bias"])
|
||||
yield (f"{dst}.pw2.weight", src[f"{prefix}.pwconv2.weight"])
|
||||
yield (f"{dst}.pw2.bias", src[f"{prefix}.pwconv2.bias"])
|
||||
|
||||
# RVQ codebooks folded through their out_proj, the biases are added once
|
||||
def rvq(prefix: str) -> tuple[Tensor, Tensor]:
|
||||
return gen[prefix + ".codebook.weight"].float() @ self._wn(gen, prefix + ".out_proj").T, gen[prefix + ".out_proj.bias"].float()
|
||||
|
||||
t, b = rvq("quantizer.semantic_quantizer.quantizers.0")
|
||||
yield (f"{p}.sem.table", t)
|
||||
yield (f"{p}.sem.bias", b)
|
||||
res = [rvq(f"quantizer.quantizer.quantizers.{i}") for i in range(self.n_codebooks - 1)]
|
||||
yield (f"{p}.res.table", torch.cat([t for t, _ in res], dim=0))
|
||||
# one bias per residual codebook, a frame decoded before all its codes are known leaves the missing ones out
|
||||
yield (f"{p}.res.bias", torch.stack([b for _, b in res]))
|
||||
|
||||
n_sem = sum(1 for k in gen if k.startswith("semantic_head.") and k.endswith(".dwconv.weight"))
|
||||
for i in range(n_sem):
|
||||
yield from convnext(f"{p}.sem.{i}", gen, f"semantic_head.{i}")
|
||||
yield (f"{p}.sem.out.weight", self._wn(gen, f"semantic_head.{n_sem}"))
|
||||
yield (f"{p}.sem.out.bias", gen[f"semantic_head.{n_sem}.bias"])
|
||||
|
||||
names = {"norm1.weight": "norm1.weight", "norm1.bias": "norm1.bias", "qkv.weight": "qkv.weight",
|
||||
"attention_out.weight": "wo.weight", "attention_scale": "attn_scale", "norm2.weight": "norm2.weight",
|
||||
"norm2.bias": "norm2.bias", "ffn.0.weight": "ffn1.weight", "ffn.3.weight": "ffn2.weight", "ffn_scale": "ffn_scale"}
|
||||
for si, s in enumerate(CODEC_STAGES):
|
||||
yield (f"{p}.stage.{si}.in.weight", head[f"stages.{s}.input_proj.weight"])
|
||||
if f"stages.{s}.output_proj.weight" in head:
|
||||
yield (f"{p}.stage.{si}.out.weight", head[f"stages.{s}.output_proj.weight"])
|
||||
n_layer = sum(1 for k in head if k.startswith(f"stages.{s}.layers.") and k.endswith(".qkv.weight"))
|
||||
for il in range(n_layer):
|
||||
for src, dst in names.items():
|
||||
yield (f"{p}.stage.{si}.blk.{il}.{dst}", head[f"stages.{s}.layers.{il}.{src}"])
|
||||
|
||||
n_head = sum(1 for k in head if k.startswith("head.blocks.") and k.endswith(".dwconv.weight"))
|
||||
for i in range(n_head):
|
||||
yield from convnext(f"{p}.head.{i}", head, f"head.blocks.{i}")
|
||||
yield (f"{p}.head.norm.weight", head["head.norm.weight"])
|
||||
yield (f"{p}.head.norm.bias", head["head.norm.bias"])
|
||||
# both iSTFT branches in one projection
|
||||
yield (f"{p}.head.out.weight", torch.cat([head["head.outs.0.weight"], head["head.outs.1.weight"]], dim=0))
|
||||
yield (f"{p}.head.out.bias", torch.cat([head["head.outs.0.bias"], head["head.outs.1.bias"]], dim=0))
|
||||
# read on the host
|
||||
yield (f"{p}.mixer.weight", head["head.mixer.weight"].squeeze(0))
|
||||
yield (f"{p}.resample.weight", gen["to_external.kernel"].squeeze(1))
|
||||
|
||||
# "reference" speaker mode, the default of the reference: the voice is a prefix of delayed code rows
|
||||
def _voices(self, tables: list[Tensor]) -> Iterable[tuple[str, Tensor]]:
|
||||
n_cb, pad = self.n_codebooks, self.global_config["text_config"]["audio_pad_id"]
|
||||
for path in sorted((self.dir_model / "gguf" / "voices").glob("*/*.codes.npy")):
|
||||
codes = torch.from_numpy(np.load(path, allow_pickle=False).astype(np.int64))
|
||||
if codes.ndim != 2 or codes.shape[1] != n_cb or codes.min() < 0 or codes.max() >= pad - 1:
|
||||
logger.warning(f"voice {path.name}: skipped, expected [frames, {n_cb}] codes")
|
||||
continue
|
||||
codes = codes[:VOICE_MAX_FRAMES]
|
||||
n_frames = codes.shape[0]
|
||||
# codebook cb of frame t is in row t + cb, the other cells are pad
|
||||
rows = torch.full((n_frames + n_cb - 1, n_cb), pad, dtype=torch.int64)
|
||||
for cb in range(n_cb):
|
||||
rows[cb:cb + n_frames, cb] = codes[:, cb]
|
||||
embd = sum(tables[cb][rows[:, cb]] for cb in range(n_cb))
|
||||
yield (f"voice.{path.name[:-len('.codes.npy')]}", embd)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
if name.startswith(("a.gen.wav.", "a.spk.")):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
# the speaker encoder ends with the speaker projection of the backbone
|
||||
if name.startswith("speaker_projection."):
|
||||
yield ("a.spk.proj." + name.split(".")[-1], data_torch)
|
||||
return
|
||||
|
||||
if name == "multi_output.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch)
|
||||
return
|
||||
|
||||
assert name.startswith("multi_embedder.")
|
||||
self._embd[name] = data_torch
|
||||
if len(self._embd) < self.n_codebooks + 1:
|
||||
return
|
||||
# every row built by the mmproj has text id = text_vocab, so that row is folded into codebook 0
|
||||
tables = [self._embd[f"multi_embedder.embedders.{cb}.weight"].float() for cb in range(self.n_codebooks)]
|
||||
tables[0] = tables[0] + self._embd[f"multi_embedder.embedders.{self.n_codebooks}.weight"][self.text_vocab].float()
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), torch.cat(tables, dim=0))
|
||||
yield from self._voices([LazyTorchTensor.to_eager(t) for t in tables])
|
||||
self._embd.clear()
|
||||
@@ -167,6 +167,7 @@ models = [
|
||||
{"name": "ufakzeka", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ufakai/ufakzeka-1", },
|
||||
{"name": "mmbert", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jhu-clsp/mmBERT-base", },
|
||||
{"name": "k2-horizon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/IFM/K2-Horizon-36B", },
|
||||
{"name": "inkling2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/thinkingmachines/Inkling-2-Small", },
|
||||
]
|
||||
|
||||
# some models are known to be broken upstream, so we will skip them as exceptions
|
||||
|
||||
@@ -692,7 +692,7 @@ llama-openvino:server --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_
|
||||
# Or Using llama-server executable
|
||||
./build/ReleaseOV/bin/llama-server -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --port 8080 -c 1024
|
||||
|
||||
# Option 1: Open your browser to http://localhost:8080 to access the web UI for the llama.cpp server.
|
||||
# Option 1: Open your browser to http://localhost:8080 to open Llama Chat, the built-in chat app of the llama.cpp server.
|
||||
# Option 2: In a NEW terminal, test the server with curl
|
||||
|
||||
# If you are behind a proxy, make sure to set NO_PROXY to avoid proxy for localhost
|
||||
|
||||
@@ -0,0 +1,354 @@
|
||||
# Text-to-speech (TTS)
|
||||
|
||||
llama.cpp supports audio generation via `libmtmd`. The following tools can be used:
|
||||
- [llama-tts](../../tools/tts/README.md), see its README for common usage and params
|
||||
- [llama-server](../../tools/server/README.md) via the `/tts` endpoint and the OpenAI-compatible `/v1/audio/speech` API
|
||||
|
||||
Supported models:
|
||||
- [Qwen3-TTS](#qwen3-tts)
|
||||
- [Pocket TTS](#pocket-tts)
|
||||
- [Soprano](#soprano)
|
||||
- [LFM3-Audio](#lfm3-audio)
|
||||
- [Chatterbox](#chatterbox)
|
||||
- [Drama 3](#drama-3)
|
||||
- [ZONOS3](#zonos3)
|
||||
- [Fish Audio S2 Pro](#fish-audio-s2-pro)
|
||||
- [YuE2](#yue2) (songs)
|
||||
- [MiniMax Music 3](#minimax-music-3) (songs)
|
||||
|
||||
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
|
||||
|
||||
## Qwen3-TTS
|
||||
|
||||
Available params:
|
||||
- `--tts-lang` can be `zh`, `en`, `de`, `it`, `pt`, `es`, `ja`, `ko`, `fr`, `ru` (default: `en`)
|
||||
- `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3)
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \
|
||||
-p "Hello world" \
|
||||
--tts-lang english \
|
||||
--tts-speaker-file speaker.mp3 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
## Pocket TTS
|
||||
|
||||
Available params:
|
||||
- `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3). It is required, the model produces almost no audio without it
|
||||
- `--tts-voice` selects one of the predefined voices shipped with the weights (e.g. `giovanni`), when the GGUF holds them. The converter reads them from `embeddings/*.safetensors` next to the weights, of both repositories, and logs how closely each one is reproduced (some packs ship voice states computed with other weights, the converter warns above 5% and skips a voice only when it does not belong to the model at all)
|
||||
- Note: `lang` is not used, the language is a property of the weights
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m pocket-tts.gguf \
|
||||
-mm mmproj-pocket-tts.gguf \
|
||||
-p "Hello world" \
|
||||
--tts-speaker-file speaker.mp3 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
The weights [without voice cloning](https://huggingface.co/kyutai/pocket-tts-without-voice-cloning) ship a set of predefined voices (`embeddings/*.safetensors`) next to each language; the converter stores them in the mmproj, as the voice prompt embeddings, and leaves out the zeroed voice encoder. With such a model `--tts-speaker-file` is rejected and a voice is selected with `--tts-voice` (the first one when it is not given):
|
||||
|
||||
```sh
|
||||
llama-tts -m pocket-tts.gguf \
|
||||
-mm mmproj-pocket-tts.gguf \
|
||||
-p "Ciao a tutti" \
|
||||
--tts-voice giovanni \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The [upstream repository](https://huggingface.co/kyutai/pocket-tts) holds one complete model per language under `languages/`, next to a set of shared files at the root. Convert one of the `languages/<name>` directories, **not** the root directory:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/pocket-tts/languages/english --outfile pocket-tts.gguf
|
||||
python convert_hf_to_gguf.py path/to/pocket-tts/languages/english --mmproj --outfile mmproj-pocket-tts.gguf
|
||||
```
|
||||
|
||||
## Soprano
|
||||
|
||||
Available params:
|
||||
- `--tts-speaker-file` is not supported; the model uses a fixed voice
|
||||
- Note: `lang` is not used. Supply normalized English text
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m soprano.gguf \
|
||||
-mm mmproj-soprano.gguf \
|
||||
-p "Hello world" \
|
||||
--temp 0 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
Both [Soprano-80M](https://huggingface.co/ekwek/Soprano-80M) and [Soprano-1.1-80M](https://huggingface.co/ekwek/Soprano-1.1-80M) are supported. Convert both files from the same model directory, including `decoder.pth`:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --outtype f32 --outfile soprano.gguf
|
||||
python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --mmproj --outtype f16 --outfile mmproj-soprano.gguf
|
||||
```
|
||||
|
||||
## LFM3-Audio
|
||||
|
||||
Available params:
|
||||
- `--tts-lang` can be `ar`, `da`, `de`, `el`, `en`, `en-gb`, `es`, `fi`, `fr`, `he`, `it`, `ja`, `ko`, `nl`, `pl`, `pt`, `sv`, `tr`, `zh` (default: `en`), other values are used as the language name
|
||||
- `--tts-voice` can be `female` (default) or `male`
|
||||
- Note: `--tts-speaker-file` is not used, the model has no voice cloning
|
||||
- Note: `--top-k` and `--top-p` are not used, audio frames are sampled with the settings of the model card (LFM3-Audio: temperature 1.0, top-k 128, repetition penalty 1.1 over 4096 frames; LFM3-Omni: temperature 0.4, repetition penalty 1.2 over 64 frames), the `audio_*` keys of `generation_config.json` override them at conversion
|
||||
- Each frame is 80 ms, generation stops at the end-of-audio frame
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m lfm3-audio.gguf \
|
||||
-mm mmproj-lfm3-audio.gguf \
|
||||
-p "Hello world" \
|
||||
--tts-voice male \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
The mmproj holds the audio encoder (for ASR), the depthformer and the codec decoder, the latter two are exported when the checkpoint has an `audio_tokenizer` directory:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/lfm3-audio --outfile lfm3-audio.gguf
|
||||
python convert_hf_to_gguf.py path/to/lfm3-audio --mmproj --outfile mmproj-lfm3-audio.gguf
|
||||
```
|
||||
|
||||
## Chatterbox
|
||||
|
||||
Two variants are supported: [Chatterbox Turbo](https://huggingface.co/ResembleAI/chatterbox-turbo) (English) and [Chatterbox Multilingual V3](https://huggingface.co/ResembleAI/chatterbox) (23 languages).
|
||||
|
||||
Available params:
|
||||
- `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3). Optional, the predefined voice is used without it
|
||||
- `--tts-voice` can be `default`, the voice shipped with the model (`conds.pt`)
|
||||
- `--tts-lang` sets the language of the multilingual variant, for example `fr` (default: none). It is not used by Turbo
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m chatterbox-mtl.gguf \
|
||||
-mm mmproj-chatterbox-mtl.gguf \
|
||||
-p "Bonjour tout le monde" \
|
||||
--tts-lang fr \
|
||||
--tts-speaker-file speaker.mp3 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Note: the multilingual variant runs classifier-free guidance with a second sequence. `llama-tts` sets this up by itself, but `llama-server` does not support it yet.
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The upstream repositories ship raw checkpoints without `config.json`. Download the files below into a directory, then convert it as usual (the variant is detected from the talker file):
|
||||
- Turbo: `t3_turbo_v1.safetensors`, `s3gen_meanflow.safetensors`, `ve.safetensors`, `conds.pt` and the GPT-2 tokenizer files (`vocab.json`, `merges.txt`, `added_tokens.json`, `special_tokens_map.json`, `tokenizer_config.json`)
|
||||
- Multilingual V3: `t3_mtl23ls_v3.safetensors`, `s3gen_v3.safetensors`, `ve.safetensors`, `conds.pt`, `mtl_tokenizer.json`
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/chatterbox --outfile chatterbox.gguf
|
||||
python convert_hf_to_gguf.py path/to/chatterbox --mmproj --outfile mmproj-chatterbox.gguf
|
||||
```
|
||||
|
||||
## Drama 3
|
||||
|
||||
[Fish Audio Drama 3](https://huggingface.co/fishaudio/drama-3): a Qwen3.5-MoE backbone (35B, 3B active) samples each frame's main token, a codebook tower samples the 9 other codes, and the Fish DAC decodes them to 44.1 kHz audio.
|
||||
|
||||
Available params:
|
||||
- Speaker tags `<|speaker:0|>` to `<|speaker:15|>` in the text give each character its own voice, a text without a tag starts with `<|speaker:0|>`
|
||||
- The language is the one of the text, `--tts-lang` is not used
|
||||
- Note: `--tts-speaker-file` is not supported yet (voice cloning)
|
||||
- Note: `--top-k` and `--top-p` are not used, frames are sampled with the settings of the reference (main token: temperature 1.0, top-p 0.9, repetition-aware redraw over the last 16 frames; other codes: temperature 1.0, top-k 30, top-p 0.9)
|
||||
- Each frame is 2048 samples (about 46 ms), generation stops at the end-of-speech token
|
||||
- The codebook tower keeps the whole sequence in its KV cache, the prompt and the generated frames are limited to 4096 positions
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m drama3.gguf \
|
||||
-mm mmproj-drama3.gguf \
|
||||
-p "<|speaker:0|> Did you remember to buy the bread? <|speaker:1|> Of course I did, it is on the kitchen table." \
|
||||
-n 1000 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The routed experts of the checkpoint are FP8, they are dequantized at conversion. The mmproj holds the codebook tower and the decoder half of `codec.pth`, it is best kept in BF16:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/drama-3 --outtype q8_0 --outfile drama3.gguf
|
||||
python convert_hf_to_gguf.py path/to/drama-3 --mmproj --outtype bf16 --outfile mmproj-drama3.gguf
|
||||
```
|
||||
|
||||
## ZONOS3
|
||||
|
||||
[Zyphra ZONOS3 Mini](https://huggingface.co/Zyphra/ZONOS3-Mini): a hybrid Gated DeltaNet / attention MoE backbone (800M, 100M active) predicts the 10 codebooks of each frame with an inter-codebook delay, and the ZAC codec decodes them to 44.1 kHz audio.
|
||||
|
||||
Available params:
|
||||
- `--tts-voice` picks one of the predefined voices of the release (`Savannah_en-American_F_1` by default); `llama-server` lists them in `GET /props`. A voice is the prompt prefix of its reference codes, the default speaker mode of the reference
|
||||
- The language is the one of the text; `--tts-lang` (`lang` in `llama-server`) only selects the text normalization
|
||||
- Numbers are spelled out by the `tts` chat template of the GGUF: cardinals, decimals, money, percents, clock times, common ordinal forms (`1st`, `1er`, `1º`), English and German years. A 4-digit number is read as a year only after a word like `in`, `since`, a month name, or `im Jahr`, `seit` in German (`in 1990` -> "in nineteen ninety", `1234 people` -> "one thousand two hundred thirty-four people"). Languages: `en`, `fr`, `de`, `es`, `it`, `pt`, `ja` and `zh` (or `cmn`), the region is ignored (`fr_fr`, `pt_br`); English when the language is empty, other languages are left as written. Set the language to the one of the text: the separators differ, e.g. `1.234` is a thousand in German
|
||||
- Note: the reference uses NeMo text normalization, which also covers units, dates, abbreviations and Korean; supply normalized text for them
|
||||
- `--tts-speaker-file` clones the voice of a clip (the first 60 s are used) with the speaker encoder of the release, the "embedding" speaker mode of the reference
|
||||
- `llama-server` also takes `speed` (relative to 15 UTF-8 bytes per second) and `loudness` (LUFS), see the `/tts` endpoint; they pick the speaking-rate and loudness condition buckets of the checkpoint, the defaults leave them unknown
|
||||
- Note: `loudness` has a strong effect with a cloned voice; a predefined voice mostly keeps the loudness of its reference clip
|
||||
- Note: `--top-k` and `--top-p` are not used, frames are sampled with the settings of the reference: temperature 0.75 (`--temp` changes it), top-k 80, top-p 0.69, repetition penalty 2.4 over the last 96 frames of codebooks 0 to 5
|
||||
- Each frame is 2940 samples (about 67 ms), generation stops at the end-of-audio code
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m zonos3-mini.gguf \
|
||||
-mm mmproj-zonos3-mini.gguf \
|
||||
-p "The old lighthouse keeper climbed the spiral stairs every evening." \
|
||||
--tts-voice Imogen_en-British_F_1 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
Convert the root of the release repository, not the `lm/` subdirectory. The mmproj holds the codebook heads, the predefined voices, the speaker encoder and the `keep2_plain` codec decoder, it is best kept in F32:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/ZONOS3-Mini --outtype q8_0 --outfile zonos3-mini.gguf
|
||||
python convert_hf_to_gguf.py path/to/ZONOS3-Mini --mmproj --outtype f32 --outfile mmproj-zonos3-mini.gguf
|
||||
```
|
||||
|
||||
## Fish Audio S2 Pro
|
||||
|
||||
[Fish Audio S2 Pro](https://huggingface.co/fishaudio/s2-pro): a Qwen3 backbone (4B) samples each frame's semantic token, a 4-layer fast AR predicts the 9 other codes, and the Fish DAC (same as Drama 3) decodes them to 44.1 kHz audio.
|
||||
|
||||
Available params:
|
||||
- Speaker tags `<|speaker:0|>`, `<|speaker:1|>`, ... in the text give each character its own voice, a text without a tag starts with `<|speaker:0|>`
|
||||
- Inline instructions in brackets control the delivery, for example `[laughing]`, `[whisper]` or `[pause]`
|
||||
- The language is the one of the text, `--tts-lang` is not used
|
||||
- Voice cloning: `--tts-speaker-file` with the reference clip, optionally `--tts-speaker-text` with its transcript (it is put in the prompt as the reference does). The clip is encoded by the codec encoder half of `codec.pth`, it is truncated to the first 30 s
|
||||
- Note: `--temp`, `--top-k` and `--top-p` are not used (only `--seed`), frames are sampled with the settings of the reference (semantic token: repetition penalty 1.1 over the last 16 frames, top-k 30, top-p 0.8, temperature 0.8, or top-p 0.9 and temperature 1.0 when the last 4 frames repeat a token; other codes: greedy)
|
||||
- Each frame is 2048 samples (about 46 ms), generation stops at the end-of-speech token
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m s2-pro.gguf \
|
||||
-mm mmproj-s2-pro.gguf \
|
||||
-p "<|speaker:0|>Did you remember to buy the bread? <|speaker:1|>Of course I did [laughing] it is on the kitchen table." \
|
||||
-n 1000 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Voice cloning (`--tts-speaker-text` can be left out):
|
||||
|
||||
```sh
|
||||
llama-tts -m s2-pro.gguf \
|
||||
-mm mmproj-s2-pro.gguf \
|
||||
--tts-speaker-file reference.wav \
|
||||
--tts-speaker-text "The words spoken in reference.wav." \
|
||||
-p "Hello, this is a cloned voice." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The mmproj holds the fast AR and `codec.pth` (the encoder half for voice cloning, the decoder half for the audio), it is best kept in BF16. The codec encoder is always stored in F16/F32, small errors in it change the codes picked by its VQ:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/s2-pro --outtype q8_0 --outfile s2-pro.gguf
|
||||
python convert_hf_to_gguf.py path/to/s2-pro --mmproj --outtype bf16 --outfile mmproj-s2-pro.gguf
|
||||
```
|
||||
|
||||
## YuE2
|
||||
|
||||
[m-a-p/YuE2-3B](https://huggingface.co/m-a-p/YuE2-3B) generates songs (vocals and accompaniment) from a style prompt and lyrics, as 48 kHz stereo. The backbone (a Qwen3 text model) writes an ABC score, then one semantic code per 40 ms frame; the NAR half of the checkpoint turns the codes into VAE latents by flow matching, attending to the keys and values of the backbone, and the [YuE2-Vae](https://huggingface.co/m-a-p/YuE2-Vae) decoder makes the audio.
|
||||
|
||||
The prompt is the text of the reference request:
|
||||
|
||||
```
|
||||
[Tags]
|
||||
City Pop, upbeat, danceable, groovy bass, electric guitar, synth
|
||||
[Lyrics]
|
||||
[Verse]
|
||||
...
|
||||
```
|
||||
|
||||
Available params:
|
||||
- `--tts-voice` picks the planning mode of the reference (`cot`): `full` (default, a chord-annotated score), `melody` (a melody-only score) or `off` (no score)
|
||||
- Cover mode: append the score to the prompt as `\n[ABC]\n<score>`, with `--tts-voice melody` or `full`; the model then goes straight to the codes
|
||||
- The generation stops at the limits of the reference (4096 score tokens, then 9000 codes, 6 minutes); `-n` sets a lower limit on the score tokens and the codes together
|
||||
- The context must keep the default size (24576 positions, the one of the reference): a smaller `-c` is rejected when the model is loaded
|
||||
- Note: `--top-k` and `--top-p` are not used, both phases are sampled with the settings of the reference (`yue2_generation_config.json`); `--temp` changes the temperature of the codes
|
||||
- Note: the backbone context must use an F16 KV cache (default): the NAR pass loads its keys and values
|
||||
- The codes are sampled with classifier-free guidance like the reference: scale 1.01 with `--tts-voice off`, off (1.0) otherwise. `llama-server` takes `cfg_scale` to change it, the reference suggests 1.2 for a stronger text guidance. The uncond prompt runs as a second sequence, so the context must also hold it (the instruction, the score and the codes once more)
|
||||
- The NAR pass solves the flow-matching ODE in 32 midpoint steps like the reference; `llama-server` takes `flow_steps` to change it. The NAR time is about proportional to it: 16 steps are about 35% faster for the NAR and the VAE together, with a small deviation from the reference (about 0.2 dB of spectral difference); it grows with fewer steps (about 0.4 dB at 8, 0.8 dB at 4)
|
||||
- The NAR pass runs on chunks of codes like the reference: one chunk for a song that fits in 24576 positions, so the audio arrives after the last code. When streaming, each 30 s of codes is a chunk, processed during the generation, so the audio arrives in 30 s steps; the chunks are independent (each sees the prompt, the score and its own codes), which may be audible at the boundaries
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m yue2-3b.gguf \
|
||||
-mm mmproj-yue2-3b.gguf \
|
||||
-f song.txt \
|
||||
--output song.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The VAE is a separate repository, download it into the `vae` subdirectory of the model:
|
||||
|
||||
```sh
|
||||
hf download m-a-p/YuE2-3B --local-dir YuE2-3B
|
||||
hf download m-a-p/YuE2-Vae --local-dir YuE2-3B/vae
|
||||
python convert_hf_to_gguf.py YuE2-3B --outfile yue2-3b.gguf
|
||||
python convert_hf_to_gguf.py YuE2-3B --mmproj --outfile mmproj-yue2-3b.gguf
|
||||
```
|
||||
|
||||
The mmproj holds the NAR transformer layers (loaded with libllama as a Qwen3 text model, in the `--outtype` type), and the latent projections and the VAE decoder, always in F32.
|
||||
|
||||
## MiniMax Music 3
|
||||
|
||||
[MiniMaxAI/MiniMax-Music3](https://huggingface.co/MiniMaxAI/MiniMax-Music3) generates songs (vocals and accompaniment) from a music description and lyrics, as 44.1 kHz stereo. For each 40 ms frame, the backbone (a Qwen3 text model) samples the semantic code and the RVQ depth decoder the 7 other codes; the hidden states of both go to a flow-matching DiT, then a DAC decoder makes the audio. Both use the settings of the reference (the diffusers pipeline).
|
||||
|
||||
The prompt has the same format as YuE2, the description goes under `[Tags]`:
|
||||
|
||||
```
|
||||
[Tags]
|
||||
Genre: acoustic pop. BPM: 96. Key: C major. Vocals: soft female lead. Arrangement: fingerpicked guitar and soft piano.
|
||||
[Lyrics]
|
||||
[verse]
|
||||
Morning light filtering through the pine
|
||||
[chorus]
|
||||
Softly the world begins to breathe
|
||||
```
|
||||
|
||||
Available params:
|
||||
- The description and the lyrics are cleaned up like the reference: markdown is removed, a line that starts with structure tags (e.g. `[Verse]`) keeps only the tags
|
||||
- The generation stops at the end-of-audio token or at the limit of the reference (9000 frames, 6 minutes); `-n` sets a lower limit in frames (25 per second)
|
||||
- `--temp` changes the temperature of the codes (default 1.0); `--top-k` and `--top-p` are not used, the codes are sampled with top-k 50 like the reference
|
||||
- The codes are sampled with classifier-free guidance, scale 1.5 like the reference; `llama-server` takes `cfg_scale` to change it (1.0 disables it). The uncond prompt runs as a second sequence that shares the context, so a song needs about 2 x (prompt tokens + frames) positions: the default context (10240) holds about 3 minutes with a short description, use a larger `-c` (e.g. 20480) for longer songs
|
||||
- The flow matching runs 30 Euler steps like the reference; `llama-server` takes `flow_steps` to change it
|
||||
- The audio is made in windows of 200 frames (8 s) every 100 frames, like the reference: a window is processed during the generation once the frame after it exists, so with streaming the audio arrives in steps of about 4 s
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m minimax-music3.gguf \
|
||||
-mm mmproj-minimax-music3.gguf \
|
||||
-f song.txt \
|
||||
--output song.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The conversion reads the diffusers layout of the repository, the `qwen_7B` directory and the `.pth` files are not needed:
|
||||
|
||||
```sh
|
||||
hf download MiniMaxAI/MiniMax-Music3 --local-dir MiniMax-Music3 --exclude "qwen_7B/*" --exclude "*.pth"
|
||||
python convert_hf_to_gguf.py MiniMax-Music3 --outfile minimax-music3.gguf
|
||||
python convert_hf_to_gguf.py MiniMax-Music3 --mmproj --outfile mmproj-minimax-music3.gguf
|
||||
```
|
||||
|
||||
The mmproj holds the depth decoder and the DiT layers (in the `--outtype` type), and the condition encoder, the small DiT projections and the DAC decoder, always in F32.
|
||||
+122
-119
@@ -12,122 +12,125 @@ Legend:
|
||||
- 🟡 Partially supported by this backend
|
||||
- ❌ Not supported by this backend
|
||||
|
||||
| Operation | BLAS | CANN | CPU | CUDA | ET | HTP | MTL | OpenCL | SYCL | Vulkan | WebGPU | ZenDNN | zDNN |
|
||||
|-----------|------|------|------|------|------|------|------|------|------|------|------|------|------|
|
||||
| ABS | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ACC | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ADD1 | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| ADD_ID | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CLAMP | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| COL2IM_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONCAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| CONT | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| CONV_1D_DW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ |
|
||||
| CONV_2D | ❌ | ❌ | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
|
||||
| CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CONV_3D | ❌ | ❌ | ✅ | ✅ | ❌ | 🟡 | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| COS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CROSS_ENTROPY_LOSS_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DSV4_HC_COMB | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DSV4_HC_POST | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DSV4_HC_PRE | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DUP | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| FLASH_ATTN_EXT | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GATED_LINEAR_ATTN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ |
|
||||
| GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ |
|
||||
| GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| IM2COL | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| LIGHTNING_INDEXER | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| LOG | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 |
|
||||
| MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 |
|
||||
| MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ |
|
||||
| MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ |
|
||||
| PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ |
|
||||
| POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| REPEAT_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| RMS_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RMS_NORM_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROLL | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROPE | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ROPE_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROUND | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RWKV_WKV6 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| RWKV_WKV7 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| SCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SET | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SET_ROWS | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| SGN | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SILU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| SIN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
|
||||
| SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SQRT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ❌ |
|
||||
| STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SUB | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SUM | ❌ | 🟡 | ✅ | 🟡 | ❌ | 🟡 | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| SUM_ROWS | ❌ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SWIGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SWIGLU_CLAMP | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| SWIGLU_OAI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TANH | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TIMESTEP_EMBEDDING | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
|
||||
| TOP_K_MASK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| UPSCALE | ❌ | 🟡 | ✅ | ✅ | ❌ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| XIELU | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| Operation | BLAS | CANN | CPU | CUDA | ET | HTP | MTL | OPENVINO | OpenCL | SYCL | Vulkan | WebGPU | ZenDNN | zDNN |
|
||||
|-----------|------|------|------|------|------|------|------|------|------|------|------|------|------|------|
|
||||
| ABS | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ACC | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ADD1 | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| ADD_ID | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CLAMP | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| COL2IM_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONCAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| CONT | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| CONV_1D_DW | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ |
|
||||
| CONV_2D | ❌ | ❌ | 🟡 | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
|
||||
| CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CONV_3D | ❌ | ❌ | ✅ | ✅ | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| COS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CROSS_ENTROPY_LOSS_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| CUMSUM | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIAG | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DIV | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| DSV4_HC_COMB | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DSV4_HC_POST | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DSV4_HC_PRE | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| DUP | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| EXPM1 | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| FILL | ❌ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| FLASH_ATTN_EXT | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| FLOOR | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GATED_DELTA_NET | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GATED_LINEAR_ATTN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| GEGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GEGLU_ERF | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GEGLU_QUICK | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU_ERF | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GELU_QUICK | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| GET_ROWS | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ |
|
||||
| GET_ROWS_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ |
|
||||
| GROUP_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| HARDSIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| HARDSWISH | ❌ | ✅ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| IM2COL | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| L2_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| LIGHTNING_INDEXER | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| LOG | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 |
|
||||
| MUL_MAT_HADAMARD | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 |
|
||||
| MUL_MAT_ID | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | 🟡 | ❌ |
|
||||
| MUL_MAT_ID_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_ID_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_PREC_SRC1_BF16 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_PREC_SRC1_BF16_SRC0 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_PREC_SRC1_F32 | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_W4A4 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| MUL_MAT_W4A8 | ❌ | ❌ | ✅ | ❌ | ❌ | 🟡 | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| NEG | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| OPT_STEP_ADAMW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| OPT_STEP_SGD | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| OUT_PROD | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ |
|
||||
| PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ |
|
||||
| POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
| REPEAT_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| RMS_NORM | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RMS_NORM_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROLL | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROPE | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| ROPE_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| ROUND | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RWKV_WKV6 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| RWKV_WKV7 | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| SCALE | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SET | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SET_ROWS | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| SGN | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SIGMOID | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SILU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| SIN | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | ❌ | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ |
|
||||
| SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SQR | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SQRT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ❌ |
|
||||
| STEP | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SUB | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SUM | ❌ | 🟡 | ✅ | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | 🟡 | 🟡 | ❌ | ❌ |
|
||||
| SUM_ROWS | ❌ | ✅ | ✅ | 🟡 | ❌ | 🟡 | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| SWIGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| SWIGLU_CLAMP | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
|
||||
| SWIGLU_OAI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TANH | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TIMESTEP_EMBEDDING | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TOP_K | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ |
|
||||
| TOP_K_MASK | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| TRI | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| TRUNC | ❌ | ❌ | ✅ | 🟡 | 🟡 | ❌ | ✅ | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| UPSCALE | ❌ | 🟡 | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| XIELU | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
|
||||
+23719
File diff suppressed because it is too large
Load Diff
+6
-6
@@ -19,8 +19,8 @@
|
||||
"OpenCL","GELU","type=f16,ne_a=[5,7,11,13],v=0","support","0","no","OpenCL"
|
||||
"OpenCL","GELU_QUICK","type=f16,ne_a=[128,2,2,2],v=0","support","0","no","OpenCL"
|
||||
"OpenCL","GELU_QUICK","type=f16,ne_a=[5,7,11,13],v=0","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[128,2,2,2],v=0","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[5,7,11,13],v=0","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[128,2,2,2],v=0","support","1","yes","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[5,7,11,13],v=0","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSWISH","type=f16,ne_a=[128,2,2,2],v=0","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSWISH","type=f16,ne_a=[5,7,11,13],v=0","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSIGMOID","type=f16,ne_a=[128,2,2,2],v=0","support","1","yes","OpenCL"
|
||||
@@ -61,8 +61,8 @@
|
||||
"OpenCL","GELU","type=f16,ne_a=[5,7,11,13],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","GELU_QUICK","type=f16,ne_a=[128,2,2,2],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","GELU_QUICK","type=f16,ne_a=[5,7,11,13],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[128,2,2,2],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[5,7,11,13],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[128,2,2,2],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","SILU","type=f16,ne_a=[5,7,11,13],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSWISH","type=f16,ne_a=[128,2,2,2],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSWISH","type=f16,ne_a=[5,7,11,13],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSIGMOID","type=f16,ne_a=[128,2,2,2],v=1","support","1","yes","OpenCL"
|
||||
@@ -145,8 +145,8 @@
|
||||
"OpenCL","GELU","type=f32,ne_a=[5,7,11,13],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","GELU_QUICK","type=f32,ne_a=[128,2,2,2],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","GELU_QUICK","type=f32,ne_a=[5,7,11,13],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f32,ne_a=[128,2,2,2],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f32,ne_a=[5,7,11,13],v=1","support","0","no","OpenCL"
|
||||
"OpenCL","SILU","type=f32,ne_a=[128,2,2,2],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","SILU","type=f32,ne_a=[5,7,11,13],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSWISH","type=f32,ne_a=[128,2,2,2],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSWISH","type=f32,ne_a=[5,7,11,13],v=1","support","1","yes","OpenCL"
|
||||
"OpenCL","HARDSIGMOID","type=f32,ne_a=[128,2,2,2],v=1","support","1","yes","OpenCL"
|
||||
|
||||
|
Can't render this file because it is too large.
|
+1
-1
@@ -12351,7 +12351,7 @@
|
||||
"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.000000,step=0.250000","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.250000,step=0.250000","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=1048576.000000,step=1.000000","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","TIMESTEP_EMBEDDING","type=f32,ne_a=[2,1,1,1],dim=320,max_period=10000","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","TIMESTEP_EMBEDDING","type=f32,ne_a=[2,1,1,1],dim=320,max_period=10000","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","LEAKY_RELU","type=f32,ne_a=[10,5,4,3],negative_slope=0.100000","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","CUMSUM","type=f32,ne=[10,5,4,3]","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","CUMSUM","type=f32,ne=[127,5,4,3]","support","1","yes","WebGPU"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -7,6 +7,8 @@ Jinja has no regex support, and exposing `std::regex` / Python `re` would not he
|
||||
- `tts_norm_scan(text, rules)`: split text into matched and unmatched segments
|
||||
- `tts_norm_sub(text, pattern, repl)`: replace every match
|
||||
|
||||
A third function, `tts_dict(name, key)`, gives the template access to dictionaries stored in the model (e.g. a pronunciation lexicon).
|
||||
|
||||
Implementations:
|
||||
- C++: [common/chat-tts-norm.cpp](../common/chat-tts-norm.cpp), injected into task templates by `common_chat_template_render()` in [common/chat.cpp](../common/chat.cpp). They are not available to chat templates.
|
||||
- Python reference: [scripts/chat_tts_norm.py](../scripts/chat_tts_norm.py), for use with `jinja2` (`env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan`)
|
||||
@@ -69,6 +71,22 @@ Replaces every match of `pattern`, with the same scanning rules as `tts_norm_sca
|
||||
|
||||
Any other `%` sequence in `repl`, or a reference to a group that does not exist, is an error.
|
||||
|
||||
### `tts_dict(name, key)`
|
||||
|
||||
Looks up `key` in the dictionary `name` of the model and returns its value, or `none` if the key is missing. An unknown dictionary name is an error. A model can ship any number of dictionaries, for example a pronunciation lexicon per language:
|
||||
|
||||
```jinja
|
||||
{%- set e = tts_dict('lexicon_en', word) -%}
|
||||
{%- if e is none -%}{{ spell(word) }}
|
||||
{%- elif e is mapping -%}{{ e[tag] if tag in e else e.DEFAULT }}
|
||||
{%- else -%}{{ e }}
|
||||
{%- endif -%}
|
||||
```
|
||||
|
||||
The lookup is exact: no case folding and no normalization, do them in the template.
|
||||
|
||||
The GGUF lists the names of the dictionaries in `tokenizer.tts_dicts`. Each one is stored as two arrays of strings of the same length: `tokenizer.tts_dict.<name>.keys`, sorted by the UTF-8 bytes of the key, and `tokenizer.tts_dict.<name>.values`. A value is a JSON document (string, number, list, object, ...) and is returned decoded. `GGUFWriter.add_tts_dicts(dicts)` writes all of this.
|
||||
|
||||
## Pattern syntax
|
||||
|
||||
Patterns operate on Unicode codepoints. Invalid UTF-8 bytes (C++ only) are seen as U+FFFD, one per byte, and preserved in the output.
|
||||
|
||||
@@ -87,36 +87,32 @@ for id: llama_token in tokens {
|
||||
|
||||
print("\n")
|
||||
|
||||
var batch = llama_batch_init(max(Int32(tokens.count), Int32(n_parallel)), 0, 1)
|
||||
let batch = llama_batch_ext_init(context)
|
||||
defer {
|
||||
llama_batch_free(batch)
|
||||
llama_batch_ext_free(batch)
|
||||
}
|
||||
|
||||
// evaluate the initial prompt
|
||||
batch.n_tokens = Int32(tokens.count)
|
||||
|
||||
for (i, token) in tokens.enumerated() {
|
||||
batch.token[i] = token
|
||||
batch.pos[i] = Int32(i)
|
||||
batch.n_seq_id[i] = 1
|
||||
// batch.seq_id[i][0] = 0
|
||||
// TODO: is this the proper way to do this?
|
||||
if let seq_id = batch.seq_id[i] {
|
||||
seq_id[0] = 0
|
||||
let idx = llama_batch_ext_add_token(batch, 0, token)
|
||||
if idx < 0 {
|
||||
print("llama_batch_ext_add_token() failed: \(idx)")
|
||||
exit(1)
|
||||
}
|
||||
batch.logits[i] = 0
|
||||
var pos = llama_pos(i)
|
||||
llama_batch_ext_set_pos(batch, idx, &pos)
|
||||
}
|
||||
|
||||
// llama_decode will output logits only for the last token of the prompt
|
||||
batch.logits[Int(batch.n_tokens) - 1] = 1
|
||||
// llama_process will output logits only for the last token of the prompt
|
||||
llama_batch_ext_set_output_logits(batch, Int32(tokens.count) - 1, true)
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed")
|
||||
exit(1)
|
||||
}
|
||||
|
||||
for i in 1 ..< n_parallel {
|
||||
llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, batch.n_tokens)
|
||||
llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, Int32(tokens.count))
|
||||
}
|
||||
|
||||
if n_parallel > 1 {
|
||||
@@ -125,16 +121,17 @@ if n_parallel > 1 {
|
||||
|
||||
var streams: [String] = .init(repeating: "", count: n_parallel)
|
||||
var streamBuffers: [[CChar]] = .init(repeating: [], count: n_parallel)
|
||||
var i_batch = [Int32](repeating: batch.n_tokens - 1, count: n_parallel)
|
||||
var i_batch = [Int32](repeating: Int32(tokens.count) - 1, count: n_parallel)
|
||||
|
||||
var n_cur = batch.n_tokens
|
||||
var n_cur = Int32(tokens.count)
|
||||
var n_decode = 0
|
||||
|
||||
let t_main_start = ggml_time_us()
|
||||
|
||||
while n_cur <= n_len {
|
||||
// prepare the next batch
|
||||
batch.n_tokens = 0
|
||||
llama_batch_ext_clear(batch)
|
||||
var n_tokens: Int32 = 0
|
||||
|
||||
// sample the next token for each parallel sequence / stream
|
||||
for i in 0 ..< n_parallel {
|
||||
@@ -165,31 +162,32 @@ while n_cur <= n_len {
|
||||
streams[i] += nextStringPiece
|
||||
|
||||
// push this new token for next evaluation
|
||||
batch.token[Int(batch.n_tokens)] = new_token_id
|
||||
batch.pos[Int(batch.n_tokens)] = n_cur
|
||||
batch.n_seq_id[Int(batch.n_tokens)] = 1
|
||||
if let seq_id = batch.seq_id[Int(batch.n_tokens)] {
|
||||
seq_id[0] = Int32(i)
|
||||
let idx = llama_batch_ext_add_token(batch, Int32(i), new_token_id)
|
||||
if idx < 0 {
|
||||
print("llama_batch_ext_add_token() failed: \(idx)")
|
||||
exit(1)
|
||||
}
|
||||
batch.logits[Int(batch.n_tokens)] = 1
|
||||
var pos = n_cur
|
||||
llama_batch_ext_set_pos(batch, idx, &pos)
|
||||
llama_batch_ext_set_output_logits(batch, idx, true)
|
||||
|
||||
i_batch[i] = batch.n_tokens
|
||||
i_batch[i] = idx
|
||||
|
||||
batch.n_tokens += 1
|
||||
n_tokens += 1
|
||||
|
||||
n_decode += 1
|
||||
}
|
||||
|
||||
// all streams are finished
|
||||
if batch.n_tokens == 0 {
|
||||
if n_tokens == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
n_cur += 1
|
||||
|
||||
// evaluate the current batch with the transformer model
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed")
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,20 +5,16 @@ enum LlamaError: Error {
|
||||
case couldNotInitializeContext
|
||||
}
|
||||
|
||||
func llama_batch_clear(_ batch: inout llama_batch) {
|
||||
batch.n_tokens = 0
|
||||
}
|
||||
|
||||
func llama_batch_add(_ batch: inout llama_batch, _ id: llama_token, _ pos: llama_pos, _ seq_ids: [llama_seq_id], _ logits: Bool) {
|
||||
batch.token [Int(batch.n_tokens)] = id
|
||||
batch.pos [Int(batch.n_tokens)] = pos
|
||||
batch.n_seq_id[Int(batch.n_tokens)] = Int32(seq_ids.count)
|
||||
for i in 0..<seq_ids.count {
|
||||
batch.seq_id[Int(batch.n_tokens)]![Int(i)] = seq_ids[i]
|
||||
@discardableResult
|
||||
func llama_batch_add(_ batch: OpaquePointer, _ id: llama_token, _ pos: llama_pos, _ seq_id: llama_seq_id, _ logits: Bool) -> Int32 {
|
||||
let idx = llama_batch_ext_add_token(batch, seq_id, id)
|
||||
if idx < 0 {
|
||||
return idx
|
||||
}
|
||||
batch.logits [Int(batch.n_tokens)] = logits ? 1 : 0
|
||||
|
||||
batch.n_tokens += 1
|
||||
var pos = pos
|
||||
llama_batch_ext_set_pos(batch, idx, &pos)
|
||||
llama_batch_ext_set_output_logits(batch, idx, logits)
|
||||
return idx
|
||||
}
|
||||
|
||||
actor LlamaContext {
|
||||
@@ -26,7 +22,7 @@ actor LlamaContext {
|
||||
private var context: OpaquePointer
|
||||
private var vocab: OpaquePointer
|
||||
private var sampling: UnsafeMutablePointer<llama_sampler>
|
||||
private var batch: llama_batch
|
||||
private var batch: OpaquePointer
|
||||
private var tokens_list: [llama_token]
|
||||
var is_done: Bool = false
|
||||
|
||||
@@ -42,7 +38,7 @@ actor LlamaContext {
|
||||
self.model = model
|
||||
self.context = context
|
||||
self.tokens_list = []
|
||||
self.batch = llama_batch_init(512, 0, 1)
|
||||
self.batch = llama_batch_ext_init(context)
|
||||
self.temporary_invalid_cchars = []
|
||||
let sparams = llama_sampler_chain_default_params()
|
||||
self.sampling = llama_sampler_chain_init(sparams)
|
||||
@@ -53,7 +49,7 @@ actor LlamaContext {
|
||||
|
||||
deinit {
|
||||
llama_sampler_free(sampling)
|
||||
llama_batch_free(batch)
|
||||
llama_batch_ext_free(batch)
|
||||
llama_model_free(model)
|
||||
llama_free(context)
|
||||
llama_backend_free()
|
||||
@@ -110,10 +106,6 @@ actor LlamaContext {
|
||||
return SwiftString
|
||||
}
|
||||
|
||||
func get_n_tokens() -> Int32 {
|
||||
return batch.n_tokens;
|
||||
}
|
||||
|
||||
func completion_init(text: String) {
|
||||
print("attempting to complete \"\(text)\"")
|
||||
|
||||
@@ -133,25 +125,27 @@ actor LlamaContext {
|
||||
print(String(cString: token_to_piece(token: id) + [0]))
|
||||
}
|
||||
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_ext_clear(batch)
|
||||
|
||||
for i1 in 0..<tokens_list.count {
|
||||
let i = Int(i1)
|
||||
llama_batch_add(&batch, tokens_list[i], Int32(i), [0], false)
|
||||
}
|
||||
batch.logits[Int(batch.n_tokens) - 1] = 1 // true
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed")
|
||||
let idx = llama_batch_add(batch, tokens_list[i], Int32(i), 0, i == tokens_list.count - 1)
|
||||
if idx < 0 {
|
||||
print("llama_batch_add() failed: \(idx)")
|
||||
}
|
||||
}
|
||||
|
||||
n_cur = batch.n_tokens
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed")
|
||||
}
|
||||
|
||||
n_cur = Int32(tokens_list.count)
|
||||
}
|
||||
|
||||
func completion_loop() -> String {
|
||||
var new_token_id: llama_token = 0
|
||||
|
||||
new_token_id = llama_sampler_sample(sampling, context, batch.n_tokens - 1)
|
||||
new_token_id = llama_sampler_sample(sampling, context, -1)
|
||||
|
||||
if llama_vocab_is_eog(vocab, new_token_id) || n_cur == n_len {
|
||||
print("\n")
|
||||
@@ -178,13 +172,13 @@ actor LlamaContext {
|
||||
print(new_token_str)
|
||||
// tokens_list.append(new_token_id)
|
||||
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_add(&batch, new_token_id, n_cur, [0], true)
|
||||
llama_batch_ext_clear(batch)
|
||||
llama_batch_add(batch, new_token_id, n_cur, 0, true)
|
||||
|
||||
n_decode += 1
|
||||
n_cur += 1
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("failed to evaluate llama!")
|
||||
}
|
||||
|
||||
@@ -201,21 +195,20 @@ actor LlamaContext {
|
||||
for _ in 0..<nr {
|
||||
// bench prompt processing
|
||||
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_ext_clear(batch)
|
||||
|
||||
let n_tokens = pp
|
||||
|
||||
for i in 0..<n_tokens {
|
||||
llama_batch_add(&batch, 0, Int32(i), [0], false)
|
||||
llama_batch_add(batch, 0, Int32(i), 0, i == n_tokens - 1)
|
||||
}
|
||||
batch.logits[Int(batch.n_tokens) - 1] = 1 // true
|
||||
|
||||
llama_memory_clear(llama_get_memory(context), false)
|
||||
|
||||
let t_pp_start = DispatchTime.now().uptimeNanoseconds / 1000;
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed during prompt")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed during prompt")
|
||||
}
|
||||
llama_synchronize(context)
|
||||
|
||||
@@ -228,14 +221,14 @@ actor LlamaContext {
|
||||
let t_tg_start = DispatchTime.now().uptimeNanoseconds / 1000;
|
||||
|
||||
for i in 0..<tg {
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_ext_clear(batch)
|
||||
|
||||
for j in 0..<pl {
|
||||
llama_batch_add(&batch, 0, Int32(i), [Int32(j)], true)
|
||||
llama_batch_add(batch, 0, Int32(i), Int32(j), true)
|
||||
}
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed during text generation")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed during text generation")
|
||||
}
|
||||
llama_synchronize(context)
|
||||
}
|
||||
|
||||
@@ -164,6 +164,11 @@
|
||||
}
|
||||
// lib.optionalAttrs (system == "x86_64-linux") {
|
||||
rocm = config.legacyPackages.llamaPackagesRocm.llama-cpp;
|
||||
|
||||
sycl = config.packages.default.override {
|
||||
useSycl = true;
|
||||
syclToolkit = pkgs.intel-oneapi-toolkit;
|
||||
};
|
||||
};
|
||||
|
||||
# Packages exposed in `.#checks` will be built by the CI and by
|
||||
|
||||
@@ -228,6 +228,7 @@ option(GGML_VULKAN_MEMORY_DEBUG "ggml: enable Vulkan memory debug ou
|
||||
option(GGML_VULKAN_SHADER_DEBUG_INFO "ggml: enable Vulkan shader debug info" OFF)
|
||||
option(GGML_VULKAN_VALIDATE "ggml: enable Vulkan validation" OFF)
|
||||
option(GGML_VULKAN_RUN_TESTS "ggml: run Vulkan tests" OFF)
|
||||
set (GGML_VULKAN_SPIRV_INCLUDE_DIR "" CACHE PATH "ggml: override the SPIRV-Headers include directory")
|
||||
option(GGML_WEBGPU "ggml: use WebGPU" OFF)
|
||||
option(GGML_WEBGPU_DEBUG "ggml: enable WebGPU debug output" OFF)
|
||||
option(GGML_WEBGPU_CPU_PROFILE "ggml: enable WebGPU profiling (CPU)" OFF)
|
||||
|
||||
@@ -698,21 +698,39 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
#endif
|
||||
}
|
||||
|
||||
#if defined(__AVX2__)
|
||||
#if defined(__AVX2__) || defined(__AVX512F__)
|
||||
// four consecutive Q2_0 weights per byte: byte b becomes b & 3, (b >> 2) & 3, (b >> 4) & 3, (b >> 6) & 3 (OR, never add: the overlapping copies only touch masked-off bits)
|
||||
static inline __m256i q2_0_unpack_32_avx2(const uint8_t * qs) {
|
||||
#if defined(__AVX512VBMI__) && defined(__AVX512VL__)
|
||||
const __m256i shifts = _mm256_setr_epi8(
|
||||
0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30,
|
||||
32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62);
|
||||
const __m256i bits = _mm256_broadcastq_epi64(_mm_loadl_epi64((const __m128i *) qs));
|
||||
return _mm256_and_si256(_mm256_multishift_epi64_epi8(shifts, bits), _mm256_set1_epi8(3));
|
||||
#else
|
||||
__m256i v = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *) qs));
|
||||
v = _mm256_or_si256(v, _mm256_slli_epi32(v, 12));
|
||||
v = _mm256_or_si256(v, _mm256_slli_epi32(v, 6));
|
||||
return _mm256_and_si256(v, _mm256_set1_epi32(0x03030303));
|
||||
#endif
|
||||
}
|
||||
|
||||
// 32 weights against one Q8_0 block: sum((v - 1) * y) = sum(v * y) - sum(y), exact for every int8 y
|
||||
static inline __m256 q2_0_dot_32_avx2(const uint8_t * qs, const int8_t * qy) {
|
||||
const __m256i v = q2_0_unpack_32_avx2(qs);
|
||||
const __m256i y = _mm256_loadu_si256((const __m256i *) qy);
|
||||
#if defined(__AVX512VNNI__) && defined(__AVX512VL__)
|
||||
const __m256i p = _mm256_dpbusd_epi32(_mm256_setzero_si256(), v, y);
|
||||
const __m256i s = _mm256_dpbusd_epi32(_mm256_setzero_si256(), _mm256_set1_epi8(1), y);
|
||||
return _mm256_cvtepi32_ps(_mm256_sub_epi32(p, s));
|
||||
#elif defined(__AVXVNNI__)
|
||||
const __m256i p = _mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), v, y);
|
||||
const __m256i s = _mm256_dpbusd_avx_epi32(_mm256_setzero_si256(), _mm256_set1_epi8(1), y);
|
||||
return _mm256_cvtepi32_ps(_mm256_sub_epi32(p, s));
|
||||
#else
|
||||
const __m256i p = _mm256_sub_epi16(_mm256_maddubs_epi16(v, y), _mm256_maddubs_epi16(_mm256_set1_epi8(1), y));
|
||||
return _mm256_cvtepi32_ps(_mm256_madd_epi16(p, _mm256_set1_epi16(1)));
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline __m256 q2_0_acc_block_avx2(const block_q2_0 * x, const block_q8_0 * y, __m256 acc) {
|
||||
|
||||
@@ -1309,14 +1309,12 @@ class tinyBLAS_RVV {
|
||||
// QUANT ZERO MATRIX MULTIPLICATION
|
||||
|
||||
#if defined(__VXE__) || defined(__VXE2__)
|
||||
template <typename TA>
|
||||
template <typename TA, typename TB>
|
||||
class tinyBLAS_Q0_VXE {
|
||||
static_assert(sizeof(TA::qs) == QK8_0, "VXE weight loads require 32 quantized bytes per block");
|
||||
|
||||
public:
|
||||
tinyBLAS_Q0_VXE(int64_t k,
|
||||
const TA * A, int64_t lda,
|
||||
const block_q8_0 * B, int64_t ldb,
|
||||
const TB * B, int64_t ldb,
|
||||
float * C, int64_t ldc,
|
||||
int32_t ith, int32_t nth)
|
||||
: A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) {
|
||||
@@ -1333,44 +1331,54 @@ class tinyBLAS_Q0_VXE {
|
||||
for (int64_t job = start; job < end; ++job) {
|
||||
const int64_t ii = tile_size * (job % row_tiles);
|
||||
const int64_t jj = tile_size * (job / row_tiles);
|
||||
|
||||
float sum_min[tile_size][tile_size] = {};
|
||||
float32x4_t v_sum[tile_size][tile_size] = {};
|
||||
|
||||
for (int64_t l = 0; l < k; ++l) {
|
||||
int8x16_t v_xl[tile_size], v_xh[tile_size], v_yl[tile_size], v_yh[tile_size];
|
||||
int8x16_t v_xl[tile_size], v_xh[tile_size];
|
||||
int8x16_t v_yl[tile_size], v_yh[tile_size];
|
||||
|
||||
float x_scale[tile_size], y_scale[tile_size];
|
||||
float x_min[tile_size], y_sum[tile_size];
|
||||
|
||||
for (int32_t i = 0; i < tile_size; ++i) {
|
||||
const TA & x = A[lda * (ii + i) + l];
|
||||
const block_q8_0 & y = B[ldb * (jj + i) + l];
|
||||
const TB & y = B[ldb * (jj + i) + l];
|
||||
|
||||
load(x, v_xl[i], v_xh[i]);
|
||||
load(y, v_yl[i], v_yh[i]);
|
||||
|
||||
const int8_t * x_qs = x.qs;
|
||||
const int8_t * y_qs = y.qs;
|
||||
v_xl[i] = vec_xl(0, x_qs);
|
||||
v_xh[i] = vec_xl(0, x_qs + QK8_0 / 2);
|
||||
v_yl[i] = vec_xl(0, y_qs);
|
||||
v_yh[i] = vec_xl(0, y_qs + QK8_0 / 2);
|
||||
x_scale[i] = unhalf(x.d);
|
||||
y_scale[i] = unhalf(y.d);
|
||||
x_min[i] = min(x);
|
||||
y_sum[i] = sum(y);
|
||||
}
|
||||
|
||||
for (int32_t j = 0; j < tile_size; ++j) {
|
||||
for (int32_t i = 0; i < tile_size; ++i) {
|
||||
const int32x4_t v_dot = dot_s8_wide(v_xl[i], v_yl[j]) + dot_s8_wide(v_xh[i], v_yh[j]);
|
||||
const int32x4_t v_dot = dot(v_xl[i], v_xh[i], v_yl[j], v_yh[j]);
|
||||
const float32x4_t v_scale = vec_splats(x_scale[i] * y_scale[j]);
|
||||
v_sum[j][i] = vec_madd(vec_float(v_dot), v_scale, v_sum[j][i]);
|
||||
if (has_min) {
|
||||
sum_min[j][i] += x_min[i] * y_sum[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (int32_t j = 0; j < tile_size; ++j) {
|
||||
for (int32_t i = 0; i < tile_size; ++i) {
|
||||
C[ldc * (jj + j) + ii + i] = hsum(v_sum[j][i]);
|
||||
C[ldc * (jj + j) + ii + i] = hsum(v_sum[j][i]) + sum_min[j][i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
// Q4_1 and Q5_1 weights have a block minimum, it pairs with the Q8_1 block sum.
|
||||
static constexpr bool has_min = std::is_same<TB, block_q8_1>::value;
|
||||
|
||||
static int32x4_t dot_s8_wide(int8x16_t v_x, int8x16_t v_y) {
|
||||
const int16x8_t v_even = vec_mule(v_x, v_y);
|
||||
const int16x8_t v_odd = vec_mulo(v_x, v_y);
|
||||
@@ -1378,8 +1386,86 @@ class tinyBLAS_Q0_VXE {
|
||||
return vec_unpackh(v_even) + vec_unpackl(v_even) + vec_unpackh(v_odd) + vec_unpackl(v_odd);
|
||||
}
|
||||
|
||||
static int32x4_t dot(int8x16_t v_xl, int8x16_t v_xh, int8x16_t v_yl, int8x16_t v_yh) {
|
||||
// block_q8_0 path
|
||||
if (std::is_same<TA, block_q8_0>::value) {
|
||||
return dot_s8_wide(v_xl, v_yl) + dot_s8_wide(v_xh, v_yh);
|
||||
}
|
||||
|
||||
// block_q8_1 path
|
||||
const int16x8_t v_one = vec_splats((int16_t)1);
|
||||
const int16x8_t v_xy = vec_meadd(v_xl, v_yl, vec_moadd(v_xl, v_yl, vec_meadd(v_xh, v_yh, vec_mulo(v_xh, v_yh))));
|
||||
return vec_meadd(v_xy, v_one, vec_mulo(v_xy, v_one));
|
||||
}
|
||||
|
||||
static void load(const block_q4_0 & x, int8x16_t & v_xl, int8x16_t & v_xh) {
|
||||
const uint8_t * x_qs = x.qs;
|
||||
const uint8x16_t v_x = vec_xl(0, x_qs);
|
||||
v_xl = (int8x16_t)vec_and(v_x, vec_splats((uint8_t)0x0F)) - vec_splats((int8_t)8);
|
||||
v_xh = (int8x16_t)vec_sr(v_x, 4) - vec_splats((int8_t)8);
|
||||
}
|
||||
|
||||
static void load(const block_q4_1 & x, int8x16_t & v_xl, int8x16_t & v_xh) {
|
||||
const uint8_t * x_qs = x.qs;
|
||||
const uint8x16_t v_x = vec_xl(0, x_qs);
|
||||
v_xl = (int8x16_t)vec_and(v_x, vec_splats((uint8_t)0x0F));
|
||||
v_xh = (int8x16_t)vec_sr(v_x, 4);
|
||||
}
|
||||
|
||||
static void load(const block_q5_0 & x, int8x16_t & v_xl, int8x16_t & v_xh) {
|
||||
const uint8_t * x_qs = x.qs;
|
||||
const uint8x16_t v_x = vec_xl(0, x_qs);
|
||||
uint8x16_t v_qhl, v_qhh;
|
||||
load_qh(x.qh, v_qhl, v_qhh);
|
||||
v_xl = (int8x16_t)vec_or(vec_and(v_x, vec_splats((uint8_t)0x0F)), v_qhl) - vec_splats((int8_t)16);
|
||||
v_xh = (int8x16_t)vec_or(vec_sr(v_x, 4), v_qhh) - vec_splats((int8_t)16);
|
||||
}
|
||||
|
||||
static void load(const block_q5_1 & x, int8x16_t & v_xl, int8x16_t & v_xh) {
|
||||
const uint8_t * x_qs = x.qs;
|
||||
const uint8x16_t v_x = vec_xl(0, x_qs);
|
||||
uint8x16_t v_qhl, v_qhh;
|
||||
load_qh(x.qh, v_qhl, v_qhh);
|
||||
v_xl = (int8x16_t)vec_or(vec_and(v_x, vec_splats((uint8_t)0x0F)), v_qhl);
|
||||
v_xh = (int8x16_t)vec_or(vec_sr(v_x, 4), v_qhh);
|
||||
}
|
||||
|
||||
static void load(const block_q8_0 & x, int8x16_t & v_xl, int8x16_t & v_xh) {
|
||||
const int8_t * x_qs = x.qs;
|
||||
v_xl = vec_xl(0, x_qs);
|
||||
v_xh = vec_xl(0, x_qs + QK8_0 / 2);
|
||||
}
|
||||
|
||||
static void load(const block_q8_1 & x, int8x16_t & v_xl, int8x16_t & v_xh) {
|
||||
const int8_t * x_qs = x.qs;
|
||||
v_xl = vec_xl(0, x_qs);
|
||||
v_xh = vec_xl(0, x_qs + QK8_1 / 2);
|
||||
}
|
||||
|
||||
// expand the 32 high bits to one byte each, with value 0 or 16
|
||||
static void load_qh(const uint8_t * qh, uint8x16_t & v_qhl, uint8x16_t & v_qhh) {
|
||||
// qh is a native 32-bit value, on big-endian its lowest bits are in byte 3
|
||||
const uint8x16_t v_bytel = { 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2 };
|
||||
const uint8x16_t v_byteh = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0 };
|
||||
const uint8x16_t v_bit = { 1, 2, 4, 8, 16, 32, 64, 128, 1, 2, 4, 8, 16, 32, 64, 128 };
|
||||
|
||||
uint32_t tmp;
|
||||
memcpy(&tmp, qh, sizeof(tmp));
|
||||
const uint8x16_t v_qh = (uint8x16_t)vec_splats(tmp);
|
||||
|
||||
v_qhl = (uint8x16_t)vec_cmpeq(vec_and(vec_perm(v_qh, v_qh, v_bytel), v_bit), v_bit) & vec_splats((uint8_t)16);
|
||||
v_qhh = (uint8x16_t)vec_cmpeq(vec_and(vec_perm(v_qh, v_qh, v_byteh), v_bit), v_bit) & vec_splats((uint8_t)16);
|
||||
}
|
||||
|
||||
static float min(const block_q4_1 & x) { return unhalf(x.m); }
|
||||
static float min(const block_q5_1 & x) { return unhalf(x.m); }
|
||||
template <typename T> static float min(const T &) { return 0.0f; }
|
||||
|
||||
static float sum(const block_q8_0 & ) { return 0.0f; }
|
||||
static float sum(const block_q8_1 & y) { return unhalf(y.s); }
|
||||
|
||||
const TA * const A;
|
||||
const block_q8_0 * const B;
|
||||
const TB * const B;
|
||||
float * const C;
|
||||
const int64_t k;
|
||||
const int64_t lda;
|
||||
@@ -4262,7 +4348,7 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
|
||||
if (m % 2 || n % 2) {
|
||||
return false;
|
||||
}
|
||||
tinyBLAS_Q0_VXE<block_q8_0> tb{
|
||||
tinyBLAS_Q0_VXE<block_q8_0, block_q8_0> tb{
|
||||
k, (const block_q8_0 *)A, lda,
|
||||
(const block_q8_0 *)B, ldb,
|
||||
(float *)C, ldc,
|
||||
@@ -4306,6 +4392,17 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#elif defined(__VXE__) || defined(__VXE2__)
|
||||
if (m % 2 || n % 2) {
|
||||
return false;
|
||||
}
|
||||
tinyBLAS_Q0_VXE<block_q4_0, block_q8_0> tb{
|
||||
k, (const block_q4_0 *)A, lda,
|
||||
(const block_q8_0 *)B, ldb,
|
||||
(float *)C, ldc,
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#elif defined(__MMA__)
|
||||
//TO-DO: Remove this condition once gemv forwarding is enabled.
|
||||
if (n < 8 && n != 4)
|
||||
@@ -4335,6 +4432,57 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#elif defined(__VXE__) || defined(__VXE2__)
|
||||
if (m % 2 || n % 2) {
|
||||
return false;
|
||||
}
|
||||
tinyBLAS_Q0_VXE<block_q5_0, block_q8_0> tb{
|
||||
k, (const block_q5_0 *)A, lda,
|
||||
(const block_q8_0 *)B, ldb,
|
||||
(float *)C, ldc,
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
case GGML_TYPE_Q4_1: {
|
||||
if (Btype != GGML_TYPE_Q8_1) {
|
||||
return false;
|
||||
}
|
||||
#if defined(__VXE__) || defined(__VXE2__)
|
||||
if (m % 2 || n % 2) {
|
||||
return false;
|
||||
}
|
||||
tinyBLAS_Q0_VXE<block_q4_1, block_q8_1> tb{
|
||||
k, (const block_q4_1 *)A, lda,
|
||||
(const block_q8_1 *)B, ldb,
|
||||
(float *)C, ldc,
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
}
|
||||
|
||||
case GGML_TYPE_Q5_1: {
|
||||
if (Btype != GGML_TYPE_Q8_1) {
|
||||
return false;
|
||||
}
|
||||
#if defined(__VXE__) || defined(__VXE2__)
|
||||
if (m % 2 || n % 2) {
|
||||
return false;
|
||||
}
|
||||
tinyBLAS_Q0_VXE<block_q5_1, block_q8_1> tb{
|
||||
k, (const block_q5_1 *)A, lda,
|
||||
(const block_q8_1 *)B, ldb,
|
||||
(float *)C, ldc,
|
||||
params->ith, params->nth};
|
||||
tb.matmul(m, n);
|
||||
return true;
|
||||
#else
|
||||
return false;
|
||||
#endif
|
||||
|
||||
@@ -22,8 +22,8 @@ static __global__ void argmax_f32(const float * __restrict__ x, int32_t * __rest
|
||||
|
||||
#pragma unroll
|
||||
for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) {
|
||||
const float val = __shfl_xor_sync(0xFFFFFFFF, maxval, offset, WARP_SIZE);
|
||||
const int col = __shfl_xor_sync(0xFFFFFFFF, argmax, offset, WARP_SIZE);
|
||||
const float val = ggml_cuda_shfl_xor(maxval, offset, WARP_SIZE);
|
||||
const int col = ggml_cuda_shfl_xor(argmax, offset, WARP_SIZE);
|
||||
if (val > maxval) {
|
||||
maxval = val;
|
||||
argmax = col;
|
||||
@@ -51,8 +51,8 @@ static __global__ void argmax_f32(const float * __restrict__ x, int32_t * __rest
|
||||
}
|
||||
#pragma unroll
|
||||
for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) {
|
||||
const float val = __shfl_xor_sync(0xFFFFFFFF, maxval, offset, WARP_SIZE);
|
||||
const int col = __shfl_xor_sync(0xFFFFFFFF, argmax, offset, WARP_SIZE);
|
||||
const float val = ggml_cuda_shfl_xor(maxval, offset, WARP_SIZE);
|
||||
const int col = ggml_cuda_shfl_xor(argmax, offset, WARP_SIZE);
|
||||
if (val > maxval) {
|
||||
maxval = val;
|
||||
argmax = col;
|
||||
|
||||
@@ -19,7 +19,6 @@
|
||||
#define GGML_COMMON_IMPL_MUSA
|
||||
#endif
|
||||
#endif
|
||||
#include "ggml-common.h"
|
||||
|
||||
#include <array>
|
||||
#include <algorithm>
|
||||
@@ -39,6 +38,8 @@
|
||||
#include "vendors/cuda.h"
|
||||
#endif // defined(GGML_USE_HIP)
|
||||
|
||||
#include "ggml-common.h"
|
||||
|
||||
#define STRINGIZE_IMPL(...) #__VA_ARGS__
|
||||
#define STRINGIZE(...) STRINGIZE_IMPL(__VA_ARGS__)
|
||||
|
||||
@@ -126,6 +127,8 @@
|
||||
# define GGML_CUDA_USE_PDL
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && (CUDART_VERSION >= 12030 || (!(defined(_MSC_VER) && !defined(__clang__)) && CUDART_VERSION >= 11080))
|
||||
|
||||
constexpr ggml_cuda_thread_mask_t GGML_CUDA_FULL_THREAD_MASK = (ggml_cuda_thread_mask_t)0xffffffffffffffffULL;
|
||||
|
||||
static __device__ __forceinline__ void ggml_cuda_syncwarp() {
|
||||
#ifndef GGML_USE_HIP
|
||||
__syncwarp();
|
||||
@@ -430,6 +433,32 @@ static constexpr __device__ int ggml_cuda_get_max_cpy_bytes() {
|
||||
#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static __device__ __forceinline__ T ggml_cuda_shfl(T value, int src_lane, int width = ggml_cuda_get_physical_warp_size(), ggml_cuda_thread_mask_t mask = GGML_CUDA_FULL_THREAD_MASK) {
|
||||
return __shfl_sync(mask, value, src_lane, width);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static __device__ __forceinline__ T ggml_cuda_shfl_up(T value, unsigned delta, int width = ggml_cuda_get_physical_warp_size(), ggml_cuda_thread_mask_t mask = GGML_CUDA_FULL_THREAD_MASK) {
|
||||
return __shfl_up_sync(mask, value, delta, width);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static __device__ __forceinline__ T ggml_cuda_shfl_xor(T value, int lane_mask, int width = ggml_cuda_get_physical_warp_size(), ggml_cuda_thread_mask_t mask = GGML_CUDA_FULL_THREAD_MASK) {
|
||||
return __shfl_xor_sync(mask, value, lane_mask, width);
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ ggml_cuda_thread_mask_t ggml_cuda_ballot(int predicate, ggml_cuda_thread_mask_t mask = GGML_CUDA_FULL_THREAD_MASK) {
|
||||
return __ballot_sync(mask, predicate);
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ int ggml_cuda_all(int predicate, ggml_cuda_thread_mask_t mask = GGML_CUDA_FULL_THREAD_MASK) {
|
||||
return __all_sync(mask, predicate);
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ int ggml_cuda_any(int predicate, ggml_cuda_thread_mask_t mask = GGML_CUDA_FULL_THREAD_MASK) {
|
||||
return __any_sync(mask, predicate);
|
||||
}
|
||||
|
||||
[[noreturn]]
|
||||
static __device__ void no_device_code(
|
||||
@@ -476,11 +505,11 @@ struct ggml_cuda_unroll<1> {
|
||||
template<int width = WARP_SIZE>
|
||||
static __device__ __forceinline__ int warp_reduce_sum(int x) {
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
|
||||
return __reduce_add_sync(0xffffffff, x);
|
||||
return __reduce_add_sync(GGML_CUDA_FULL_THREAD_MASK, x);
|
||||
#else
|
||||
#pragma unroll
|
||||
for (int offset = width/2; offset > 0; offset >>= 1) {
|
||||
x += __shfl_xor_sync(0xffffffff, x, offset, width);
|
||||
x += ggml_cuda_shfl_xor(x, offset, width);
|
||||
}
|
||||
return x;
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_AMPERE
|
||||
@@ -490,7 +519,7 @@ template<int width = WARP_SIZE>
|
||||
static __device__ __forceinline__ float warp_reduce_sum(float x) {
|
||||
#pragma unroll
|
||||
for (int offset = width/2; offset > 0; offset >>= 1) {
|
||||
x += __shfl_xor_sync(0xffffffff, x, offset, width);
|
||||
x += ggml_cuda_shfl_xor(x, offset, width);
|
||||
}
|
||||
return x;
|
||||
}
|
||||
@@ -499,8 +528,8 @@ template<int width = WARP_SIZE>
|
||||
static __device__ __forceinline__ float2 warp_reduce_sum(float2 a) {
|
||||
#pragma unroll
|
||||
for (int offset = width/2; offset > 0; offset >>= 1) {
|
||||
a.x += __shfl_xor_sync(0xffffffff, a.x, offset, width);
|
||||
a.y += __shfl_xor_sync(0xffffffff, a.y, offset, width);
|
||||
a.x += ggml_cuda_shfl_xor(a.x, offset, width);
|
||||
a.y += ggml_cuda_shfl_xor(a.y, offset, width);
|
||||
}
|
||||
return a;
|
||||
}
|
||||
@@ -510,7 +539,7 @@ static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {
|
||||
#ifdef FP16_AVAILABLE
|
||||
#pragma unroll
|
||||
for (int offset = width/2; offset > 0; offset >>= 1) {
|
||||
a = __hadd2(a, __shfl_xor_sync(0xffffffff, a, offset, width));
|
||||
a = __hadd2(a, ggml_cuda_shfl_xor(a, offset, width));
|
||||
}
|
||||
return a;
|
||||
|
||||
@@ -523,11 +552,11 @@ static __device__ __forceinline__ half2 warp_reduce_sum(half2 a) {
|
||||
template<int width = WARP_SIZE>
|
||||
static __device__ __forceinline__ int warp_reduce_all(int x) {
|
||||
if (width == ggml_cuda_get_physical_warp_size()) {
|
||||
return __all_sync(0xffffffff, x);
|
||||
return ggml_cuda_all(x);
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int offset = width/2; offset > 0; offset >>= 1) {
|
||||
x = __shfl_xor_sync(0xffffffff, x, offset, width) && x;
|
||||
x = ggml_cuda_shfl_xor(x, offset, width) && x;
|
||||
}
|
||||
return x;
|
||||
}
|
||||
@@ -536,11 +565,11 @@ static __device__ __forceinline__ int warp_reduce_all(int x) {
|
||||
template<int width = WARP_SIZE>
|
||||
static __device__ __forceinline__ int warp_reduce_any(int x) {
|
||||
if (width == ggml_cuda_get_physical_warp_size()) {
|
||||
return __any_sync(0xffffffff, x);
|
||||
return ggml_cuda_any(x);
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int offset = width/2; offset > 0; offset >>= 1) {
|
||||
x = __shfl_xor_sync(0xffffffff, x, offset, width) || x;
|
||||
x = ggml_cuda_shfl_xor(x, offset, width) || x;
|
||||
}
|
||||
return x;
|
||||
}
|
||||
@@ -550,7 +579,7 @@ template<int width = WARP_SIZE>
|
||||
static __device__ __forceinline__ float warp_reduce_max(float x) {
|
||||
#pragma unroll
|
||||
for (int offset = width/2; offset > 0; offset >>= 1) {
|
||||
x = fmaxf(x, __shfl_xor_sync(0xffffffff, x, offset, width));
|
||||
x = fmaxf(x, ggml_cuda_shfl_xor(x, offset, width));
|
||||
}
|
||||
return x;
|
||||
}
|
||||
@@ -560,7 +589,7 @@ static __device__ __forceinline__ T warp_prefix_inclusive_sum(T x) {
|
||||
const int lane_id = threadIdx.x % width;
|
||||
#pragma unroll
|
||||
for (int offset = 1; offset < width; offset <<= 1) {
|
||||
const T t = __shfl_up_sync(0xffffffff, x, offset, width);
|
||||
const T t = ggml_cuda_shfl_up(x, offset, width);
|
||||
if (lane_id >= offset) {
|
||||
x += t;
|
||||
}
|
||||
@@ -573,8 +602,8 @@ static __device__ __forceinline__ float2 warp_prefix_inclusive_sum(float2 a) {
|
||||
const int lane_id = threadIdx.x % width;
|
||||
#pragma unroll
|
||||
for (int offset = 1; offset < width; offset <<= 1) {
|
||||
const float t_x = __shfl_up_sync(0xffffffff, a.x, offset, width);
|
||||
const float t_y = __shfl_up_sync(0xffffffff, a.y, offset, width);
|
||||
const float t_x = ggml_cuda_shfl_up(a.x, offset, width);
|
||||
const float t_y = ggml_cuda_shfl_up(a.y, offset, width);
|
||||
if (lane_id >= offset) {
|
||||
a.x += t_x;
|
||||
a.y += t_y;
|
||||
@@ -589,7 +618,7 @@ static __device__ __forceinline__ half2 warp_prefix_inclusive_sum(half2 a) {
|
||||
const int lane_id = threadIdx.x % width;
|
||||
#pragma unroll
|
||||
for (int offset = 1; offset < width; offset <<= 1) {
|
||||
const half2 t = __shfl_up_sync(0xffffffff, a, offset, width);
|
||||
const half2 t = ggml_cuda_shfl_up(a, offset, width);
|
||||
if (lane_id >= offset) {
|
||||
a = __hadd2(a, t);
|
||||
}
|
||||
@@ -716,7 +745,7 @@ template<int width = WARP_SIZE>
|
||||
static __device__ __forceinline__ half2 warp_reduce_max(half2 x) {
|
||||
#pragma unroll
|
||||
for (int offset = width/2; offset > 0; offset >>= 1) {
|
||||
x = ggml_cuda_hmax2(x, __shfl_xor_sync(0xffffffff, x, offset, width));
|
||||
x = ggml_cuda_hmax2(x, ggml_cuda_shfl_xor(x, offset, width));
|
||||
}
|
||||
return x;
|
||||
}
|
||||
@@ -867,7 +896,7 @@ static __device__ __forceinline__ float ggml_cuda_e8m0_to_fp32(uint8_t x) {
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ float ggml_cuda_ue4m3_to_fp32(uint8_t x) {
|
||||
#if defined(GGML_USE_HIP) && defined(CDNA3) && defined(FP8_AVAILABLE) && HIP_VERSION >= 60200000
|
||||
#if defined(GGML_USE_HIP) && defined(CDNA3) && defined(FP8_AVAILABLE)
|
||||
// ROCm does not support fp8 in software on devices with fp8 hardware,
|
||||
// but CDNA3 supports only e4m3_fnuz (no inf).
|
||||
const uint32_t bits = x * (x != 0x7F && x != 0xFF); // Convert NaN to 0.0f to match CPU implementation.
|
||||
|
||||
@@ -40,7 +40,7 @@ typedef void (* fattn_kernel_t)(
|
||||
const int32_t nb11, const int32_t nb12, const int64_t nb13,
|
||||
const int32_t nb21, const int32_t nb22, const int64_t nb23,
|
||||
const int32_t ne31, const int32_t ne32, const int32_t ne33,
|
||||
const int32_t nb31, const int32_t nb32, const int64_t nb33);
|
||||
const int32_t nb31, const int64_t nb32, const int64_t nb33);
|
||||
|
||||
typedef float (*vec_dot_KQ_t)(
|
||||
const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8 , const void * __restrict__ Q_ds);
|
||||
@@ -376,8 +376,8 @@ static __device__ __forceinline__ void quantize_q8_1_to_shared(
|
||||
}
|
||||
#pragma unroll
|
||||
for (int mask = QI8_1/2; mask > 0; mask >>= 1) {
|
||||
amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, mask, 32));
|
||||
sum += __shfl_xor_sync(0xFFFFFFFF, sum, mask, 32);
|
||||
amax = fmaxf(amax, ggml_cuda_shfl_xor(amax, mask, 32));
|
||||
sum += ggml_cuda_shfl_xor(sum, mask, 32);
|
||||
}
|
||||
|
||||
const float d = amax / 127;
|
||||
@@ -1170,7 +1170,7 @@ void launch_fattn(
|
||||
// Optional optimization where the mask is scanned to determine whether part of the calculation can be skipped.
|
||||
// Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or
|
||||
// multiple sequences of possibly different lengths, or kv_rows lists that end with padding.
|
||||
const bool scan_mask = !use_sparse && mask && n_kv_cols % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1 || kv_rows);
|
||||
const bool scan_mask = !use_sparse && mask && mask->ne[2] == 1 && n_kv_cols % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1 || kv_rows);
|
||||
if (scan_mask) {
|
||||
const int64_t s31 = mask->nb[1] / sizeof(half2);
|
||||
const int64_t s33 = mask->nb[3] / sizeof(half2);
|
||||
|
||||
@@ -639,6 +639,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
float * const __restrict__ KQ_rowsum,
|
||||
const int jt,
|
||||
const int kb0,
|
||||
const int kb0_next,
|
||||
const int k_VKQ_sup) {
|
||||
#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
@@ -818,7 +819,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
for (int col = 0; col < cols_per_thread; ++col) {
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset >= 4; offset >>= 1) {
|
||||
KQ_max_new[col] = fmaxf(KQ_max_new[col], __shfl_xor_sync(0xFFFFFFFF, KQ_max_new[col], offset, warp_size));
|
||||
KQ_max_new[col] = fmaxf(KQ_max_new[col], ggml_cuda_shfl_xor(KQ_max_new[col], offset, warp_size));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -911,7 +912,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
#endif // defined(TURING_MMA_AVAILABLE)
|
||||
#pragma unroll
|
||||
for (int offset = offset_first; offset >= offset_last; offset >>= 1) {
|
||||
KQ_max_new[col] = fmaxf(KQ_max_new[col], __shfl_xor_sync(0xFFFFFFFF, KQ_max_new[col], offset, warp_size));
|
||||
KQ_max_new[col] = fmaxf(KQ_max_new[col], ggml_cuda_shfl_xor(KQ_max_new[col], offset, warp_size));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1030,10 +1031,10 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
if (!last_iter) {
|
||||
if (ncols2 > 1 || mask_h) {
|
||||
flash_attn_ext_f16_load_mask<ncols1, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
|
||||
(mask_h, tile_mask, stride_mask, k_VKQ_0 + nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
|
||||
(mask_h, tile_mask, stride_mask, kb0_next*nbatch_fa, k_VKQ_sup, jt*ncols1, ne01, nullptr);
|
||||
}
|
||||
flash_attn_ext_f16_load_tile<stride_tile_K, nwarps, nbatch_fa, use_cp_async, oob_check, use_sparse>
|
||||
(K_h2, tile_K, min(nbatch_K2, DKQ/2), stride_K, k_VKQ_0 + nbatch_fa, k_VKQ_sup, indices);
|
||||
(K_h2, tile_K, min(nbatch_K2, DKQ/2), stride_K, kb0_next*nbatch_fa, k_VKQ_sup, indices);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1108,7 +1109,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_iter(
|
||||
scale, slope, logit_softcap, ne01, ne02,
|
||||
stride_K, stride_V, stride_mask,
|
||||
tile_Q, tile_K, tile_V, tile_mask,
|
||||
Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0);
|
||||
Q_B, VKQ_C, KQ_max, KQ_rowsum, kb0, kb0_next);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
|
||||
}
|
||||
@@ -1285,7 +1286,8 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
const int jt,
|
||||
const int zt_gqa,
|
||||
const int kb0_start,
|
||||
const int kb0_stop) {
|
||||
const int kb0_stop,
|
||||
const int kb0_max) {
|
||||
#if defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
|
||||
//In this kernel Q, K, V are matrices while i, j, k are matrix indices.
|
||||
|
||||
@@ -1438,7 +1440,13 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
int kb0 = kb0_start;
|
||||
// With multi-stage loading the K/V block order is rotated, see flash_attn_ext_f16.
|
||||
// Then K/V blocks >= iter_k wrap around to 0 and the fully masked K/V blocks >= kb0_max are skipped.
|
||||
const int iter_k = (ne11 + nbatch_fa - 1) / nbatch_fa;
|
||||
const int nkb_pre = max(0, min(kb0_stop, kb0_max) - kb0_start);
|
||||
const int nkb_post = max(0, min(kb0_stop - iter_k, kb0_max));
|
||||
const int nkb = max(1, nkb_pre + nkb_post);
|
||||
int kb0 = nstages > 1 && nkb_pre == 0 && nkb_post > 0 ? 0 : kb0_start;
|
||||
|
||||
// Preload mask and K data for first iteration when using cp_async with multiple stages:
|
||||
if constexpr (nstages > 1) {
|
||||
@@ -1456,7 +1464,29 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
}
|
||||
|
||||
// kb0_start is always < kb0_stop so the last iter can be executed unconditionally.
|
||||
if constexpr (ncols2 == 1 || use_sparse) {
|
||||
if constexpr (nstages > 1) {
|
||||
constexpr bool oob_check = false;
|
||||
for (int ikb = 1; ikb < nkb; ++ikb) {
|
||||
const int kb0_next = kb0 + 1 == kb0_max ? 0 : kb0 + 1;
|
||||
constexpr bool last_iter = false;
|
||||
constexpr int k_VKQ_sup = nbatch_fa;
|
||||
flash_attn_ext_f16_iter
|
||||
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, kb0_next, k_VKQ_sup);
|
||||
kb0 = kb0_next;
|
||||
}
|
||||
constexpr bool last_iter = true;
|
||||
constexpr int k_VKQ_sup = nbatch_fa;
|
||||
flash_attn_ext_f16_iter
|
||||
<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup, last_iter, oob_check,
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, kb0, k_VKQ_sup);
|
||||
} else if constexpr (ncols2 == 1 || use_sparse) {
|
||||
constexpr bool oob_check = true;
|
||||
for (; kb0 < kb0_stop-1; ++kb0) {
|
||||
constexpr bool last_iter = false;
|
||||
@@ -1466,7 +1496,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
|
||||
KQ_max, KQ_rowsum, jt, kb0, kb0 + 1, k_VKQ_sup);
|
||||
}
|
||||
constexpr bool last_iter = true;
|
||||
const int k_VKQ_sup = ne11 - kb0*nbatch_fa;
|
||||
@@ -1475,7 +1505,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
|
||||
KQ_max, KQ_rowsum, jt, kb0, kb0, k_VKQ_sup);
|
||||
} else {
|
||||
constexpr bool oob_check = false;
|
||||
for (; kb0 < kb0_stop-1; ++kb0) {
|
||||
@@ -1486,7 +1516,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
|
||||
KQ_max, KQ_rowsum, jt, kb0, kb0 + 1, k_VKQ_sup);
|
||||
}
|
||||
constexpr bool last_iter = true;
|
||||
constexpr int k_VKQ_sup = nbatch_fa;
|
||||
@@ -1495,7 +1525,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
T_A_KQ, T_B_KQ, T_C_KQ, T_A_VKQ, T_B_VKQ, T_C_VKQ>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, dstk, dstk_fixup, scale, slope, logit_softcap,
|
||||
ne01, ne02, stride_K, stride_V, stride_mask, tile_Q, tile_K, tile_V, tile_mask, Q_B, VKQ_C,
|
||||
KQ_max, KQ_rowsum, jt, kb0, k_VKQ_sup);
|
||||
KQ_max, KQ_rowsum, jt, kb0, kb0, k_VKQ_sup);
|
||||
}
|
||||
|
||||
// With multi-stage loading there is no __syncthreads at the end of the iter,
|
||||
@@ -1527,7 +1557,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
for (int col = 0; col < cols_per_thread; ++col) {
|
||||
#pragma unroll
|
||||
for (int offset = offset_first; offset >= offset_last; offset >>= 1) {
|
||||
KQ_rowsum[col] += __shfl_xor_sync(0xFFFFFFFF, KQ_rowsum[col], offset, warp_size);
|
||||
KQ_rowsum[col] += ggml_cuda_shfl_xor(KQ_rowsum[col], offset, warp_size);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1710,7 +1740,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
#pragma unroll
|
||||
for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) {
|
||||
if (offset < warp_size) {
|
||||
KQ_cmn = fmaxf(KQ_cmn, __shfl_xor_sync(0xFFFFFFFF, KQ_cmn, offset, warp_size));
|
||||
KQ_cmn = fmaxf(KQ_cmn, ggml_cuda_shfl_xor(KQ_cmn, offset, warp_size));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1727,7 +1757,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
#pragma unroll
|
||||
for (int offset = np*cols_per_warp/2; offset >= cols_per_warp; offset >>= 1) {
|
||||
if (offset < warp_size) {
|
||||
KQ_crs += __shfl_xor_sync(0xFFFFFFFF, KQ_crs, offset, warp_size);
|
||||
KQ_crs += ggml_cuda_shfl_xor(KQ_crs, offset, warp_size);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1893,7 +1923,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
|
||||
GGML_UNUSED_VARS(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dstk_fixup,
|
||||
scale, slope, logit_softcap, ne01, ne02, gqa_ratio,
|
||||
stride_Q1, stride_Q2, stride_K, stride_V, stride_mask,
|
||||
jt, kb0_start, kb0_stop);
|
||||
jt, kb0_start, kb0_stop, kb0_max);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE)
|
||||
}
|
||||
@@ -1930,7 +1960,7 @@ static __global__ void flash_attn_ext_f16(
|
||||
const int32_t nb11, const int32_t nb12, const int64_t nb13,
|
||||
const int32_t nb21, const int32_t nb22, const int64_t nb23,
|
||||
const int32_t ne31, const int32_t ne32, const int32_t ne33,
|
||||
const int32_t nb31, const int32_t nb32, const int64_t nb33) {
|
||||
const int32_t nb31, const int64_t nb32, const int64_t nb33) {
|
||||
ggml_cuda_pdl_sync(); // TODO optimize placement
|
||||
#if defined(FLASH_ATTN_AVAILABLE) && (defined(VOLTA_MMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) || defined(AMD_MFMA_AVAILABLE))
|
||||
const char * GGML_CUDA_RESTRICT Q = Q_ptr;
|
||||
@@ -2021,6 +2051,18 @@ static __global__ void flash_attn_ext_f16(
|
||||
int kb0_start = kbc % iter_k;
|
||||
int kb0_stop = min(iter_k, kb0_start + kbc_stop - kbc);
|
||||
|
||||
// Rotate the K/V block order: kbc uses K/V block (kbc - kbc_tile) % iter_k instead of kbc % iter_k, kbc_tile is the start of the CUDA block that started the tile.
|
||||
// Then concurrent CUDA blocks use the same K/V blocks and can share them in L2. Only with multi-stage loading, else they all wait for the same K/V data.
|
||||
constexpr bool rotate = ggml_cuda_fattn_mma_get_nstages(DKQ, DV, ncols1, ncols2, use_sparse) > 1;
|
||||
const int kbc0 = kbc;
|
||||
int kbc_tile = kbc;
|
||||
if (rotate && kb0_start != 0) {
|
||||
// Get the CUDA block that started the tile, inverse of the kbc formula above:
|
||||
const int64_t nkbc = int64_t(iter_k)*iter_j*iter_z_gqa*ne12*ne03;
|
||||
const int64_t b_first = ((kbc - kb0_start + 1)*int64_t(gridDim.x) - 1) / nkbc;
|
||||
kbc_tile = b_first*nkbc / gridDim.x;
|
||||
}
|
||||
|
||||
while (kbc < kbc_stop && kb0_stop == iter_k) {
|
||||
// z_KV == K/V head index, zt_gqa = Q head start index per K/V head, jt = token position start index
|
||||
const int sequence = kbc /(iter_k*iter_j*iter_z_gqa*ne12);
|
||||
@@ -2033,7 +2075,7 @@ static __global__ void flash_attn_ext_f16(
|
||||
const float2 * Q_f2 = (const float2 *) (Q + nb03*sequence + nb02*zt_Q);
|
||||
const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*z_KV);
|
||||
const half * mask_h = ncols2 == 1 && !mask ? nullptr :
|
||||
(const half *) (mask + nb33*(sequence % ne33));
|
||||
(const half *) (mask + nb33*(sequence % ne33) + nb32*(zt_Q % ne32));
|
||||
float2 * dstk = ((float2 *) dst) + (sequence*ne01.z*ne02 + zt_Q) * (DV/2);
|
||||
|
||||
const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);
|
||||
@@ -2043,26 +2085,30 @@ static __global__ void flash_attn_ext_f16(
|
||||
|
||||
const float slope = ncols2 == 1 ? get_alibi_slope(max_bias, zt_Q, n_head_log2, m0, m1) : 1.0f;
|
||||
|
||||
// With rotation kb0_stop is not cut, process_tile skips the fully masked K/V blocks >= kb0_max.
|
||||
if (use_sparse) {
|
||||
kb0_stop = min(kb0_stop, (KV_max[(sequence % ne33)*iter_j + jt] + nbatch_fa - 1) / nbatch_fa);
|
||||
} else if (KV_max) {
|
||||
} else if (KV_max && !rotate) {
|
||||
kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa);
|
||||
}
|
||||
const int kb0_max = rotate && KV_max ? KV_max[sequence*iter_j + jt] / nbatch_fa : iter_k;
|
||||
const int kb0_rot = rotate ? (kbc - kbc_tile) % iter_k : kb0_start;
|
||||
constexpr bool is_fixup = false; // All but (potentially) the last iterations write their data to dst rather than the fixup buffer.
|
||||
if (kb0_start == 0) {
|
||||
constexpr bool needs_fixup = false; // CUDA block is working on an entire tile.
|
||||
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_rot, kb0_rot + kb0_stop - kb0_start, kb0_max);
|
||||
} else {
|
||||
constexpr bool needs_fixup = true; // CUDA block is missing the beginning of a tile.
|
||||
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_rot, kb0_rot + kb0_stop - kb0_start, kb0_max);
|
||||
}
|
||||
|
||||
kbc += iter_k;
|
||||
kbc -= kbc % iter_k;
|
||||
kbc_tile = kbc0;
|
||||
|
||||
kb0_start = 0;
|
||||
kb0_stop = min(iter_k, kbc_stop - kbc);
|
||||
@@ -2083,7 +2129,7 @@ static __global__ void flash_attn_ext_f16(
|
||||
const float2 * Q_f2 = (const float2 *) (Q + nb03*sequence + nb02*zt_Q);
|
||||
const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*z_KV);
|
||||
const half * mask_h = ncols2 == 1 && !mask ? nullptr :
|
||||
(const half *) (mask + nb33*(sequence % ne33));
|
||||
(const half *) (mask + nb33*(sequence % ne33) + nb32*(zt_Q % ne32));
|
||||
float2 * dstk = ((float2 *) dst) + (sequence*ne01.z*ne02 + zt_Q) * (DV/2);
|
||||
|
||||
const half2 * V_h2 = V_is_K_view ? K_h2 : (const half2 *) (V + nb23*sequence + nb22*z_KV);
|
||||
@@ -2095,15 +2141,17 @@ static __global__ void flash_attn_ext_f16(
|
||||
|
||||
if (use_sparse) {
|
||||
kb0_stop = min(kb0_stop, (KV_max[(sequence % ne33)*iter_j + jt] + nbatch_fa - 1) / nbatch_fa);
|
||||
} else if (KV_max) {
|
||||
} else if (KV_max && !rotate) {
|
||||
kb0_stop = min(kb0_stop, KV_max[sequence*iter_j + jt] / nbatch_fa);
|
||||
}
|
||||
const int kb0_max = rotate && KV_max ? KV_max[sequence*iter_j + jt] / nbatch_fa : iter_k;
|
||||
const int kb0_rot = rotate ? (kbc - kbc_tile) % iter_k : kb0_start;
|
||||
|
||||
constexpr bool is_fixup = true; // Last index writes its data to fixup buffer to avoid data races with other blocks.
|
||||
constexpr bool needs_fixup = false;
|
||||
flash_attn_ext_f16_process_tile<DKQ, DV, ncols1, ncols2, nwarps, use_logit_softcap, V_is_K_view, use_sparse, needs_fixup, is_fixup>
|
||||
(Q_f2, K_h2, V_h2, mask_h, indices, sinks_f, dstk, dst_meta, scale, slope, logit_softcap,
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_start, kb0_stop);
|
||||
ne01, ne02, gqa_ratio, ne11, stride_Q1, stride_Q2, stride_K, stride_V, stride_mask, jt, zt_gqa, kb0_rot, kb0_rot + kb0_stop - kb0_start, kb0_max);
|
||||
#else
|
||||
GGML_UNUSED_VARS(Q_ptr, K_ptr, V_ptr, mask_ptr, sinks_ptr, KV_max_ptr, KV_rows_ptr, dst_ptr, dst_meta_ptr, scale,
|
||||
max_bias, m0, m1, n_head_log2, logit_softcap,
|
||||
|
||||
@@ -812,7 +812,7 @@ static __global__ void flash_attn_tile(
|
||||
const int32_t nb11, const int32_t nb12, const int64_t nb13,
|
||||
const int32_t nb21, const int32_t nb22, const int64_t nb23,
|
||||
const int32_t ne31, const int32_t ne32, const int32_t ne33,
|
||||
const int32_t nb31, const int32_t nb32, const int64_t nb33) {
|
||||
const int32_t nb31, const int64_t nb32, const int64_t nb33) {
|
||||
#ifdef FLASH_ATTN_AVAILABLE
|
||||
const char * GGML_CUDA_RESTRICT Q = Q_ptr;
|
||||
const char * GGML_CUDA_RESTRICT K = K_ptr;
|
||||
@@ -859,7 +859,7 @@ static __global__ void flash_attn_tile(
|
||||
const half2 * K_h2 = (const half2 *) (K + nb13*sequence + nb12*(head0 / gqa_ratio));
|
||||
const half2 * V_h2 = (const half2 *) (V + nb23*sequence + nb22*(head0 / gqa_ratio)); // K and V have same shape
|
||||
|
||||
const half * maskh = mask ? (const half *) (mask + nb33*(sequence % ne33)) : nullptr;
|
||||
const half * maskh = mask ? (const half *) (mask + nb33*(sequence % ne33) + nb32*(head0 % ne32)) : nullptr;
|
||||
|
||||
const int stride_K2 = nb11 / sizeof(half2);
|
||||
const int stride_V2 = nb21 / sizeof(half2);
|
||||
@@ -1252,7 +1252,8 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm
|
||||
// However, for DKQ == 576, DV == 512 only the kernel variant with GQA optimizations is implemented.
|
||||
const bool nvidia = GGML_CUDA_CC_IS_NVIDIA(ggml_cuda_info().devices[ggml_cuda_get_device()].cc);
|
||||
const int gqa_limit = nvidia && gqa_ratio <= 4 && DV <= 256 ? 16 : INT_MAX;
|
||||
const bool use_gqa_opt = mask && max_bias == 0.0f && Q->ne[1] <= gqa_limit && K->ne[1] % FATTN_KQ_STRIDE == 0;
|
||||
// a mask with a head dimension is not shared by the Q heads of a K/V head
|
||||
const bool use_gqa_opt = mask && mask->ne[2] == 1 && max_bias == 0.0f && Q->ne[1] <= gqa_limit && K->ne[1] % FATTN_KQ_STRIDE == 0;
|
||||
|
||||
if constexpr (DKQ == 320) {
|
||||
// This branch is only used for Mistral Small 4 which has a GQA ratio of 32.
|
||||
|
||||
@@ -40,7 +40,7 @@ static __global__ void flash_attn_ext_vec(
|
||||
const int32_t nb11, const int32_t nb12, const int64_t nb13,
|
||||
const int32_t nb21, const int32_t nb22, const int64_t nb23,
|
||||
const int32_t ne31, const int32_t ne32, const int32_t ne33,
|
||||
const int32_t nb31, const int32_t nb32, const int64_t nb33) {
|
||||
const int32_t nb31, const int64_t nb32, const int64_t nb33) {
|
||||
ggml_cuda_pdl_lc();
|
||||
#ifdef FLASH_ATTN_AVAILABLE
|
||||
const char * GGML_CUDA_RESTRICT Q = Q_ptr;
|
||||
@@ -112,7 +112,7 @@ static __global__ void flash_attn_ext_vec(
|
||||
K += nb13*sequence + nb12*(head / gqa_ratio);
|
||||
V += nb23*sequence + nb22*(head / gqa_ratio);
|
||||
|
||||
const half * maskh = (const half *) (mask + nb33*(sequence % ne33) + nb31*ic0);
|
||||
const half * maskh = mask ? (const half *) (mask + nb33*(sequence % ne33) + nb32*(head % ne32) + nb31*ic0) : nullptr;
|
||||
|
||||
const float slope = get_alibi_slope(max_bias, head, n_head_log2, m0, m1);
|
||||
|
||||
@@ -316,7 +316,7 @@ static __global__ void flash_attn_ext_vec(
|
||||
for (int j = 0; j < ncols; ++j) {
|
||||
#pragma unroll
|
||||
for (int offset = nthreads_KQ; offset < WARP_SIZE; offset <<= 1) {
|
||||
KQ_max_new[j] = fmaxf(KQ_max_new[j], __shfl_xor_sync(0xFFFFFFFF, KQ_max_new[j], offset, WARP_SIZE));
|
||||
KQ_max_new[j] = fmaxf(KQ_max_new[j], ggml_cuda_shfl_xor(KQ_max_new[j], offset, WARP_SIZE));
|
||||
}
|
||||
const float KQ_max_scale = expf(KQ_max[j] - KQ_max_new[j]);
|
||||
KQ_max[j] = KQ_max_new[j];
|
||||
|
||||
@@ -60,7 +60,7 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
selected |= (!oob || q < q1 - q0) && isfinite(__half2float(mask[q*s31 + i]));
|
||||
}
|
||||
}
|
||||
selected_warp[item] = (lane_mask_t)__ballot_sync(ballot_mask, (int) selected);
|
||||
selected_warp[item] = (lane_mask_t)ggml_cuda_ballot((int) selected, ballot_mask);
|
||||
warp_count += popc(selected_warp[item]);
|
||||
}
|
||||
|
||||
@@ -224,7 +224,8 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols2(ggml_backend_cuda_con
|
||||
|
||||
// Edge cases like no mask, ALiBi, unpadded K/V, or misaligned addresses for large data transfers
|
||||
// are put into the template specialization without GQA optimizations.
|
||||
bool use_gqa_opt = mask && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0;
|
||||
// a mask with a head dimension is not shared by the Q heads of a K/V head
|
||||
bool use_gqa_opt = mask && mask->ne[2] == 1 && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0;
|
||||
for (const ggml_tensor * t : {Q, K, V, mask}) {
|
||||
if (t == nullptr || ggml_is_quantized(t->type)) {
|
||||
continue;
|
||||
@@ -607,8 +608,9 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
|
||||
memcpy(&max_bias, (const float *) KQV->op_params + 1, sizeof(float));
|
||||
|
||||
// The effective batch size for the kernel can be increased by gqa_ratio.
|
||||
// The kernel versions without this optimization are also used for ALiBi, if there is no mask, or if the KV cache is not padded,
|
||||
bool gqa_opt_applies = gqa_ratio >= 2 && mask && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0;
|
||||
// The kernel versions without this optimization are also used for ALiBi, if there is no mask, if the mask has a head dimension,
|
||||
// or if the KV cache is not padded,
|
||||
bool gqa_opt_applies = gqa_ratio >= 2 && mask && mask->ne[2] == 1 && max_bias == 0.0f && n_kv % FATTN_KQ_STRIDE == 0;
|
||||
for (const ggml_tensor * t : {Q, K, V, mask}) {
|
||||
if (t == nullptr || ggml_is_quantized(t->type)) {
|
||||
continue;
|
||||
@@ -677,7 +679,16 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
|
||||
}
|
||||
|
||||
if (mask && mask->ne[2] != 1) {
|
||||
return BEST_FATTN_KERNEL_NONE;
|
||||
// the kv_rows mask is compacted per sequence, it has no head dimension
|
||||
if (kv_rows) {
|
||||
return BEST_FATTN_KERNEL_NONE;
|
||||
}
|
||||
// the logit softcap kernels are compiled for these head sizes only
|
||||
float logit_softcap;
|
||||
memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float));
|
||||
if (logit_softcap != 0.0f && Q->ne[0] != 128 && Q->ne[0] != 256) {
|
||||
return BEST_FATTN_KERNEL_NONE;
|
||||
}
|
||||
}
|
||||
|
||||
// only the MMA kernel reads K/V through kv_rows
|
||||
@@ -722,7 +733,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
|
||||
|
||||
const int ncols2_max = Q->ne[0] == 320 ? 32 : ((Q->ne[0] == 576 || Q->ne[0] == 192) ? 16 : 8);
|
||||
int gqa_ratio_eff = 1;
|
||||
while (max_bias == 0.0f && gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) {
|
||||
while (max_bias == 0.0f && (!mask || mask->ne[2] == 1) && gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) {
|
||||
gqa_ratio_eff *= 2;
|
||||
}
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ __global__ void fwht_cuda(const T * src, float * dst, const int64_t n_rows, cons
|
||||
#pragma unroll
|
||||
for (int j = 0; j < el_w; j++) {
|
||||
const float val = reg[j];
|
||||
const float val2 = __shfl_xor_sync(0xFFFFFFFF, val, h, warp_size);
|
||||
const float val2 = ggml_cuda_shfl_xor(val, h, warp_size);
|
||||
|
||||
reg[j] = (lane & h) == 0 ? val + val2 : val2 - val;
|
||||
}
|
||||
@@ -99,7 +99,7 @@ __global__ void fwht_cuda_block(const T * src, float * dst, const int64_t n_rows
|
||||
#pragma unroll
|
||||
for (int j = 0; j < NE; j++) {
|
||||
const float val = reg[j];
|
||||
const float val2 = __shfl_xor_sync(0xFFFFFFFF, val, h, warp_size);
|
||||
const float val2 = ggml_cuda_shfl_xor(val, h, warp_size);
|
||||
reg[j] = (lane & h) == 0 ? val + val2 : val2 - val;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,11 +9,16 @@ constexpr int D = 128;
|
||||
using bf16 = nv_bfloat16;
|
||||
|
||||
template <int R, int C> struct matrix {
|
||||
bf16 hi[R * C], lo[R * C];
|
||||
#ifdef GGML_USE_HIP
|
||||
static constexpr int alloc = C + 4;
|
||||
#else
|
||||
static constexpr int alloc = C;
|
||||
#endif // GGML_USE_HIP
|
||||
bf16 hi[R * alloc], lo[R * alloc];
|
||||
|
||||
static __device__ __forceinline__ int index(int r, int c) {
|
||||
#ifdef GGML_USE_HIP
|
||||
return r * C + c;
|
||||
return r * alloc + c;
|
||||
#else
|
||||
return 2 * ggml_cuda_mma::swizzle<C / 2, nv_bfloat162>(r * (C / 2) + c / 2, r) + c % 2;
|
||||
#endif // GGML_USE_HIP
|
||||
@@ -56,11 +61,11 @@ template <int C, int V> struct alignas(128) shared {
|
||||
static __device__ __forceinline__ int lower_index(int r, int c) { return r * C + c; }
|
||||
};
|
||||
|
||||
#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
|
||||
#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
|
||||
using namespace ggml_cuda_mma;
|
||||
#ifdef GGML_USE_HIP
|
||||
using acc = tile<16, 16, float, DATA_LAYOUT_J_MAJOR>;
|
||||
using a_tile = tile<16, 8, nv_bfloat162, DATA_LAYOUT_I_MAJOR_MIRRORED>;
|
||||
using a_tile = tile<16, 8, nv_bfloat162, get_input_data_layout()>;
|
||||
using b_tile = a_tile;
|
||||
#else
|
||||
using acc = tile<16, 8, float>;
|
||||
@@ -73,6 +78,23 @@ __device__ __forceinline__ tile_t load(const matrix<R, C> & m, int row, int col,
|
||||
tile_t t;
|
||||
const bf16 * data = low ? m.lo : m.hi;
|
||||
#ifdef GGML_USE_HIP
|
||||
#ifdef AMD_MFMA_AVAILABLE
|
||||
if constexpr (!TRANS) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < tile_t::ne; ++i) {
|
||||
const int r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i);
|
||||
ggml_cuda_memcpy_1<4>(t.x + i, data + m.index(r, c));
|
||||
}
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < tile_t::ne; ++i) {
|
||||
const int r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i);
|
||||
const bf16 x = data[m.index(c, r)];
|
||||
const bf16 y = data[m.index(c + 1, r)];
|
||||
t.x[i] = __float22bfloat162_rn(make_float2(__bfloat162float(x), __bfloat162float(y)));
|
||||
}
|
||||
}
|
||||
#else
|
||||
#pragma unroll
|
||||
for (int i = 0; i < tile_t::ne; ++i) {
|
||||
const int r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i);
|
||||
@@ -80,6 +102,7 @@ __device__ __forceinline__ tile_t load(const matrix<R, C> & m, int row, int col,
|
||||
const bf16 y = data[TRANS ? m.index(c + 1, r) : m.index(r, c + 1)];
|
||||
t.x[i] = __float22bfloat162_rn(make_float2(__bfloat162float(x), __bfloat162float(y)));
|
||||
}
|
||||
#endif // AMD_MFMA_AVAILABLE
|
||||
#else
|
||||
const auto * packed = reinterpret_cast<const nv_bfloat162 *>(data);
|
||||
if constexpr (TRANS) {
|
||||
@@ -118,15 +141,17 @@ template <int R, int C> __device__ __forceinline__ void store(matrix<R, C> & m,
|
||||
}
|
||||
#endif // GGML_USE_HIP
|
||||
}
|
||||
#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
|
||||
#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
|
||||
|
||||
// Each block owns V independent value columns and advances by C tokens.
|
||||
// Keep recurrent state in FP32; pack high/residual BF16 operands inside the block.
|
||||
template <int C, int V, int BLOCKS = 1>
|
||||
static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_mma_args a) {
|
||||
#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
|
||||
static_assert(ggml_cuda_get_physical_warp_size() == 32, "GDN MMA requires 32-lane warps");
|
||||
constexpr int WARPS = 8;
|
||||
#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
static_assert(256 % warp_size == 0, "block must span a whole number of waves");
|
||||
constexpr int warps = 256 / warp_size;
|
||||
constexpr int input_rows = warp_size / 32; // input rows a wave stages per step
|
||||
#ifdef GGML_USE_HIP
|
||||
constexpr int N = 16;
|
||||
#else
|
||||
@@ -134,34 +159,37 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
#endif // GGML_USE_HIP
|
||||
static_assert(C >= 16 && (C & (C - 1)) == 0 && V % N == 0 && D % V == 0, "invalid GDN tile");
|
||||
extern __shared__ __align__(128) unsigned char bytes[];
|
||||
auto & s = *reinterpret_cast<shared<C, V> *>(bytes);
|
||||
const int lane = threadIdx.x, warp = threadIdx.y, tid = warp * 32 + lane;
|
||||
const int slice = blockIdx.x % (D / V);
|
||||
const int h = (blockIdx.x / (D / V)) % a.H;
|
||||
const int seq = blockIdx.x / ((D / V) * a.H);
|
||||
const int v0 = slice * V;
|
||||
const int64_t soff = ((int64_t) seq * a.H + h) * D * D;
|
||||
const int64_t qoff = (seq / a.rq3) * a.sq3 + (h % a.H_k) * a.sq1;
|
||||
const int64_t voff = seq * a.sv3 + h * a.sv1 + v0;
|
||||
const int64_t goff = seq * a.sb3 + h * a.sb1;
|
||||
float * out = a.dst + ((int64_t) seq * a.n_tokens * a.H + h) * D + v0;
|
||||
constexpr int STATE_TILES = (D / 16) * (V / N);
|
||||
constexpr int PER_WARP = STATE_TILES / WARPS;
|
||||
static_assert(STATE_TILES % WARPS == 0, "state must divide across warps");
|
||||
auto & s = *reinterpret_cast<shared<C, V> *>(bytes);
|
||||
const int lane = threadIdx.x;
|
||||
const int warp = threadIdx.y;
|
||||
const int tid = warp * warp_size + lane;
|
||||
const int sub = warp_size > 32 ? lane / 32 : 0;
|
||||
const int slane = warp_size > 32 ? lane % 32 : lane;
|
||||
const int slice = blockIdx.x % (D / V);
|
||||
const int h = (blockIdx.x / (D / V)) % a.H;
|
||||
const int seq = blockIdx.x / ((D / V) * a.H);
|
||||
const int v0 = slice * V;
|
||||
const int64_t soff = ((int64_t) seq * a.H + h) * D * D;
|
||||
const int64_t qoff = (seq / a.rq3) * a.sq3 + (h % a.H_k) * a.sq1;
|
||||
const int64_t voff = seq * a.sv3 + h * a.sv1 + v0;
|
||||
const int64_t goff = seq * a.sb3 + h * a.sb1;
|
||||
float * out = a.dst + ((int64_t) seq * a.n_tokens * a.H + h) * D + v0;
|
||||
constexpr int state_tiles = (D / 16) * (V / N);
|
||||
constexpr int PER_WARP = state_tiles / warps;
|
||||
static_assert(state_tiles % warps == 0, "state must divide across warps");
|
||||
acc state[PER_WARP];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < PER_WARP; ++j) {
|
||||
const int tile_index = warp + j * WARPS, r = (tile_index / (V / N)) * 16, c = (tile_index % (V / N)) * N;
|
||||
const int tile_index = warp + j * warps, r = (tile_index / (V / N)) * 16, c = (tile_index % (V / N)) * N;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
state[j].x[i] = a.state[soff + (v0 + c + acc::get_j(i)) * D + r + acc::get_i(i)];
|
||||
}
|
||||
}
|
||||
auto read_qk = [&](int64_t t, float4 & q, float4 & k) {
|
||||
t += warp;
|
||||
q = k = make_float4(0, 0, 0, 0);
|
||||
if (t < a.n_tokens) {
|
||||
const int64_t off = qoff + t * a.sq2 + 4 * lane;
|
||||
const int64_t off = qoff + t * a.sq2 + 4 * slane;
|
||||
if ((((uintptr_t) (a.q + off) | (uintptr_t) (a.k + off)) & 15) == 0) {
|
||||
q = *reinterpret_cast<const float4 *>(a.q + off);
|
||||
k = *reinterpret_cast<const float4 *>(a.k + off);
|
||||
@@ -172,22 +200,26 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
}
|
||||
};
|
||||
// Carry the first eight input rows across chunks to hide global-load latency.
|
||||
constexpr int QK_ROWS = warps * input_rows;
|
||||
constexpr bool PREFETCH = C == 16 && V == 32;
|
||||
float4 next_q, next_k;
|
||||
float4 next_q;
|
||||
float4 next_k;
|
||||
if constexpr (PREFETCH) {
|
||||
read_qk(0, next_q, next_k);
|
||||
read_qk(warp * input_rows + sub, next_q, next_k);
|
||||
}
|
||||
for (int64_t t0 = 0; t0 < a.n_tokens; t0 += C) {
|
||||
const int valid = min((int64_t) C, a.n_tokens - t0);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < C / 8; ++j) {
|
||||
const int t = warp + 8 * j, d = 4 * lane;
|
||||
float4 q, k;
|
||||
for (int j = 0; j < C / QK_ROWS; ++j) {
|
||||
const int t = warp * input_rows + sub + QK_ROWS * j;
|
||||
const int d = 4 * slane;
|
||||
float4 q;
|
||||
float4 k;
|
||||
if (PREFETCH && j == 0) {
|
||||
q = next_q;
|
||||
k = next_k;
|
||||
} else {
|
||||
read_qk(t0 + 8 * j, q, k);
|
||||
read_qk(t0 + t, q, k);
|
||||
}
|
||||
s.q.store2(t, d, make_float2(q.x * a.scale, q.y * a.scale));
|
||||
s.q.store2(t, d + 2, make_float2(q.z * a.scale, q.w * a.scale));
|
||||
@@ -197,13 +229,13 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
// Load gates in parallel before the FP32 prefix scan.
|
||||
if (warp == 0) {
|
||||
float carry = 0.f;
|
||||
for (int base = 0; base < C; base += 32) {
|
||||
for (int base = 0; base < C; base += warp_size) {
|
||||
const int t = base + lane;
|
||||
float g = t < valid ? a.g[goff + (t0 + t) * a.sb2] : 0.f;
|
||||
const float beta = t < valid ? a.beta[goff + (t0 + t) * a.sb2] : 0.f;
|
||||
#pragma unroll
|
||||
for (int offset = 1; offset < 32; offset *= 2) {
|
||||
const float previous = __shfl_up_sync(0xffffffff, g, offset, 32);
|
||||
for (int offset = 1; offset < warp_size; offset *= 2) {
|
||||
const float previous = ggml_cuda_shfl_up(g, offset, warp_size);
|
||||
if (lane >= offset) {
|
||||
g += previous;
|
||||
}
|
||||
@@ -213,23 +245,23 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
s.prefix[t] = g;
|
||||
s.beta[t] = beta;
|
||||
}
|
||||
carry = __shfl_sync(0xffffffff, g, 31, 32);
|
||||
carry = ggml_cuda_shfl(g, warp_size - 1, warp_size);
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < PER_WARP; ++j) {
|
||||
const int ti = warp + j * WARPS;
|
||||
const int ti = warp + j * warps;
|
||||
store(s.scratch.state, state[j], (ti / (V / N)) * 16, (ti % (V / N)) * N);
|
||||
}
|
||||
__syncthreads();
|
||||
if constexpr (PREFETCH) {
|
||||
if (t0 + C < a.n_tokens) {
|
||||
read_qk(t0 + C, next_q, next_k);
|
||||
read_qk(t0 + C + warp * input_rows + sub, next_q, next_k);
|
||||
}
|
||||
}
|
||||
// V32 overlaps Gram work with one output tile per projection warp.
|
||||
constexpr bool SPLIT_PROJECTION = C == 16 && V == 32;
|
||||
acc base_output[((C / 16) * (V / N) + WARPS - 1) / WARPS];
|
||||
acc base_output[((C / 16) * (V / N) + warps - 1) / warps];
|
||||
// delta = beta * (v - decay * K S), base output = decay * Q S
|
||||
auto project = [&]() {
|
||||
if constexpr (V == 32) {
|
||||
@@ -237,23 +269,26 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
return;
|
||||
}
|
||||
}
|
||||
for (int ti = (V == 32 ? warp - 4 : warp); ti < (C / 16) * (V / N); ti += (V == 32 ? 4 : WARPS)) {
|
||||
const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
acc qs, ks;
|
||||
for (int ti = (V == 32 ? warp - 4 : warp); ti < (C / 16) * (V / N); ti += (V == 32 ? 4 : warps)) {
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
acc qs;
|
||||
acc ks;
|
||||
product<D, false, true>(qs, s.q, s.scratch.state, r, c);
|
||||
product<D, false, true>(ks, s.k, s.scratch.state, r, c);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
const int t = r + acc::get_i(i), v = c + acc::get_j(i);
|
||||
const int t = r + acc::get_i(i);
|
||||
const int v = c + acc::get_j(i);
|
||||
const float decay = expf(s.prefix[t]);
|
||||
const float vv = t < valid ? a.v[voff + (t0 + t) * a.sv2 + v] : 0.f;
|
||||
s.delta.store(t, v, s.beta[t] * (vv - decay * ks.x[i]));
|
||||
base_output[ti / WARPS].x[i] = decay * qs.x[i];
|
||||
base_output[ti / warps].x[i] = decay * qs.x[i];
|
||||
}
|
||||
}
|
||||
};
|
||||
// A = beta * strict_tril(K K^T * decay), P = tril(Q K^T * decay)
|
||||
for (int ti = warp; ti < 2 * (C / 16) * (C / N) && (V != 32 || warp < 4); ti += (V == 32 ? 4 : WARPS)) {
|
||||
for (int ti = warp; ti < 2 * (C / 16) * (C / N) && (V != 32 || warp < 4); ti += (V == 32 ? 4 : warps)) {
|
||||
const bool gram = ti < (C / 16) * (C / N);
|
||||
const int t = ti % ((C / 16) * (C / N)), r = (t / (C / N)) * 16, c = (t % (C / N)) * N;
|
||||
acc x;
|
||||
@@ -280,27 +315,27 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
}
|
||||
__syncthreads();
|
||||
// inverse = (I + A)^-1 by forward substitution
|
||||
for (int row = warp; row < C; row += WARPS) {
|
||||
float x[(C + 31) / 32];
|
||||
for (int row = warp; row < C; row += warps) {
|
||||
float x[(C + warp_size - 1) / warp_size];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < (C + 31) / 32; ++j) {
|
||||
const int col = lane + 32 * j;
|
||||
x[j] = col == row ? 1.f : (col < row ? -s.lower[s.lower_index(row, col)] : 0.f);
|
||||
for (int j = 0; j < (C + warp_size - 1) / warp_size; ++j) {
|
||||
const int col = lane + warp_size * j;
|
||||
x[j] = col == row ? 1.f : (col < row ? -s.lower[s.lower_index(row, col)] : 0.f);
|
||||
}
|
||||
for (int pivot = row - 1; pivot > 0; --pivot) {
|
||||
const float xp = __shfl_sync(0xffffffff, x[pivot / 32], pivot % 32, 32);
|
||||
const float xp = ggml_cuda_shfl(x[pivot / warp_size], pivot % warp_size, warp_size);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < (C + 31) / 32; ++j) {
|
||||
const int col = lane + 32 * j;
|
||||
for (int j = 0; j < (C + warp_size - 1) / warp_size; ++j) {
|
||||
const int col = lane + warp_size * j;
|
||||
if (col < pivot) {
|
||||
x[j] -= xp * s.lower[s.lower_index(pivot, col)];
|
||||
}
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < (C + 31) / 32; ++j) {
|
||||
if (lane + 32 * j < C) {
|
||||
s.inverse.store(row, lane + 32 * j, x[j]);
|
||||
for (int j = 0; j < (C + warp_size - 1) / warp_size; ++j) {
|
||||
if (lane + warp_size * j < C) {
|
||||
s.inverse.store(row, lane + warp_size * j, x[j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -309,8 +344,9 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
}
|
||||
__syncthreads();
|
||||
// solved = inverse * delta
|
||||
for (int ti = warp; ti < (C / 16) * (V / N); ti += WARPS) {
|
||||
const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
for (int ti = warp; ti < (C / 16) * (V / N); ti += warps) {
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
acc x;
|
||||
product<C, false, true>(x, s.inverse, s.delta, r, c);
|
||||
store(s.scratch.solved, x, r, c);
|
||||
@@ -318,21 +354,25 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
__syncthreads();
|
||||
// out = base output + P * solved
|
||||
for (int ti = (SPLIT_PROJECTION ? warp - 4 : warp); ti < (C / 16) * (V / N) && (!SPLIT_PROJECTION || warp >= 4);
|
||||
ti += WARPS) {
|
||||
const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
ti += warps) {
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
acc x;
|
||||
product<C, false, true>(x, s.p, s.scratch.solved, r, c);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
const int t = r + acc::get_i(i), v = c + acc::get_j(i);
|
||||
const int t = r + acc::get_i(i);
|
||||
const int v = c + acc::get_j(i);
|
||||
if (t < valid) {
|
||||
out[(t0 + t) * a.H * D + v] = base_output[ti / WARPS].x[i] + x.x[i];
|
||||
out[(t0 + t) * a.H * D + v] = base_output[ti / warps].x[i] + x.x[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
// S = decay(chunk) * S + K^T * (solved * decay to chunk end)
|
||||
for (int i = tid; i < C * V; i += 256) {
|
||||
const int t = i / V, v = i % V, j = s.scratch.solved.index(t, v);
|
||||
for (int i = tid; i < C * V; i += warps * warp_size) {
|
||||
const int t = i / V;
|
||||
const int v = i % V;
|
||||
const int j = s.scratch.solved.index(t, v);
|
||||
const float x = __bfloat162float(s.scratch.solved.hi[j]) + __bfloat162float(s.scratch.solved.lo[j]);
|
||||
s.delta.store(t, v, x * expf(s.prefix[valid - 1] - s.prefix[t]));
|
||||
}
|
||||
@@ -340,7 +380,9 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
const float decay = expf(s.prefix[valid - 1]);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < PER_WARP; ++j) {
|
||||
const int ti = warp + j * WARPS, r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
const int ti = warp + j * warps;
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
state[j].x[i] *= decay;
|
||||
@@ -351,7 +393,9 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < PER_WARP; ++j) {
|
||||
const int ti = warp + j * WARPS, r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
const int ti = warp + j * warps;
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
a.state_out[soff + (v0 + c + acc::get_j(i)) * D + r + acc::get_i(i)] = state[j].x[i];
|
||||
@@ -360,10 +404,11 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
#else
|
||||
GGML_UNUSED_VARS(a);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
|
||||
#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
|
||||
}
|
||||
|
||||
template <int C, int V, int BLOCKS = 1> bool init(int device) {
|
||||
template <int C, int V, int BLOCKS = 1>
|
||||
static bool init(int device) {
|
||||
static std::once_flag once[GGML_CUDA_MAX_DEVICES];
|
||||
static bool available[GGML_CUDA_MAX_DEVICES] = {};
|
||||
std::call_once(once[device], [device] {
|
||||
@@ -381,7 +426,13 @@ template <int C, int V, int BLOCKS = 1> bool init(int device) {
|
||||
return available[device];
|
||||
}
|
||||
|
||||
enum class gdn_path { ar, slice32_one_block, slice32_two_blocks, slice64, whole_head };
|
||||
enum class gdn_path {
|
||||
ar,
|
||||
slice32_one_block,
|
||||
slice32_two_blocks,
|
||||
slice64,
|
||||
whole_head
|
||||
};
|
||||
|
||||
static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a) {
|
||||
GGML_ASSERT(device >= 0 && device < GGML_CUDA_MAX_DEVICES);
|
||||
@@ -391,10 +442,25 @@ static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a
|
||||
}
|
||||
const auto & info = ggml_cuda_info().devices[device];
|
||||
#ifdef GGML_USE_HIP
|
||||
if (!GGML_CUDA_CC_IS_RDNA3_5(info.cc) || a.n_tokens < 2048) {
|
||||
return gdn_path::ar;
|
||||
if (GGML_CUDA_CC_IS_RDNA3_5(info.cc)) {
|
||||
if (a.n_tokens < 2048) {
|
||||
return gdn_path::ar;
|
||||
}
|
||||
return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar;
|
||||
}
|
||||
return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar;
|
||||
if (GGML_CUDA_CC_IS_CDNA(info.cc)) {
|
||||
if (!amd_mfma_available(info.cc)) {
|
||||
return gdn_path::ar;
|
||||
}
|
||||
const int64_t blocks = a.H * a.n_seqs * 2;
|
||||
const int64_t blocks_nwaves = (blocks + info.nsm - 1) / info.nsm;
|
||||
const int64_t blocks_efficiency = 100 * blocks / (info.nsm * blocks_nwaves);
|
||||
if (blocks_nwaves > 1 && blocks_efficiency < 60) {
|
||||
return gdn_path::ar;
|
||||
}
|
||||
return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar;
|
||||
}
|
||||
return gdn_path::ar;
|
||||
#elif defined(GGML_USE_MUSA)
|
||||
GGML_UNUSED_VARS(info);
|
||||
return gdn_path::ar;
|
||||
@@ -428,7 +494,8 @@ bool ggml_cuda_gdn_mma_launch(int device, const ggml_cuda_gdn_mma_args & a, cuda
|
||||
return false;
|
||||
}
|
||||
#ifdef GGML_USE_HIP
|
||||
gdn_single<16, 64><<<a.H * a.n_seqs * 2, dim3(32, 8), sizeof(shared<16, 64>), stream>>>(a);
|
||||
const int warp_size = ggml_cuda_info().devices[device].warp_size;
|
||||
gdn_single<16, 64><<<a.H * a.n_seqs * 2, dim3(warp_size, 256 / warp_size), sizeof(shared<16, 64>), stream>>>(a);
|
||||
#else
|
||||
if (path == gdn_path::slice32_two_blocks) {
|
||||
gdn_single<16, 32, 2><<<a.H * a.n_seqs * 4, dim3(32, 8), sizeof(shared<16, 32>), stream>>>(a);
|
||||
|
||||
@@ -2526,7 +2526,7 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c
|
||||
} else if (src0->type == GGML_TYPE_F8_E4M3 &&
|
||||
ggml_cuda_should_use_mmvf(src0, cc, warp_size, dst->ne[2])) {
|
||||
return false;
|
||||
} else if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) {
|
||||
} else if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc) && src0->ne[0] % 2 == 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -2570,7 +2570,8 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
|
||||
ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst);
|
||||
return;
|
||||
} else {
|
||||
if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) {
|
||||
// MMVF reads src0 in pairs of values, so it needs an even number of columns
|
||||
if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc) && ne00 % 2 == 0) {
|
||||
ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst);
|
||||
return;
|
||||
}
|
||||
@@ -3635,10 +3636,6 @@ static const struct {
|
||||
};
|
||||
|
||||
static bool ggml_cuda_topk_moe_group_fusion(const struct ggml_cgraph * cgraph, int node_idx, const ggml_tensor * selection, ggml_cuda_topk_moe_args & args) {
|
||||
#if defined(GGML_USE_HIP) && HIP_VERSION < 60300000
|
||||
GGML_UNUSED_VARS(cgraph, node_idx, selection, args);
|
||||
return false;
|
||||
#else
|
||||
const int n_ops = (int) std::size(topk_moe_group_pattern);
|
||||
if (node_idx + n_ops > cgraph->n_nodes) {
|
||||
return false;
|
||||
@@ -3667,7 +3664,6 @@ static bool ggml_cuda_topk_moe_group_fusion(const struct ggml_cgraph * cgraph, i
|
||||
args.n_groups = n_groups;
|
||||
args.n_group_used = n_group_used;
|
||||
return true;
|
||||
#endif // defined(GGML_USE_HIP) && HIP_VERSION < 60300000
|
||||
}
|
||||
|
||||
static bool ggml_cuda_topk_moe_fusion(const struct ggml_cgraph * cgraph, int node_idx, ggml_cuda_topk_moe_args & args) {
|
||||
|
||||
@@ -52,8 +52,8 @@ static __device__ __forceinline__ int ggml_cuda_movmatrix(const int x) {
|
||||
const int shift_low = ((src_j + 0) % 2) * 16;
|
||||
const int shift_high = ((src_j + 1) % 2) * 16;
|
||||
|
||||
const int ret_low = (__shfl_sync(0xFFFFFFFF, x, src_laneid_low, WARP_SIZE) >> shift_low) & 0x0000FFFF;
|
||||
const int ret_high = (__shfl_sync(0xFFFFFFFF, x, src_laneid_high, WARP_SIZE) << shift_high) & 0xFFFF0000;
|
||||
const int ret_low = (ggml_cuda_shfl(x, src_laneid_low, WARP_SIZE) >> shift_low) & 0x0000FFFF;
|
||||
const int ret_high = (ggml_cuda_shfl(x, src_laneid_high, WARP_SIZE) << shift_high) & 0xFFFF0000;
|
||||
|
||||
return ret_low | ret_high;
|
||||
}
|
||||
@@ -736,7 +736,7 @@ namespace ggml_cuda_mma {
|
||||
int i = threadIdx.x / 16;
|
||||
tmp[i] = tile_float.x[l];
|
||||
i ^= 1;
|
||||
tmp[i] = __shfl_xor_sync(0xFFFFFFFF, tile_float.x[l], 16, WARP_SIZE);
|
||||
tmp[i] = ggml_cuda_shfl_xor(tile_float.x[l], 16, WARP_SIZE);
|
||||
ret.x[l] = make_half2(tmp[0], tmp[1]);
|
||||
}
|
||||
return ret;
|
||||
@@ -767,8 +767,8 @@ namespace ggml_cuda_mma {
|
||||
|
||||
// On Volta FP16 and FP32 tiles have a different memory layout,
|
||||
// for the conversion threads with an offset of 2 need to exchange half their values:
|
||||
ret.x[l0/2 + (((threadIdx.x % 4) / 2) ^ 1)] = __shfl_xor_sync(
|
||||
0xFFFFFFFF, ret.x[l0/2 + (((threadIdx.x % 4) / 2) ^ 1)], 2, WARP_SIZE);
|
||||
ret.x[l0/2 + (((threadIdx.x % 4) / 2) ^ 1)] = ggml_cuda_shfl_xor(
|
||||
ret.x[l0/2 + (((threadIdx.x % 4) / 2) ^ 1)], 2, WARP_SIZE);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -88,7 +88,7 @@ static __global__ void mm_ids_helper(
|
||||
int it_compact_add_lower = 0;
|
||||
#pragma unroll
|
||||
for (int offset = neu_padded; offset < warp_size; offset += neu_padded) {
|
||||
const int tmp = __shfl_up_sync(0xFFFFFFFF, it_compact_add_self, offset, warp_size);
|
||||
const int tmp = ggml_cuda_shfl_up(it_compact_add_self, offset, warp_size);
|
||||
if (threadIdx.x >= static_cast<unsigned int>(offset)) {
|
||||
it_compact_add_lower += tmp;
|
||||
}
|
||||
@@ -99,7 +99,7 @@ static __global__ void mm_ids_helper(
|
||||
}
|
||||
|
||||
// The thread with the highest index in the warp always has the sum over the whole warp, use it to increment all threads:
|
||||
it_compact += __shfl_sync(0xFFFFFFFF, it_compact_add_lower + it_compact_add_self, warp_size - 1, warp_size);
|
||||
it_compact += ggml_cuda_shfl(it_compact_add_lower + it_compact_add_self, warp_size - 1, warp_size);
|
||||
}
|
||||
}
|
||||
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
|
||||
|
||||
@@ -1079,8 +1079,14 @@ static __global__ void mul_mat_q(
|
||||
const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);
|
||||
const int wt = tmp2.x;
|
||||
int zt = tmp2.y;
|
||||
#if defined(GGML_USE_HIP)
|
||||
// Row tiles innermost for MoE, see launch_mul_mat_q.
|
||||
const int jt = ids_dst ? blockIdx.y : blockIdx.x;
|
||||
const int it = ids_dst ? blockIdx.x : blockIdx.y;
|
||||
#else
|
||||
const int jt = blockIdx.x;
|
||||
const int it = blockIdx.y;
|
||||
#endif // defined(GGML_USE_HIP)
|
||||
|
||||
// Defaults for regular matrix multiplication:
|
||||
int col_low = jt*J;
|
||||
@@ -1475,16 +1481,20 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
|
||||
|
||||
const dim3 block_dims(warp_size, nwarps, 1);
|
||||
|
||||
CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, /*fallback =*/ false, prec_src1, has_fusion>), nbytes_shared);
|
||||
if constexpr (!has_fusion) {
|
||||
CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, /*fallback =*/ true, prec_src1, false>), nbytes_shared);
|
||||
}
|
||||
CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, fallback, prec_src1, has_fusion>), nbytes_shared);
|
||||
|
||||
const int I_dst = has_fusion ? config.I/2 : config.I; // Rows of dst per tile, see mul_mat_q.
|
||||
const int nti = (args.nrows_x + I_dst - 1) / I_dst;
|
||||
const int ntzw = args.nchannels_y * args.nsamples_y;
|
||||
// Column tiles innermost so that the concurrently running blocks share the same tile of x and read it from L2.
|
||||
const dim3 block_nums_xy_tiling(args.ntj, nti, ntzw);
|
||||
#if defined(GGML_USE_HIP)
|
||||
// For MoE the column tiles are tied to experts, so the column-innermost order does not improve the reuse of x in L2.
|
||||
// It also groups blocks with very different runtimes which schedules poorly on AMD GPUs, see #31599.
|
||||
// Therefore the row tiles are innermost for MoE. The column-innermost order is kept for dense multiplication.
|
||||
const bool rows_innermost = args.ids_dst != nullptr;
|
||||
#else
|
||||
constexpr bool rows_innermost = false;
|
||||
#endif // defined(GGML_USE_HIP)
|
||||
const dim3 block_nums_xy_tiling = rows_innermost ? dim3(nti, args.ntj, ntzw) : dim3(args.ntj, nti, ntzw);
|
||||
|
||||
GGML_ASSERT(args.ids_dst || args.nchannels_y % args.nchannels_x == 0);
|
||||
GGML_ASSERT( args.nsamples_y % args.nsamples_x == 0);
|
||||
@@ -1499,7 +1509,7 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
|
||||
const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio);
|
||||
|
||||
if (has_fusion || !config.stream_k) {
|
||||
GGML_ASSERT(nti <= 65535); // gridDim.y limit
|
||||
GGML_ASSERT((rows_innermost ? args.ntj : nti) <= 65535); // gridDim.y limit
|
||||
mul_mat_q<type, J, fallback, prec_src1, has_fusion><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
|
||||
(args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.tile_spec, args.dst, nullptr,
|
||||
args.y_scale, args.fusion.glu_op, args.fusion.glu_limit, (const float *) args.fusion.x_scale, (const float *) args.fusion.gate_scale,
|
||||
|
||||
@@ -378,6 +378,14 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
|
||||
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
||||
}
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_RDNA3_5(cc)) {
|
||||
switch (type) { // tuned on Radeon 8060S (gfx1151)
|
||||
case GGML_TYPE_Q8_0:
|
||||
return ne11 <= 6;
|
||||
default:
|
||||
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
||||
}
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_CDNA(cc)) {
|
||||
if (GGML_CUDA_CC_IS_CDNA1(cc)) {
|
||||
switch (type) {
|
||||
|
||||
@@ -100,7 +100,7 @@ static __global__ void pool2d_nchw_kernel_warp(
|
||||
|
||||
#pragma unroll
|
||||
for (int offset = WARP_SIZE/2; offset > 0; offset >>= 1) {
|
||||
const To other = __shfl_xor_sync(0xFFFFFFFF, res, offset, WARP_SIZE);
|
||||
const To other = ggml_cuda_shfl_xor(res, offset, WARP_SIZE);
|
||||
res = (op == GGML_OP_POOL_MAX) ? max(res, other) : res + other;
|
||||
}
|
||||
|
||||
|
||||
@@ -389,7 +389,7 @@ static __global__ void quantize_mmq_mxfp4(const float * __restrict__ x,
|
||||
float amax = fabsf(xi);
|
||||
#pragma unroll
|
||||
for (int mask = 16; mask > 0; mask >>= 1) {
|
||||
amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, mask, WARP_SIZE));
|
||||
amax = fmaxf(amax, ggml_cuda_shfl_xor(amax, mask, WARP_SIZE));
|
||||
}
|
||||
|
||||
const uint8_t e = compute_e8m0_scale(amax);
|
||||
@@ -399,10 +399,10 @@ static __global__ void quantize_mmq_mxfp4(const float * __restrict__ x,
|
||||
#if CUDART_VERSION >= 12080
|
||||
const float scaled_val = xi * inv_s;
|
||||
|
||||
const float val0 = __shfl_sync(0xFFFFFFFF, scaled_val, base, WARP_SIZE);
|
||||
const float val1 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 16, WARP_SIZE);
|
||||
const float val2 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 1, WARP_SIZE);
|
||||
const float val3 = __shfl_sync(0xFFFFFFFF, scaled_val, base + 17, WARP_SIZE);
|
||||
const float val0 = ggml_cuda_shfl(scaled_val, base, WARP_SIZE);
|
||||
const float val1 = ggml_cuda_shfl(scaled_val, base + 16, WARP_SIZE);
|
||||
const float val2 = ggml_cuda_shfl(scaled_val, base + 1, WARP_SIZE);
|
||||
const float val3 = ggml_cuda_shfl(scaled_val, base + 17, WARP_SIZE);
|
||||
|
||||
__nv_fp4x4_e2m1 fp4_packed(make_float4(val0, val1, val2, val3));
|
||||
packed[b] = *(char2 *) &fp4_packed;
|
||||
@@ -410,10 +410,10 @@ static __global__ void quantize_mmq_mxfp4(const float * __restrict__ x,
|
||||
// Fallback: manual FP4 conversion using LUT
|
||||
const uint8_t q_val = ggml_cuda_float_to_fp4_e2m1(xi, inv_s);
|
||||
|
||||
const uint8_t q_lo_0 = __shfl_sync(0xFFFFFFFF, q_val, base, WARP_SIZE);
|
||||
const uint8_t q_lo_1 = __shfl_sync(0xFFFFFFFF, q_val, base + 1, WARP_SIZE);
|
||||
const uint8_t q_hi_0 = __shfl_sync(0xFFFFFFFF, q_val, base + 16, WARP_SIZE);
|
||||
const uint8_t q_hi_1 = __shfl_sync(0xFFFFFFFF, q_val, base + 17, WARP_SIZE);
|
||||
const uint8_t q_lo_0 = ggml_cuda_shfl(q_val, base, WARP_SIZE);
|
||||
const uint8_t q_lo_1 = ggml_cuda_shfl(q_val, base + 1, WARP_SIZE);
|
||||
const uint8_t q_hi_0 = ggml_cuda_shfl(q_val, base + 16, WARP_SIZE);
|
||||
const uint8_t q_hi_1 = ggml_cuda_shfl(q_val, base + 17, WARP_SIZE);
|
||||
|
||||
char2 q;
|
||||
q.x = (q_hi_0 << 4) | q_lo_0;
|
||||
@@ -497,7 +497,7 @@ static __global__ void quantize_mmq_q8_1(
|
||||
// Exchange max. abs. value between vals_per_scale/4 threads.
|
||||
#pragma unroll
|
||||
for (int offset = vals_per_scale/8; offset > 0; offset >>= 1) {
|
||||
amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE));
|
||||
amax = fmaxf(amax, ggml_cuda_shfl_xor(amax, offset, WARP_SIZE));
|
||||
}
|
||||
|
||||
const float d_inv = 127.0f / amax;
|
||||
@@ -515,7 +515,7 @@ static __global__ void quantize_mmq_q8_1(
|
||||
// Calculate sums across vals_per_sum/4 threads.
|
||||
#pragma unroll
|
||||
for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) {
|
||||
sum_q += __shfl_xor_sync(0xFFFFFFFF, sum_q, offset, WARP_SIZE);
|
||||
sum_q += ggml_cuda_shfl_xor(sum_q, offset, WARP_SIZE);
|
||||
}
|
||||
|
||||
sum = d * float(sum_q);
|
||||
|
||||
+160
-76
@@ -142,7 +142,7 @@ static __global__ void soft_max_f32(
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Template to allow keeping ncols in registers if they fit
|
||||
template <int n_elem_per_thread = 0>
|
||||
static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __restrict__ x,
|
||||
float * __restrict__ dst,
|
||||
float * __restrict__ tmp_maxs,
|
||||
@@ -154,98 +154,160 @@ static __device__ void soft_max_f32_parallelize_cols_single_row(const float * __
|
||||
|
||||
const cg::grid_group g = cg::this_grid();
|
||||
|
||||
const int tid = threadIdx.x;
|
||||
const int col_start = blockIdx.x * blockDim.x + tid;
|
||||
const int n_elem_per_thread = 4;
|
||||
const int tid = threadIdx.x;
|
||||
const int col_start = blockIdx.x * blockDim.x + tid;
|
||||
const int step_size = gridDim.x * blockDim.x;
|
||||
|
||||
float local_vals[n_elem_per_thread] = { -INFINITY, -INFINITY, -INFINITY, -INFINITY };
|
||||
float local_max = -INFINITY;
|
||||
const int step_size = gridDim.x * blockDim.x;
|
||||
if constexpr (n_elem_per_thread > 0) {
|
||||
float local_vals[n_elem_per_thread];
|
||||
float local_max = -INFINITY;
|
||||
|
||||
// Compute thread-local max
|
||||
for (int col = col_start; col < p.ncols;) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < n_elem_per_thread; i++) {
|
||||
const int idx = col + i * step_size;
|
||||
const int idx = col_start + i * step_size;
|
||||
local_vals[i] = idx < p.ncols ? x[idx] : -INFINITY;
|
||||
local_max = fmaxf(local_max, local_vals[i]);
|
||||
}
|
||||
|
||||
local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals_max);
|
||||
|
||||
if (tid == 0) {
|
||||
tmp_maxs[blockIdx.x] = local_max;
|
||||
}
|
||||
g.sync();
|
||||
|
||||
assert(gridDim.x < blockDim.x);
|
||||
if (tid < gridDim.x) {
|
||||
local_max = tmp_maxs[tid];
|
||||
} else {
|
||||
local_max = -INFINITY;
|
||||
}
|
||||
local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals_max);
|
||||
|
||||
float tmp_expf = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < n_elem_per_thread; i++) {
|
||||
local_max = fmaxf(local_max, local_vals[i]);
|
||||
}
|
||||
col += step_size * n_elem_per_thread;
|
||||
}
|
||||
|
||||
// Compute CTA-level max
|
||||
local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals_max);
|
||||
|
||||
// Store CTA-level max to GMEM
|
||||
if (tid == 0) {
|
||||
tmp_maxs[blockIdx.x] = local_max;
|
||||
}
|
||||
g.sync();
|
||||
|
||||
// Compute compute global max from CTA-level maxs
|
||||
assert(gridDim.x < blockDim.x); // currently we only support this case
|
||||
if (tid < gridDim.x) {
|
||||
local_max = tmp_maxs[tid];
|
||||
} else {
|
||||
local_max = -INFINITY;
|
||||
}
|
||||
local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals_max);
|
||||
|
||||
// Compute softmax dividends, accumulate divisor
|
||||
float tmp_expf = 0.0f;
|
||||
for (int col = col_start; col < p.ncols;) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < n_elem_per_thread; i++) {
|
||||
const int idx = col + i * step_size;
|
||||
local_vals[i] = idx < p.ncols ? x[idx] : -INFINITY;
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < n_elem_per_thread; i++) {
|
||||
const int idx = col + i * step_size;
|
||||
const int idx = col_start + i * step_size;
|
||||
if (idx < p.ncols) {
|
||||
const float tmp = expf(local_vals[i] - local_max);
|
||||
tmp_expf += tmp;
|
||||
dst[idx] = tmp;
|
||||
local_vals[i] = tmp;
|
||||
tmp_expf += tmp;
|
||||
}
|
||||
}
|
||||
col += step_size * n_elem_per_thread;
|
||||
}
|
||||
|
||||
// Reduce divisor within CTA
|
||||
tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals_sum);
|
||||
tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals_sum);
|
||||
|
||||
// Store CTA-level sum to GMEM
|
||||
if (tid == 0) {
|
||||
tmp_sums[blockIdx.x] = tmp_expf;
|
||||
}
|
||||
g.sync();
|
||||
|
||||
// Compute global sum from CTA-level sums
|
||||
if (tid < gridDim.x) {
|
||||
tmp_expf = tmp_sums[tid];
|
||||
} else {
|
||||
tmp_expf = 0.0f;
|
||||
}
|
||||
tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals_sum);
|
||||
|
||||
// Divide dividend by global sum + store data
|
||||
for (int col = col_start; col < p.ncols;) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < n_elem_per_thread; i++) {
|
||||
const int idx = col + i * step_size;
|
||||
local_vals[i] = idx < p.ncols ? dst[idx] : -INFINITY;
|
||||
if (tid == 0) {
|
||||
tmp_sums[blockIdx.x] = tmp_expf;
|
||||
}
|
||||
g.sync();
|
||||
|
||||
if (tid < gridDim.x) {
|
||||
tmp_expf = tmp_sums[tid];
|
||||
} else {
|
||||
tmp_expf = 0.0f;
|
||||
}
|
||||
tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals_sum);
|
||||
|
||||
const float inv_sum = 1.0f / tmp_expf;
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < n_elem_per_thread; i++) {
|
||||
const int idx = col + i * step_size;
|
||||
const int idx = col_start + i * step_size;
|
||||
if (idx < p.ncols) {
|
||||
dst[idx] = local_vals[i] / tmp_expf;
|
||||
dst[idx] = local_vals[i] * inv_sum;
|
||||
}
|
||||
}
|
||||
col += step_size * n_elem_per_thread;
|
||||
} else {
|
||||
const int n_elem = 4;
|
||||
float local_vals[n_elem] = { -INFINITY, -INFINITY, -INFINITY, -INFINITY };
|
||||
float local_max = -INFINITY;
|
||||
|
||||
// Compute thread-local max
|
||||
for (int col = col_start; col < p.ncols;) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < n_elem; i++) {
|
||||
const int idx = col + i * step_size;
|
||||
local_vals[i] = idx < p.ncols ? x[idx] : -INFINITY;
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < n_elem; i++) {
|
||||
local_max = fmaxf(local_max, local_vals[i]);
|
||||
}
|
||||
col += step_size * n_elem;
|
||||
}
|
||||
|
||||
// Compute CTA-level max
|
||||
local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals_max);
|
||||
|
||||
// Store CTA-level max to GMEM
|
||||
if (tid == 0) {
|
||||
tmp_maxs[blockIdx.x] = local_max;
|
||||
}
|
||||
g.sync();
|
||||
|
||||
// Compute compute global max from CTA-level maxs
|
||||
assert(gridDim.x < blockDim.x); // currently we only support this case
|
||||
if (tid < gridDim.x) {
|
||||
local_max = tmp_maxs[tid];
|
||||
} else {
|
||||
local_max = -INFINITY;
|
||||
}
|
||||
local_max = block_reduce<block_reduce_method::MAX>(local_max, shared_vals_max);
|
||||
|
||||
// Compute softmax dividends, accumulate divisor
|
||||
float tmp_expf = 0.0f;
|
||||
for (int col = col_start; col < p.ncols;) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < n_elem; i++) {
|
||||
const int idx = col + i * step_size;
|
||||
local_vals[i] = idx < p.ncols ? x[idx] : -INFINITY;
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < n_elem; i++) {
|
||||
const int idx = col + i * step_size;
|
||||
if (idx < p.ncols) {
|
||||
const float tmp = expf(local_vals[i] - local_max);
|
||||
tmp_expf += tmp;
|
||||
dst[idx] = tmp;
|
||||
}
|
||||
}
|
||||
col += step_size * n_elem;
|
||||
}
|
||||
|
||||
// Reduce divisor within CTA
|
||||
tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals_sum);
|
||||
|
||||
// Store CTA-level sum to GMEM
|
||||
if (tid == 0) {
|
||||
tmp_sums[blockIdx.x] = tmp_expf;
|
||||
}
|
||||
g.sync();
|
||||
|
||||
// Compute global sum from CTA-level sums
|
||||
if (tid < gridDim.x) {
|
||||
tmp_expf = tmp_sums[tid];
|
||||
} else {
|
||||
tmp_expf = 0.0f;
|
||||
}
|
||||
tmp_expf = block_reduce<block_reduce_method::SUM>(tmp_expf, shared_vals_sum);
|
||||
|
||||
// Divide dividend by global sum + store data
|
||||
for (int col = col_start; col < p.ncols;) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < n_elem; i++) {
|
||||
const int idx = col + i * step_size;
|
||||
local_vals[i] = idx < p.ncols ? dst[idx] : -INFINITY;
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < n_elem; i++) {
|
||||
const int idx = col + i * step_size;
|
||||
if (idx < p.ncols) {
|
||||
dst[idx] = local_vals[i] / tmp_expf;
|
||||
}
|
||||
}
|
||||
col += step_size * n_elem;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -305,6 +367,7 @@ static void launch_soft_max_kernels(const float * x, const T * mask, const float
|
||||
soft_max_f32<true, 0, 0><<<block_nums, block_dims, nbytes_shared, stream>>>(x, mask, sinks, dst, p);
|
||||
}
|
||||
|
||||
template <int n_elem_per_thread = 0>
|
||||
__launch_bounds__(8*WARP_SIZE, 1) static __global__ void soft_max_f32_parallelize_cols(const float * __restrict__ x,
|
||||
float * __restrict__ dst,
|
||||
float * __restrict__ tmp_maxs,
|
||||
@@ -319,8 +382,8 @@ __launch_bounds__(8*WARP_SIZE, 1) static __global__ void soft_max_f32_paralleliz
|
||||
__shared__ float shared_vals[2][32];
|
||||
|
||||
for (int rowx = 0; rowx < p.ne01 * p.ne02 * p.ne03; rowx++) {
|
||||
soft_max_f32_parallelize_cols_single_row(x + int64_t(rowx) * p.ncols, dst + int64_t(rowx) * p.ncols, tmp_maxs,
|
||||
tmp_sums, shared_vals[0], shared_vals[1], p);
|
||||
soft_max_f32_parallelize_cols_single_row<n_elem_per_thread>(x + int64_t(rowx) * p.ncols, dst + int64_t(rowx) * p.ncols, tmp_maxs,
|
||||
tmp_sums, shared_vals[0], shared_vals[1], p);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -360,7 +423,28 @@ static void soft_max_f32_cuda(const float * x,
|
||||
|
||||
void * kernel_args[] = { (void *) &x, (void *) &dst, (void *) &tmp_maxs_alloc.ptr,
|
||||
(void *) &tmp_sums_alloc.ptr, (void *) const_cast<soft_max_params *>(¶ms) };
|
||||
CUDA_CHECK(cudaLaunchCooperativeKernel((void *) soft_max_f32_parallelize_cols,
|
||||
|
||||
const int total_threads = ggml_cuda_info().devices[id].nsm * (WARP_SIZE * 8);
|
||||
const int required_n_elem = (params.ncols + total_threads - 1) / total_threads;
|
||||
|
||||
void * kernel_fn = nullptr;
|
||||
if (required_n_elem <= 1) {
|
||||
kernel_fn = (void *) soft_max_f32_parallelize_cols<1>;
|
||||
} else if (required_n_elem <= 2) {
|
||||
kernel_fn = (void *) soft_max_f32_parallelize_cols<2>;
|
||||
} else if (required_n_elem <= 4) {
|
||||
kernel_fn = (void *) soft_max_f32_parallelize_cols<4>;
|
||||
} else if (required_n_elem <= 8) {
|
||||
kernel_fn = (void *) soft_max_f32_parallelize_cols<8>;
|
||||
} else if (required_n_elem <= 16) {
|
||||
kernel_fn = (void *) soft_max_f32_parallelize_cols<16>;
|
||||
} else if (required_n_elem <= 32) {
|
||||
kernel_fn = (void *) soft_max_f32_parallelize_cols<32>;
|
||||
} else {
|
||||
kernel_fn = (void *) soft_max_f32_parallelize_cols<0>;
|
||||
}
|
||||
|
||||
CUDA_CHECK(cudaLaunchCooperativeKernel(kernel_fn,
|
||||
dim3(ggml_cuda_info().devices[id].nsm, 1, 1),
|
||||
dim3(WARP_SIZE * 8, 1, 1), kernel_args, 0, stream));
|
||||
} else {
|
||||
|
||||
@@ -184,10 +184,7 @@ static __global__ void top_k_radix_gather(
|
||||
constexpr int NBINS = 1 << RADIX_BITS;
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
constexpr int nwarps = BLOCK_SIZE / warp_size;
|
||||
using lane_mask_t = std::conditional_t<warp_size == 64, uint64_t, uint32_t>;
|
||||
// HIP requires a 64-bit mask also for a 32-wide warp
|
||||
constexpr uint64_t ballot_mask = ~(lane_mask_t) 0;
|
||||
const auto popc = [](const lane_mask_t mask) {
|
||||
const auto popc = [](const ggml_cuda_thread_mask_t mask) {
|
||||
if constexpr (warp_size == 64) {
|
||||
return __popcll(mask);
|
||||
} else {
|
||||
@@ -249,8 +246,8 @@ static __global__ void top_k_radix_gather(
|
||||
const int64_t col = tile + lane;
|
||||
const bool is_tie = col < warp_end && top_k_float_to_ordered(row_src[col]) == threshold;
|
||||
|
||||
const lane_mask_t ties = (lane_mask_t) __ballot_sync(ballot_mask, is_tie);
|
||||
const int tie_pos = pos + popc(ties & (((lane_mask_t) 1 << lane) - 1));
|
||||
const ggml_cuda_thread_mask_t ties = ggml_cuda_ballot(is_tie);
|
||||
const int tie_pos = pos + popc(ties & (((ggml_cuda_thread_mask_t) 1 << lane) - 1));
|
||||
if (is_tie && tie_pos < pos_end) {
|
||||
row_dst[k - rank + tie_pos] = col;
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user