mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-16 15:57:37 -06:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e8fbcad855 | ||
|
|
95e837aaf1 | ||
|
|
d998c676ff | ||
|
|
92192d31e4 | ||
|
|
ad5cbfa911 | ||
|
|
d261e49e7e | ||
|
|
4ddf2b9470 | ||
|
|
4303cd94ae | ||
|
|
c7518dbec5 | ||
|
|
a68c6136bb | ||
|
|
3a7660371d | ||
|
|
e8698ac163 | ||
|
|
7804be0634 | ||
|
|
ef6bc6f367 | ||
|
|
a06044b934 | ||
|
|
62a6d3a302 | ||
|
|
ef3848f9d7 | ||
|
|
47fe133318 | ||
|
|
301db63370 | ||
|
|
a001af915e | ||
|
|
164ab018f9 | ||
|
|
5cb9516798 | ||
|
|
8bfd64b5ac | ||
|
|
e1925b3695 | ||
|
|
95db855bce | ||
|
|
1d2e880bad | ||
|
|
adf554d239 | ||
|
|
6b01a4767f | ||
|
|
f477c1b90a | ||
|
|
21fa9c043f | ||
|
|
a946eb541a | ||
|
|
05db09a4d3 | ||
|
|
9fba51d2c9 | ||
|
|
b4c0e00a4f | ||
|
|
1719dbb0ee | ||
|
|
fc504f91d8 | ||
|
|
18b74fa41d | ||
|
|
7ae3c6b231 | ||
|
|
f4d0673917 |
@@ -19,12 +19,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${CANN_BASE_IMAGE} AS build
|
||||
@@ -40,7 +40,7 @@ WORKDIR /app
|
||||
# -- Copy project files --
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
# -- Set CANN environment variables (required for compilation) --
|
||||
# Using ENV instead of `source` allows environment variables to persist across the entire image layer
|
||||
|
||||
@@ -9,12 +9,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -28,7 +28,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON; \
|
||||
|
||||
@@ -17,12 +17,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_CUDA_DEV_CONTAINER} AS build
|
||||
@@ -40,7 +40,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "${CUDA_DOCKER_ARCH}" != "default" ]; then \
|
||||
export CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CUDA_DOCKER_ARCH}"; \
|
||||
|
||||
@@ -11,12 +11,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build
|
||||
@@ -41,7 +41,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "${GGML_SYCL_F16}" = "ON" ]; then \
|
||||
echo "GGML_SYCL_F16 is set" \
|
||||
|
||||
@@ -15,12 +15,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_MUSA_DEV_CONTAINER} AS build
|
||||
@@ -45,7 +45,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "${MUSA_DOCKER_ARCH}" != "default" ]; then \
|
||||
export CMAKE_ARGS="-DMUSA_ARCHITECTURES=${MUSA_DOCKER_ARCH}"; \
|
||||
|
||||
@@ -133,11 +133,11 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
src = lib.cleanSource ../../.;
|
||||
};
|
||||
|
||||
# Builds the webui locally, taking care not to require updating any sha256 hash.
|
||||
# Builds Llama Chat locally, taking care not to require updating any sha256 hash.
|
||||
webui = stdenvNoCC.mkDerivation {
|
||||
pname = "webui";
|
||||
version = llamaVersion;
|
||||
src = lib.cleanSource ../../tools/ui;
|
||||
src = lib.cleanSource ../../ui/apps/chat;
|
||||
|
||||
nativeBuildInputs = [
|
||||
nodejs
|
||||
@@ -146,7 +146,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
|
||||
# no sha256 required when using buildNodeModules
|
||||
npmDeps = importNpmLock.buildNodeModules {
|
||||
npmRoot = ../../tools/ui;
|
||||
npmRoot = ../../ui/apps/chat;
|
||||
inherit nodejs;
|
||||
};
|
||||
|
||||
@@ -156,8 +156,8 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
};
|
||||
|
||||
postPatch = lib.optionalString useWebUi ''
|
||||
cp -r ${finalAttrs.webui} tools/ui/dist
|
||||
chmod -R u+w tools/ui/dist
|
||||
cp -r ${finalAttrs.webui} ui/apps/chat/dist
|
||||
chmod -R u+w ui/apps/chat/dist
|
||||
'';
|
||||
|
||||
# With PR#6015 https://github.com/ggml-org/llama.cpp/pull/6015,
|
||||
|
||||
@@ -28,12 +28,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
## Build Image
|
||||
@@ -83,7 +83,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
# Build Stage
|
||||
RUN bash -c "source ${OpenVINO_DIR}/setupvars.sh && \
|
||||
|
||||
@@ -17,12 +17,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
### Build image
|
||||
@@ -52,7 +52,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \
|
||||
cmake -S . -B build \
|
||||
|
||||
@@ -9,12 +9,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -31,7 +31,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN cmake -B build -DGGML_NATIVE=OFF -DGGML_VULKAN=ON -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON && \
|
||||
cmake --build build --config Release -j$(nproc)
|
||||
|
||||
@@ -9,12 +9,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -28,7 +28,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_ZENDNN=ON && \
|
||||
cmake --build build -j $(nproc)
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
|
||||
build*/
|
||||
|
||||
tools/ui/node_modules/
|
||||
ui/apps/chat/node_modules/
|
||||
|
||||
models/*
|
||||
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ insert_final_newline = unset
|
||||
[vendor/**.patch]
|
||||
trim_trailing_whitespace = unset
|
||||
|
||||
[tools/ui/**]
|
||||
[ui/**]
|
||||
indent_style = unset
|
||||
indent_size = unset
|
||||
end_of_line = unset
|
||||
|
||||
@@ -14,7 +14,9 @@ runs:
|
||||
run: |
|
||||
BUILD_NUMBER="$(git rev-list --count HEAD)"
|
||||
SHORT_HASH="$(git rev-parse --short=7 HEAD)"
|
||||
if [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
if [[ "$GITHUB_REF_TYPE" == "tag" && "${{ env.BRANCH_NAME }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "name=${{ env.BRANCH_NAME }}" >> $GITHUB_OUTPUT
|
||||
elif [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
|
||||
|
||||
+1
-2
@@ -48,7 +48,6 @@ examples:
|
||||
- all-globs-to-all-files:
|
||||
- '!tools/server/**'
|
||||
- '!tools/mtmd/**'
|
||||
- '!tools/ui/**'
|
||||
testing:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
@@ -72,7 +71,7 @@ android:
|
||||
server/ui:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- tools/ui/**
|
||||
- ui/**
|
||||
server:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
|
||||
@@ -16,6 +16,10 @@ on:
|
||||
description: "Skip the s390x build target (useful for fast test runs that do not need the IBM Z runner)"
|
||||
type: boolean
|
||||
default: false
|
||||
skip_ref_check:
|
||||
description: "Skip the requirement to run from a tag (allows branch refs)"
|
||||
type: boolean
|
||||
default: false
|
||||
schedule:
|
||||
# Rebuild daily rather than on every push because it is expensive
|
||||
- cron: '12 4 * * *'
|
||||
@@ -38,6 +42,14 @@ jobs:
|
||||
source_tag: ${{ steps.srctag.outputs.name }}
|
||||
|
||||
steps:
|
||||
- name: Check manual run ref
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.skip_ref_check }}
|
||||
run: |
|
||||
if [[ "$GITHUB_REF_TYPE" != "tag" ]]; then
|
||||
echo "::error::Manual runs of this workflow must be started from a tag"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
@@ -170,7 +182,7 @@ jobs:
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
# - name: Set up QEMU
|
||||
# if: ${{ contains(matrix.config.platforms, 'linux/amd64') }}
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
BODY: ${{ github.event.pull_request.body }}
|
||||
HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'video demo') }}
|
||||
run: |
|
||||
# sections written by demo:pr (see tools/ui/demo/cli.ts): the managed comment
|
||||
# sections written by demo:pr (see ui/apps/chat/demo/cli.ts): the managed comment
|
||||
# markers, or a preview linking to a demo video uploaded for this PR
|
||||
has_demo=false
|
||||
if { grep -qF '<!-- demo:start -->' <<< "$BODY" && grep -qF '<!-- demo:end -->' <<< "$BODY"; } ||
|
||||
|
||||
@@ -130,7 +130,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -196,7 +196,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -274,7 +274,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -381,7 +381,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -479,7 +479,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v5
|
||||
@@ -571,7 +571,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -624,7 +624,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -672,7 +672,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -894,7 +894,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Install Ninja
|
||||
run: |
|
||||
@@ -1460,7 +1460,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -1532,7 +1532,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Free up disk space
|
||||
uses: ggml-org/free-disk-space@v1.3.1
|
||||
@@ -1647,7 +1647,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
|
||||
@@ -7,6 +7,40 @@ on:
|
||||
description: 'Commit SHA1 to build'
|
||||
required: false
|
||||
type: string
|
||||
spec_methods:
|
||||
description: 'Speculative methods to test'
|
||||
required: true
|
||||
default: 'compact'
|
||||
type: choice
|
||||
options:
|
||||
- compact
|
||||
- nightly
|
||||
- all
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/server-speculative.yml',
|
||||
'common/speculative.*',
|
||||
'src/models/dflash*.cpp',
|
||||
'src/models/dspark.cpp',
|
||||
'src/models/eagle3.cpp',
|
||||
'src/models/lilicorr.cpp',
|
||||
'src/models/qwen35.cpp',
|
||||
'tools/server/tests/unit/test_spec_methods.py'
|
||||
]
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/server-speculative.yml',
|
||||
'common/speculative.*',
|
||||
'src/models/dflash*.cpp',
|
||||
'src/models/dspark.cpp',
|
||||
'src/models/eagle3.cpp',
|
||||
'src/models/lilicorr.cpp',
|
||||
'src/models/qwen35.cpp',
|
||||
'tools/server/tests/unit/test_spec_methods.py'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
@@ -56,8 +90,9 @@ jobs:
|
||||
|
||||
- name: Test speculative decoding
|
||||
env:
|
||||
SPEC_METHODS: compact
|
||||
SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || format('{0}/spec-models', runner.temp) }}
|
||||
SPEC_METHODS: ${{ github.event.inputs.spec_methods || 'compact' }}
|
||||
# pull requests from forks do not get repository variables
|
||||
SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || '/mnt/cache/spec-models' }}
|
||||
N_GPU_LAYERS: 99
|
||||
run: |
|
||||
cd tools/server/tests
|
||||
|
||||
@@ -23,20 +23,20 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build application
|
||||
run: npm run build
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Upload built UI
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
retention-days: 1
|
||||
|
||||
@@ -37,26 +37,26 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build application
|
||||
env:
|
||||
LLAMA_BUILD_NUMBER: ${{ steps.version.outputs.ui_version }}
|
||||
run: npm run build
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run PWA unit tests (versioned build output)
|
||||
run: npx vitest --project=unit --run tests/unit/pwa.spec.ts
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Upload built UI
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
retention-days: 1
|
||||
|
||||
@@ -33,15 +33,15 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
run-id: ${{ inputs.run_id }}
|
||||
github-token: ${{ github.token }}
|
||||
|
||||
- name: Create distribution archive
|
||||
run: |
|
||||
tar -czf dist.tar.gz -C tools/ui/dist .
|
||||
tar -czf dist.tar.gz -C ui/apps/chat/dist .
|
||||
sha256sum dist.tar.gz > dist.tar.gz.sha256
|
||||
mv dist.tar.gz dist.tar.gz.sha256 tools/ui/dist/
|
||||
mv dist.tar.gz dist.tar.gz.sha256 ui/apps/chat/dist/
|
||||
|
||||
- name: Install Hugging Face Hub CLI
|
||||
run: pip install -U huggingface_hub
|
||||
@@ -52,12 +52,12 @@ jobs:
|
||||
- name: Sync built files to Hugging Face bucket (version tag)
|
||||
run: |
|
||||
# Upload the built files to the Hugging Face bucket under the release version
|
||||
hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet
|
||||
hf buckets sync ui/apps/chat/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet
|
||||
|
||||
- name: Sync built files to Hugging Face bucket (latest)
|
||||
run: |
|
||||
# Also upload to the 'latest' directory for fallback downloads
|
||||
hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet
|
||||
hf buckets sync ui/apps/chat/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet
|
||||
|
||||
- name: Verify upload
|
||||
run: |
|
||||
|
||||
@@ -17,7 +17,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -25,7 +25,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
@@ -67,33 +67,33 @@ jobs:
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Run type checking
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run linting
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run lint
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Client tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:client
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Unit tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:unit
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
@@ -109,25 +109,25 @@ jobs:
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Build Storybook
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run build-storybook
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run UI tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:ui -- --testTimeout=60000
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run E2E tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:e2e
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
+17
-17
@@ -13,7 +13,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -21,7 +21,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
@@ -66,46 +66,46 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
if: ${{ steps.node.conclusion == 'success' }}
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run type checking
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run linting
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run lint
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npx playwright install --with-deps
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Client tests
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:client
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Unit tests (uses pre-built dist/ from ui-build)
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:unit
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
@@ -124,38 +124,38 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
if: ${{ steps.node.conclusion == 'success' }}
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Download built UI artifacts (reuses ui-build)
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npx playwright install --with-deps
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build Storybook
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run build-storybook
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run UI tests
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:ui -- --testTimeout=60000
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run E2E tests (uses pre-built dist/ from ui-build)
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:e2e
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# date: Wed Nov 4 12:03:31 EET 2026
|
||||
# date: Mon Nov 16 10:05:56 EET 2026
|
||||
# this file is auto-generated by scripts/gen-authors.sh
|
||||
|
||||
Нияз Гарифзянов <112617865+garrnizon@users.noreply.github.com>
|
||||
@@ -43,7 +43,9 @@ Aarni Koskela <akx@iki.fi>
|
||||
Aaron Miller <apage43@ninjawhale.com>
|
||||
Aaron Teo <57927438+taronaeo@users.noreply.github.com>
|
||||
Aaron Teo <aaron.teo1@ibm.com>
|
||||
Aaron Thomas <84493696+arxhr007@users.noreply.github.com>
|
||||
Aaryaman Vasishta <aaryaman.vasishta@amd.com>
|
||||
Aayan Karmali <44980366+StockerMC@users.noreply.github.com>
|
||||
Abheek Gulati <abheekg@hotmail.com>
|
||||
abhijain1204fujitsu <139222713+abhijain1204fujitsu@users.noreply.github.com>
|
||||
Abhijit Ramesh <abhijitramesh2k@gmail.com>
|
||||
@@ -112,10 +114,12 @@ Aleksander Grygier <admin@serveurperso.com>
|
||||
Aleksander Grygier <aleksander.grygier@gmail.com>
|
||||
Aleksander Grygier <thichthat@gmail.com>
|
||||
Aleksei Nikiforov <103434461+AlekseiNikiforovIBM@users.noreply.github.com>
|
||||
Alen Požun <kalyx.solutions@proton.me>
|
||||
Alessandro de Oliveira Faria (A.K.A.CABELO) <cabelo@opensuse.org>
|
||||
Alessandro98-git <61804547+Alessandro98-git@users.noreply.github.com>
|
||||
Alex <18387287+wadealexc@users.noreply.github.com>
|
||||
Alex <59368173+AlexGabbia@users.noreply.github.com>
|
||||
Alex <alexzkhr@gmail.com>
|
||||
Alex <awhill19@icloud.com>
|
||||
Alex Azarov <alex@azarov.by>
|
||||
Alex Azarov <alexander.azarov@mapbox.com>
|
||||
@@ -132,6 +136,7 @@ Alex Trotta <44127594+Ahajha@users.noreply.github.com>
|
||||
Alex Tuddenham <61622354+AlexsCode@users.noreply.github.com>
|
||||
Alex von Gluck IV <kallisti5@unixzen.com>
|
||||
Alex Wu <dindinw@users.noreply.github.com>
|
||||
Alex-JP-93 <Jianpeng.Li@amd.com>
|
||||
alex-spacemit <jinghui.huang@spacemit.com>
|
||||
Alexander Batischev <eual.jp@gmail.com>
|
||||
Alexander Heisler <126129661+heislera763@users.noreply.github.com>
|
||||
@@ -143,6 +148,7 @@ Alexis Williams <typedrat@users.noreply.github.com>
|
||||
alexpinel <93524949+alexpinel@users.noreply.github.com>
|
||||
Alfred <zxu3@clemson.edu>
|
||||
Ali Chraghi <63465728+alichraghi@users.noreply.github.com>
|
||||
Ali F. Al-Ramadan <55710790+aalramadan@users.noreply.github.com>
|
||||
Ali Nehzat <ali.nehzat@thanks.dev>
|
||||
Ali Tariq <ali.tariq@10xengineers.ai>
|
||||
Ali Tariq <alitariq4589@gmail.com>
|
||||
@@ -158,6 +164,7 @@ Aman Karki <amankarki151@gmail.com>
|
||||
Aman Karki <itsamankarki@gmail.com>
|
||||
amd-dwang <dong.wang@amd.com>
|
||||
amd-lalithnc <lalithnc@amd.com>
|
||||
AMIR <127450356+amirthn6533@users.noreply.github.com>
|
||||
Amir <amir_zia@outlook.com>
|
||||
amirai21 <89905406+amirai21@users.noreply.github.com>
|
||||
AmirAli Mirian <37371367+amiralimi@users.noreply.github.com>
|
||||
@@ -213,6 +220,7 @@ Antoine Viallon <antoine@lesviallon.fr>
|
||||
Anton I <inflyer@hotmail.com>
|
||||
Anton Mitkov <anton_b_mitkov@abv.bg>
|
||||
Anton Mitkov <anton.mitkov@codeplay.com>
|
||||
Anton Sokolchenko <wsevendays@gmail.com>
|
||||
Antonis Makropoulos <benuix@gmail.com>
|
||||
anubhavfujitsu <Anubhav.Srivastava@fujitsu.com>
|
||||
Anudit Nagar <nagaranudit@gmail.com>
|
||||
@@ -233,10 +241,12 @@ arlo-phoenix <140345165+arlo-phoenix@users.noreply.github.com>
|
||||
Armen Kaleshian <kriation@users.noreply.github.com>
|
||||
Arsen Arutunan <58118221+limloop@users.noreply.github.com>
|
||||
Arseniy Obolenskiy <good.doog.2012@gmail.com>
|
||||
ArshM17-NV <amaknojia@nvidia.com>
|
||||
Artem <guinmoon@gmail.com>
|
||||
Artem Zinnatullin <ceo@abstractny.gay>
|
||||
Artyom Lebedev <vagran.ast@gmail.com>
|
||||
aryantandon01 <80969509+aryantandon01@users.noreply.github.com>
|
||||
Asahi-Prv <246092086+Asahi-Prv@users.noreply.github.com>
|
||||
Asahi-Prv <asahihori7@gmail.com>
|
||||
asbelin <71226900+asbelin@users.noreply.github.com>
|
||||
Asbjørn Olling <asbjornolling@gmail.com>
|
||||
@@ -258,6 +268,7 @@ AustinMroz <austinmroz@utexas.edu>
|
||||
AUTOMATIC1111 <16777216c@gmail.com>
|
||||
automaticcat <daogiatuank54@gmail.com>
|
||||
awatuna <23447591+awatuna@users.noreply.github.com>
|
||||
Awss <gitj.18@alt.awss1i.com>
|
||||
b4b4o <zwbao@foxmail.com>
|
||||
Bach Le <bach@bullno1.com>
|
||||
BADR <contact@pythops.com>
|
||||
@@ -351,6 +362,7 @@ CarterLi999 <664681047@qq.com>
|
||||
Casey Primozic <casey@cprimozic.net>
|
||||
Casey Primozic <me@ameo.link>
|
||||
CausalLM <148736309+CausalLM@users.noreply.github.com>
|
||||
CC <65151926+realMoana@users.noreply.github.com>
|
||||
ccbinn <ccbinn@163.com>
|
||||
cduk <19917266+cduk@users.noreply.github.com>
|
||||
cebtenzzre <cebtenzzre@gmail.com>
|
||||
@@ -379,6 +391,7 @@ Cheng Shao <terrorjack@type.dance>
|
||||
Chenguang Li <757486878@qq.com>
|
||||
Chenguang Li <87689256+noemotiovon@users.noreply.github.com>
|
||||
chiheb ben cheikh <100644963+ChihebBENCHEIKH1@users.noreply.github.com>
|
||||
chinmaykamjith001 <chinmaykamjith@gmail.com>
|
||||
Chipmunk <101038159+CHIPMUNK-T0T@users.noreply.github.com>
|
||||
chiranko <96988916+chiranko@users.noreply.github.com>
|
||||
Chris Danis <cdanis@gmail.com>
|
||||
@@ -434,6 +447,7 @@ crasm <crasm@git.vczf.us>
|
||||
crat0z <11581854+crat0z@users.noreply.github.com>
|
||||
CRD716 <crd716@gmail.com>
|
||||
CrispStrobe <154636388+CrispStrobe@users.noreply.github.com>
|
||||
Cristian Adam <cristian.adam@gmail.com>
|
||||
Cristiano Pinto <140563307+crowmoed@users.noreply.github.com>
|
||||
crsawyer <7572190+crsawyer@users.noreply.github.com>
|
||||
Csaba Kecskemeti <csaba.kecskemeti@gmail.com>
|
||||
@@ -527,6 +541,7 @@ Djip007 <3705339+Djip007@users.noreply.github.com>
|
||||
Djip007 <djip.perois@free.fr>
|
||||
dm4 <dm4@secondstate.io>
|
||||
dm4 <sunrisedm4@gmail.com>
|
||||
Dmitrii Panchev <dmnobunaga@users.noreply.github.com>
|
||||
Dmitry Atamanov <data-man@users.noreply.github.com>
|
||||
Dmytro Minochkin <dmytro.minochkin@gmail.com>
|
||||
Dmytro Romanov <casteldazur@gmail.com>
|
||||
@@ -567,6 +582,7 @@ Ed Addario <29247825+EAddario@users.noreply.github.com>
|
||||
Ed Lee <edilee@mozilla.com>
|
||||
Ed Lepedus <ed.lepedus@googlemail.com>
|
||||
Eddie-Wang <wangjinheng1120@163.com>
|
||||
eddiechen <Echen1246@gmail.com>
|
||||
Eden <aa9736195201@gmail.com>
|
||||
eduardopessin <100053075+eduardopessin@users.noreply.github.com>
|
||||
Edward Taylor <edeetee@gmail.com>
|
||||
@@ -598,6 +614,7 @@ Erik Winter <93544977+ewintr@users.noreply.github.com>
|
||||
Ervin Áron Tasnádi <etasnadi@protonmail.com>
|
||||
Esko Toivonen <eskot98@gmail.com>
|
||||
Ethan Guo <ethanguo.dev@gmail.com>
|
||||
Ethan Lin <linsen458@gmail.com>
|
||||
Ethan Turner <eturner64@gmail.com>
|
||||
Ettore Di Giacinto <mudler@users.noreply.github.com>
|
||||
EugeoSynthesisThirtyTwo <gabriel.dhimoila@gmail.com>
|
||||
@@ -625,6 +642,7 @@ fanyang <fanyang89@outlook.com>
|
||||
Farbod Bijary <110523279+farbodbj@users.noreply.github.com>
|
||||
Fathi Boudra <fathi.boudra@linaro.org>
|
||||
Fattire <528174+fat-tire@users.noreply.github.com>
|
||||
Feiteng <lifeiteng0422@gmail.com>
|
||||
felix <felix314159@users.noreply.github.com>
|
||||
Felix <stenbackfelix@gmail.com>
|
||||
Felix Ye <felixyjs@gmail.com>
|
||||
@@ -632,6 +650,7 @@ fengerhu1 <2748250768@qq.com>
|
||||
feni6 <fenissova@proton.me>
|
||||
fidoriel <49869342+fidoriel@users.noreply.github.com>
|
||||
fiesh <fiesh@zefix.tv>
|
||||
figkcoci-debug <figkcoci@gmail.com>
|
||||
FINAL-Bench <kkms1116@koreacu.ac.kr>
|
||||
Finn Voorhees <finnvoorhees@gmail.com>
|
||||
Firat <firatkiral@gmail.com>
|
||||
@@ -646,6 +665,7 @@ Foad Abo Dahood <32059146+masterFoad@users.noreply.github.com>
|
||||
Folko-Ven <71110216+Folko-Ven@users.noreply.github.com>
|
||||
forforever73 <63285796+forforever73@users.noreply.github.com>
|
||||
Foul-Tarnished <107711110+Foul-Tarnished@users.noreply.github.com>
|
||||
fox5611 <66219952+fox5611@users.noreply.github.com>
|
||||
Francisco Herrera <ppaanncchhoo507@gmail.com>
|
||||
Francisco Melo <43780565+francis2tm@users.noreply.github.com>
|
||||
Francois Dugast <francois.dugast@intel.com>
|
||||
@@ -709,6 +729,7 @@ GiviMAD <GiviMAD@users.noreply.github.com>
|
||||
gliptic <gliptic@users.noreply.github.com>
|
||||
gn64 <yukikaze.jp@gmail.com>
|
||||
goerch <jhr.walter@t-online.de>
|
||||
Gotya-OFFCIAL <zhengfreeman@gmail.com>
|
||||
Govlzkoy <gotope@users.noreply.github.com>
|
||||
grahameth <96447521+grahameth@users.noreply.github.com>
|
||||
Gregor Jasny <gjasny@googlemail.com>
|
||||
@@ -749,6 +770,7 @@ Haowei Wu <breadcyanide@icloud.com>
|
||||
Haoxiang Fei <tonyfettes@tonyfettes.com>
|
||||
Harald Fernengel <harald.fernengel@here.com>
|
||||
Harapan Rachman <harapanrachman@gmail.com>
|
||||
Harkanwal Bedi <bedi1887@gmail.com>
|
||||
Harkirat Gill <harkirat.gill@amd.com>
|
||||
HarrisonSec <gzxharrison@gmail.com>
|
||||
Hatsune Miku <129688334+at8u@users.noreply.github.com>
|
||||
@@ -768,6 +790,7 @@ Henrik Forstén <henrik.forsten@gmail.com>
|
||||
Henry Linjamäki <henry.linjamaki@gmail.com>
|
||||
Henry Linjamäki <henry.mikael.linjamaki@intel.com>
|
||||
Henry147147 <44851451+Henry147147@users.noreply.github.com>
|
||||
Hera <58063257+hera2019@users.noreply.github.com>
|
||||
Herman Semenoff <GermanAizek@yandex.ru>
|
||||
Herman Semenov <GermanAizek@yandex.ru>
|
||||
Hesen Peng <hesen.peng@gmail.com>
|
||||
@@ -846,6 +869,7 @@ Imad Saddik <79410781+ImadSaddik@users.noreply.github.com>
|
||||
iMil <imil@NetBSD.org>
|
||||
Incarnas <119618389+bit-incarnas@users.noreply.github.com>
|
||||
InflexCZE <martin.pavlicek11@gmail.com>
|
||||
Iñigo Imaz <49730431+inimaz@users.noreply.github.com>
|
||||
Intel AI Get-to Market Customer Success and Solutions <ai.gtm.css@gmail.com>
|
||||
intelmatt <61025942+intelmatt@users.noreply.github.com>
|
||||
iohub <rickyang.pro@gmail.com>
|
||||
@@ -857,7 +881,9 @@ IsaacDynamo <61521674+IsaacDynamo@users.noreply.github.com>
|
||||
Ishaan Gandhi <Ishaangandhi@gmail.com>
|
||||
iSma <ismail.senhaji@gmail.com>
|
||||
Ismail <115064057+AlrIsmail@users.noreply.github.com>
|
||||
IsNot <209144869@qq.com>
|
||||
issixx <46835150+issixx@users.noreply.github.com>
|
||||
Iswarya Alex <47045679+iswaryaalex@users.noreply.github.com>
|
||||
itsnotoger <19309683+itsnotoger@users.noreply.github.com>
|
||||
itterative <190138728+itterative@users.noreply.github.com>
|
||||
Ivan <nekotekina@gmail.com>
|
||||
@@ -888,6 +914,7 @@ James Francis <6763899+JamesFranc@users.noreply.github.com>
|
||||
James O'Leary <65884233+jpohhhh@users.noreply.github.com>
|
||||
James Reynolds <magnusviri@users.noreply.github.com>
|
||||
jameswu2014 <545426914@qq.com>
|
||||
JamieBordoni <jamie.bordoni@protonmail.com>
|
||||
Jan Boon <jan.boon@kaetemi.be>
|
||||
Jan Boon <kaetemi@gmail.com>
|
||||
Jan Ekström <jeebjp@gmail.com>
|
||||
@@ -956,8 +983,10 @@ Jinyang He <hejinyang@loongson.cn>
|
||||
jinzihao <jinzihao1996@gmail.com>
|
||||
Jiří Podivín <66251151+jpodivin@users.noreply.github.com>
|
||||
Jiří Sejkora <Sejseloid@gmail.com>
|
||||
Jiří Svítil <32205472+mistrjirka@users.noreply.github.com>
|
||||
Jiwoong Song <55657581+jwsong98@users.noreply.github.com>
|
||||
JJJYmmm <92386084+JJJYmmm@users.noreply.github.com>
|
||||
jjperry69 <jjperry69@gmail.com>
|
||||
jklincn <985765408@qq.com>
|
||||
jklincn <jklincn@outlook.com>
|
||||
JM Robles <roblesjm@gmail.com>
|
||||
@@ -970,6 +999,7 @@ Joe Rowell <joerowell4@gmail.com>
|
||||
Joe Todd <joe.todd@codeplay.com>
|
||||
joecryptotoo <80373433+joecryptotoo@users.noreply.github.com>
|
||||
Joel <agnel.joel@gmail.com>
|
||||
Joel Fernandes <joelagnelf@nvidia.com>
|
||||
Johan <JohanAR@users.noreply.github.com>
|
||||
Johannes Gäßler <johannesg@5d6.de>
|
||||
Johannes Rudolph <johannes.rudolph@gmail.com>
|
||||
@@ -988,6 +1018,7 @@ Jonas J <111707981+John-194@users.noreply.github.com>
|
||||
Jonas Jankaitis <111707981+John-194@users.noreply.github.com>
|
||||
Jonas Wunderlich <32615971+jonas-w@users.noreply.github.com>
|
||||
Jonathan <47618606+jbuchananr@users.noreply.github.com>
|
||||
Jonathan Buck <jonathan.j.buck@gmail.com>
|
||||
Jonathan Clohessy <jonathan.clohessy@arm.com>
|
||||
Jonathan Graehl <99024+graehl@users.noreply.github.com>
|
||||
Jorge A <161275481+jorgealias@users.noreply.github.com>
|
||||
@@ -1097,6 +1128,7 @@ klosax <131523366+klosax@users.noreply.github.com>
|
||||
KnightYao <KnightYao@users.noreply.github.com>
|
||||
KokerZhou <111279477+KokerZhou@users.noreply.github.com>
|
||||
Kolen Cheung <ickc@users.noreply.github.com>
|
||||
Kong Jiale <2049447146@qq.com>
|
||||
kononnable <kononnable@gmail.com>
|
||||
Konrad Moren <kmoren@nvidia.com>
|
||||
konradmb <konradmb@o2.pl>
|
||||
@@ -1167,6 +1199,7 @@ li-lizhe <82501526+li-lizhe@users.noreply.github.com>
|
||||
liangliangchang <lichang@amd.com>
|
||||
liminfei-amd <91481003+liminfei-amd@users.noreply.github.com>
|
||||
limitedAtonement <limitedAtonement@users.noreply.github.com>
|
||||
Lines <linellazatin@gmail.com>
|
||||
lingyezhixing <144504450+lingyezhixing@users.noreply.github.com>
|
||||
linsen458-spec <linsen458@gmail.com>
|
||||
Linux User <linuxusergd@outlook.com>
|
||||
@@ -1227,6 +1260,7 @@ makomk <makosoft@googlemail.com>
|
||||
manayang <jackmanayang@gmail.com>
|
||||
manikbhandari <mbbhandarimanik2@gmail.com>
|
||||
Manuel <44313466+makuche@users.noreply.github.com>
|
||||
ManuelPDM <153872964+ManuelPDM@users.noreply.github.com>
|
||||
maor-ps <154728172+maor-ps@users.noreply.github.com>
|
||||
Maphist0 <28743569+Maphist0@users.noreply.github.com>
|
||||
Marc Köhlbrugge <subscriptions@marckohlbrugge.com>
|
||||
@@ -1336,6 +1370,7 @@ Michael Hueschen <m@mhueschen.dev>
|
||||
Michael Kesper <mkesper@schokokeks.org>
|
||||
Michael Klimenko <mklimenko29@gmail.com>
|
||||
Michael Lamothe <michael.lamothe@gmail.com>
|
||||
Michael Paulauski <mike10010100@gmail.com>
|
||||
Michael Podvitskiy <podvitskiymichael@gmail.com>
|
||||
Michael Potter <NanoTekGuy@Gmail.com>
|
||||
Michael Taylor <162068037+mctylr-gh@users.noreply.github.com>
|
||||
@@ -1455,6 +1490,7 @@ Nikolay Popov <131475237+npopov-vst@users.noreply.github.com>
|
||||
Nils Gladitz <nilsgladitz@gmail.com>
|
||||
Nindaleth <Nindaleth@users.noreply.github.com>
|
||||
ningshanwutuobang <ningshanwutuobang@gmail.com>
|
||||
Nisbenz <benazzaanis783@gmail.com>
|
||||
Noah <99681487+NoahOksuz@users.noreply.github.com>
|
||||
nold <Nold360@users.noreply.github.com>
|
||||
nopperl <54780682+nopperl@users.noreply.github.com>
|
||||
@@ -1462,6 +1498,7 @@ nullname <chraac@gmail.com>
|
||||
Nuno <rare-magma@posteo.eu>
|
||||
nuri <yoonuri1@gmail.com>
|
||||
nusu-github <29514220+nusu-github@users.noreply.github.com>
|
||||
nv-vankit <162085734+nv-vankit@users.noreply.github.com>
|
||||
nwyin <tommynguyen0512@gmail.com>
|
||||
o7si <32285332+o7si@users.noreply.github.com>
|
||||
Oğuzhan Akkaya <oakkaya@ymail.com>
|
||||
@@ -1485,6 +1522,7 @@ oobabooga <oobabooga4@gmail.com>
|
||||
opparco <parco.opaai@gmail.com>
|
||||
Ori Pekelman <ori@pekelman.com>
|
||||
Oscar Barenys <rtfss1@gmail.com>
|
||||
Oscar M. Herrera <oscar@kss-inc.com>
|
||||
OSecret <135510162+OLSecret@users.noreply.github.com>
|
||||
ostix360 <55257054+ostix360@users.noreply.github.com>
|
||||
Ouadie EL FAROUKI <ouadie.elfarouki@codeplay.com>
|
||||
@@ -1611,6 +1649,7 @@ Raj Hammeer Singh Hada <hammeerraj@gmail.com>
|
||||
Rajendra Matcha <matcraje@qti.qualcomm.com>
|
||||
Ralph Soika <ralph.soika@imixs.com>
|
||||
Raman Shinde <raman.shinde15@gmail.com>
|
||||
Ramzi <146260554+rmz-24@users.noreply.github.com>
|
||||
Rand Xie <randxiexyy29@gmail.com>
|
||||
Randall Fitzgerald <randall@dasaku.net>
|
||||
Random Fly <renfei8@live.cn>
|
||||
@@ -1651,6 +1690,7 @@ Rich Dougherty <rich@rd.nz>
|
||||
Richard <r-burton@hotmail.co.uk>
|
||||
Richard Davison <richard.davison1@gmail.com>
|
||||
Richard Kiss <him@richardkiss.com>
|
||||
Richard Palethorpe <io@richiejp.com>
|
||||
Richard Roberson <richardr1126@gmail.com>
|
||||
RichardScottOZ <rnmscott@netspace.net.au>
|
||||
Rick G <26732651+TheFlipbook@users.noreply.github.com>
|
||||
@@ -1695,6 +1735,7 @@ Roni <sulpher@gmx.net>
|
||||
Ronny Brendel <ronnyb@nvidia.com>
|
||||
Ronny Brendel <ronnybrendel@gmail.com>
|
||||
Ronsor <ronsor@ronsor.pw>
|
||||
Roshan Kumar <52444089+roshankumarg529@users.noreply.github.com>
|
||||
Rotem Dan <rotemdan@gmail.com>
|
||||
Rowan Hart <rowanbhart@gmail.com>
|
||||
rspOverflow <217881046+rspOverflow@users.noreply.github.com>
|
||||
@@ -1727,9 +1768,11 @@ Saba Fallah <10401143+sfallah@users.noreply.github.com>
|
||||
Saba Fallah <sabafallah@gmail.com>
|
||||
Sachin Desai <smdesai@gmail.com>
|
||||
Sachin Sharma <sachin@zettabolt.com>
|
||||
Saejin Kim-Heinert <saejinmh@gmail.com>
|
||||
Safi Ullah <safiullah.3915@gmail.com>
|
||||
safranowith <bsh155762@gmail.com>
|
||||
Said Taghadouini <84044788+staghado@users.noreply.github.com>
|
||||
SaidSultani <62264302+SaidSultani@users.noreply.github.com>
|
||||
Sait Furkan Teke <35101659+stfurkan@users.noreply.github.com>
|
||||
SakuraUmi <yukinon244@gmail.com>
|
||||
Salvador E. Tropea <stropea@inti.gob.ar>
|
||||
@@ -1754,6 +1797,7 @@ Sang-Kil Park <sang.park@42dot.ai>
|
||||
Sanjay Ahari <sanjayahari1704@gmail.com>
|
||||
Sarah Wu <sarahwu185@gmail.com>
|
||||
Sarah Wu <sarwu@nvidia.com>
|
||||
sargentmajord <jasonlather13@gmail.com>
|
||||
Sascha Rogmann <59577610+srogmann@users.noreply.github.com>
|
||||
sasha0552 <admin@sasha0552.org>
|
||||
Satinder Grewal <grewal.satinder@gmail.com>
|
||||
@@ -1786,6 +1830,7 @@ Sergiu <8598216+mzsergiu@users.noreply.github.com>
|
||||
serhii-nakon <57632032+serhii-nakon@users.noreply.github.com>
|
||||
Sertaç Özercan <852750+sozercan@users.noreply.github.com>
|
||||
seryogakovalyov <seryogakovalyov@gmail.com>
|
||||
Seungjun Bae <linkedlatch@gmail.com>
|
||||
Seungmin Kim <8457324+ehfd@users.noreply.github.com>
|
||||
SeungWon Jeong <65549245+redlion0929@users.noreply.github.com>
|
||||
Seyoung Jeong <seyoungjeong@gmail.com>
|
||||
@@ -1871,6 +1916,7 @@ SmartestWashingMachine <ottobizness@gmail.com>
|
||||
smugman-dot <wbsmoke101@gmail.com>
|
||||
SnA1lGo <44647694+skrandy@users.noreply.github.com>
|
||||
snadampal <87143774+snadampal@users.noreply.github.com>
|
||||
Sofian Elmotiem <sofianelmotiem@gmail.com>
|
||||
SoftwareRenderer <138734813+SoftwareRenderer@users.noreply.github.com>
|
||||
Someone <sergei.kozlukov@aalto.fi>
|
||||
Someone Serge <sergei.kozlukov@aalto.fi>
|
||||
@@ -1886,6 +1932,7 @@ sprayandwipe <spray.and.wipe@gmail.com>
|
||||
Sreeram <sreeram.sivasubramony@amd.com>
|
||||
SRHMorris <69468379+SRHMorris@users.noreply.github.com>
|
||||
Srihari-mcw <96763064+Srihari-mcw@users.noreply.github.com>
|
||||
Srijan Keshri <212402043+arcusbuilds@users.noreply.github.com>
|
||||
Srinivas Billa <nivibilla@gmail.com>
|
||||
srkizer <github.soreepeong.prqovzj1d@srkizer.net>
|
||||
ssweens <1149151+ssweens@users.noreply.github.com>
|
||||
@@ -1919,6 +1966,7 @@ Sutou Kouhei <kou@cozmixng.org>
|
||||
Svetlozar Georgiev <55534064+sgeor255@users.noreply.github.com>
|
||||
Swigler <124839156+Swigler@users.noreply.github.com>
|
||||
swittk <switt1995@gmail.com>
|
||||
sxq <18856096915@163.com>
|
||||
SXX <song_xiaoxi@126.com>
|
||||
SXX <sxx1136965276@gmail.com>
|
||||
Tai Duc Nguyen <taiducnguyen.drexel@gmail.com>
|
||||
@@ -2044,6 +2092,7 @@ Ulrich Drepper <drepper@gmail.com>
|
||||
unbounded <haakon@likedan.net>
|
||||
Unbug Lee <tidelgl@gmail.com>
|
||||
unraido <127105806+unraido@users.noreply.github.com>
|
||||
usrlocalben <benjamin@rqdq.com>
|
||||
uvos <carl@uvos.xyz>
|
||||
uvos <devnull@uvos.xyz>
|
||||
uvos <philipp@uvos.xyz>
|
||||
@@ -2084,6 +2133,7 @@ vk <89937361+itsvedantkumar@users.noreply.github.com>
|
||||
Vlad <spitfireage@gmail.com>
|
||||
Vladimir <bogdad@gmail.com>
|
||||
Vladimir Malyutin <first-leon@yandex.ru>
|
||||
Vladimir Sotnikov <bubert9@gmail.com>
|
||||
Vladimir Vuksanovic <109677816+vvuksanovic@users.noreply.github.com>
|
||||
Vladimir Zorin <vladimir@deviant.guru>
|
||||
Vladislav <vladplotnikov34@gmail.com>
|
||||
@@ -2139,6 +2189,7 @@ Wroclaw <wroclaw223@outlook.com>
|
||||
wsbagnsv1 <sclumpfpapa36@gmail.com>
|
||||
Wu Jian Ping <wujjpp@hotmail.com>
|
||||
Wu Jian Ping <wujp@greatld.com>
|
||||
WuXin <apuravmanhas123@gmail.com>
|
||||
wwoodsTM <104587230+wwoodsTM@users.noreply.github.com>
|
||||
Wyatt Caldwell <218154709+Detensable@users.noreply.github.com>
|
||||
wzy <32936898+Freed-Wu@users.noreply.github.com>
|
||||
@@ -2162,6 +2213,7 @@ Xinpeng Dou <15529241576@163.com>
|
||||
Xinpeng Dou <81913537+Dou-Git@users.noreply.github.com>
|
||||
xloem <0xloem@gmail.com>
|
||||
xris99 <79798089+xris99@users.noreply.github.com>
|
||||
Xuan Son Nguyen <son@huggingface.co>
|
||||
Xuan Son Nguyen <thichthat@gmail.com>
|
||||
Xuan-Son Nguyen <son@huggingface.co>
|
||||
Xuan-Son Nguyen <thichthat@gmail.com>
|
||||
@@ -2247,6 +2299,7 @@ zhangrunda <zhangrunda1234@outlook.com>
|
||||
zhangtao2-1 <478679312@qq.com>
|
||||
ZHAOKAI WANG <sanxianwei@163.com>
|
||||
Zhaolun Yin <129580161+ZhaolunYin@users.noreply.github.com>
|
||||
zhaoyshine <30611065+zhaoyshine@users.noreply.github.com>
|
||||
Zheng.Deng <32841220+dengzheng-cloud@users.noreply.github.com>
|
||||
zhentaoyu <zhentao.yu@intel.com>
|
||||
Zhenwei Jin <109658203+kylo5aby@users.noreply.github.com>
|
||||
|
||||
+1
-1
@@ -134,7 +134,7 @@ option(LLAMA_BUILD_TOOLS "llama: build tools"
|
||||
option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_SERVER "llama: build server example" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_APP "llama: build the unified binary" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_UI "llama: build the embedded Web UI for server" OFF)
|
||||
option(LLAMA_BUILD_UI "llama: build Llama Chat, the embedded chat app, for server" OFF)
|
||||
option(LLAMA_USE_PREBUILT_UI "llama: use prebuilt UI from HF Bucket when available" ON)
|
||||
option(LLAMA_CONNECT "llama: include the llama-connect tool (downloads a prebuilt binary)" OFF)
|
||||
|
||||
|
||||
+1
-1
@@ -102,7 +102,7 @@
|
||||
/tools/rpc/ @ggml-org/ggml-rpc
|
||||
/tools/server/* @ggml-org/llama-server # no subdir
|
||||
/tools/server/tests/ @ggml-org/llama-server
|
||||
/tools/ui/ @ggml-org/llama-ui
|
||||
/ui/ @ggml-org/llama-ui
|
||||
/tools/tokenize/ @ggerganov
|
||||
/tools/tts/ @ggerganov
|
||||
/vendor/ @ggerganov
|
||||
|
||||
@@ -16,6 +16,8 @@ The project differentiates between 3 levels of contributors:
|
||||
>
|
||||
> Detailed information regarding permissible and restricted uses of AI can be found in the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
### Requirements
|
||||
|
||||
If AI is used to generate any portion of the code, contributors must adhere to the following requirements:
|
||||
|
||||
1. Explicitly disclose the manner in which AI was employed.
|
||||
@@ -24,6 +26,16 @@ If AI is used to generate any portion of the code, contributors must adhere to t
|
||||
4. Be prepared to explain every line of code you submit when asked about it by a maintainer.
|
||||
5. It is strictly prohibited to use AI to write your posts for you (bug reports, feature requests, pull request descriptions, Github discussions, responding to humans, ...).
|
||||
|
||||
### Checklist
|
||||
|
||||
> [!IMPORTANT]
|
||||
>
|
||||
> **If you use AI, do all of the following:**
|
||||
>
|
||||
> 1. **Make sure [AGENTS.md](AGENTS.md) is loaded into your agent's context.** Skipping this step is one of the most common reasons PRs get rejected.
|
||||
> 2. **Search for related discussions, PRs and issues** to get the broader context before writing any code.
|
||||
> 3. **Run a local review pass before pushing anything**, using the [`code-review`](skills/code-review/SKILL.md) skill, once the change is implemented. Fix what it finds.
|
||||
|
||||
For more info, please refer to the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
# Pull requests (for contributors & collaborators)
|
||||
|
||||
@@ -13,10 +13,16 @@
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
|
||||
|
||||
[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
[v1.0.0](https://huggingface.co/blog/ggml-org/llama-cpp-v1) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
|
||||
</div>
|
||||
|
||||
## Description
|
||||
|
||||
The main goal of `llama.cpp` is to enable LLM (and MLLM) inference with minimal setup and state-of-the-art performance on a wide range of hardware - locally and in the cloud.
|
||||
|
||||
The `llama.cpp` project is built on top of the [ggml](https://github.com/ggml-org/ggml) library.
|
||||
|
||||
## Quick start
|
||||
|
||||
A few options to get `llama.cpp` installed on your machine:
|
||||
@@ -51,50 +57,12 @@ llama serve -hf ggml-org/Qwen3.5-0.8B-GGUF
|
||||
<i>VLM session with <b>llama cli</b></i>
|
||||
</td>
|
||||
<td align="center">
|
||||
<img width="1392" height="958" alt="Built-in web UI against `llama serve` running Qwen 3.6" src="https://github.com/user-attachments/assets/b402f972-2e32-4def-8771-8d849f08cf2e" />
|
||||
<i>Built-in web UI against <b>llama serve</b></i>
|
||||
<img width="1392" height="958" alt="Llama Chat, the built-in chat app, against `llama serve` running Qwen 3.6" src="https://github.com/user-attachments/assets/b402f972-2e32-4def-8771-8d849f08cf2e" />
|
||||
<i>Llama Chat, the built-in chat app, against <b>llama serve</b></i>
|
||||
</td>
|
||||
</tr>
|
||||
<table>
|
||||
|
||||
## Description
|
||||
|
||||
The main goal of `llama.cpp` is to enable LLM (and VLM) inference with minimal setup and state-of-the-art performance on
|
||||
a wide range of hardware - locally and in the cloud.
|
||||
|
||||
- Plain C/C++ implementation without any dependencies
|
||||
- Apple silicon is a first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks
|
||||
- AVX, AVX2, AVX512 and AMX support for x86 architectures
|
||||
- RVV, ZVFH, ZFH, ZICBOP and ZIHINTPAUSE support for RISC-V architectures
|
||||
- 1.5-bit, 2-bit, 3-bit, 4-bit, 5-bit, 6-bit, and 8-bit integer quantization for faster inference and reduced memory use
|
||||
- Custom CUDA kernels for running LLMs on NVIDIA GPUs (support for AMD GPUs via HIP and Moore Threads GPUs via MUSA)
|
||||
- Vulkan and SYCL backend support
|
||||
- CPU+GPU hybrid inference to partially accelerate models larger than the total VRAM capacity
|
||||
|
||||
The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-org/ggml) library.
|
||||
|
||||
## Supported backends
|
||||
|
||||
| Backend | Target devices |
|
||||
| --- | --- |
|
||||
| [BLAS](docs/build.md#blas-build) | All |
|
||||
| [BLIS](docs/backend/BLIS.md) | All |
|
||||
| [CANN](docs/build.md#cann) | Huawei Ascend NPU |
|
||||
| [CUDA](docs/build.md#cuda) | Nvidia GPU |
|
||||
| [HIP](docs/build.md#hip) | AMD GPU |
|
||||
| [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |
|
||||
| [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
|
||||
| [MUSA](docs/build.md#musa) | Moore Threads GPU |
|
||||
| [Metal](docs/build.md#metal-build) | Apple Silicon |
|
||||
| [OpenCL](docs/backend/OPENCL.md) | Adreno GPU |
|
||||
| [OpenVINO [In Progress]](docs/backend/OPENVINO.md) | Intel CPUs, GPUs, and NPUs |
|
||||
| [RPC](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) | All |
|
||||
| [SYCL](docs/backend/SYCL.md) | Intel GPU |
|
||||
| [VirtGPU](docs/backend/VirtGPU.md) | VirtGPU APIR |
|
||||
| [Vulkan](docs/build.md#vulkan) | GPU |
|
||||
| [WebGPU](docs/build.md#webgpu) | All |
|
||||
| [ZenDNN](docs/build.md#zendnn) | AMD CPU |
|
||||
|
||||
## Documentation
|
||||
|
||||
#### Tools
|
||||
|
||||
+1
-1
@@ -46,7 +46,7 @@ Only vulnerabilities that fall within these parts of the project are considered
|
||||
- `ggml/**/*`
|
||||
- `gguf-py/**/*`
|
||||
- `tools/server/*`, **excluding** the following topics:
|
||||
- Web UI
|
||||
- Llama Chat, the built-in chat app (`ui/apps/chat`)
|
||||
- Features marked as experimental
|
||||
- Features not recommended for use in untrusted environments (e.g., router, MCP)
|
||||
- Bugs that can lead to Denial-of-Service attack
|
||||
|
||||
+12
-4
@@ -2829,7 +2829,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
const std::regex regex{ R"([,/]+)" };
|
||||
std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
|
||||
std::vector<std::string> split_arg{ it, {} };
|
||||
if (split_arg.size() >= llama_max_devices()) {
|
||||
if (split_arg.size() > llama_max_devices()) {
|
||||
throw std::invalid_argument(
|
||||
string_format("got %zu input configs, but system only has %zu devices", split_arg.size(), llama_max_devices())
|
||||
);
|
||||
@@ -2895,7 +2895,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
const std::regex regex{ R"([,/]+)" };
|
||||
std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
|
||||
std::vector<std::string> split_arg{ it, {} };
|
||||
if (split_arg.size() >= llama_max_devices()) {
|
||||
if (split_arg.size() > llama_max_devices()) {
|
||||
throw std::invalid_argument(
|
||||
string_format("got %zu input configs, but system only has %zu devices", split_arg.size(), llama_max_devices())
|
||||
);
|
||||
@@ -3416,7 +3416,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui-mcp-proxy", "--webui-mcp-proxy"},
|
||||
{"--no-ui-mcp-proxy", "--no-webui-mcp-proxy"},
|
||||
"experimental: whether to enable MCP CORS proxy, for the Web UI to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)\n"
|
||||
"experimental: whether to enable MCP CORS proxy, for Llama Chat to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)\n"
|
||||
"note: for security reasons, this will limit --cors-origins to localhost by default",
|
||||
[](common_params & params, bool value) {
|
||||
params.ui_mcp_proxy = value;
|
||||
@@ -3485,7 +3485,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui", "--webui"},
|
||||
{"--no-ui", "--no-webui"},
|
||||
string_format("whether to enable the Web UI (default: %s)", params.ui ? "enabled" : "disabled"),
|
||||
string_format("whether to enable Llama Chat, the built-in chat app (default: %s)", params.ui ? "enabled" : "disabled"),
|
||||
[](common_params & params, bool value) {
|
||||
params.ui = value;
|
||||
}
|
||||
@@ -4522,6 +4522,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.tts_speaker_file = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-speaker-text"}, "TEXT",
|
||||
"transcript of --tts-speaker-file, for models that need it to clone the voice\n"
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_speaker_text = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
|
||||
//
|
||||
// diffusion params
|
||||
|
||||
+25
-4
@@ -827,6 +827,11 @@ static std::string string_lower(std::string s) {
|
||||
return s;
|
||||
}
|
||||
|
||||
// VibeVoice-ASR: default system prompt of the template written by the converter
|
||||
static bool is_vibevoice_asr_template(const std::string & src) {
|
||||
return src.find("transcribes audio input into text output in JSON format") != std::string::npos;
|
||||
}
|
||||
|
||||
// LFM3-Audio: language code -> name
|
||||
static std::string lfm3_language_name(const std::string & language) {
|
||||
static const std::map<std::string, std::string> names = {
|
||||
@@ -847,7 +852,8 @@ static std::string lfm3_asr_system_prompt(const std::string & language) {
|
||||
return "Perform ASR in " + lfm3_language_name(language) + ".";
|
||||
}
|
||||
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language,
|
||||
double audio_length, const std::string & context) {
|
||||
common_chat_prompt_preset asr_preset;
|
||||
asr_preset.system = "";
|
||||
asr_preset.user = "Transcribe audio to text";
|
||||
@@ -861,6 +867,17 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates
|
||||
asr_preset.system = lfm3_asr_system_prompt(language);
|
||||
asr_preset.user = "";
|
||||
asr_preset.has_language = true;
|
||||
} else if (is_vibevoice_asr_template(src)) {
|
||||
// without the audio length the model does not know where the audio ends and loops
|
||||
const char * keys = "transcribe it with these keys: Start time, End time, Speaker ID, Content";
|
||||
if (context.empty()) {
|
||||
asr_preset.user = string_format("\nThis is a %.2f seconds audio, please %s", audio_length, keys);
|
||||
} else {
|
||||
asr_preset.user = string_format("\nThis is a %.2f seconds audio, with extra info: %s\n\nPlease %s", audio_length, context.c_str(), keys);
|
||||
asr_preset.has_context = true;
|
||||
}
|
||||
asr_preset.media_first = true;
|
||||
asr_preset.needs_audio_length = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1427,6 +1444,12 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
return common_chat_params_init_qwen3_coder;
|
||||
}
|
||||
|
||||
// Inkling 1 and 2
|
||||
if (is_inkling_template(src)) {
|
||||
LOG_DBG("Using specialized template: Inkling\n");
|
||||
return common_chat_params_init_inkling;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
@@ -1609,9 +1632,6 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
|
||||
auto_params.thinking_end_tags = {std::move(end_tag)};
|
||||
}
|
||||
}
|
||||
if (is_inkling2_template(src)) {
|
||||
common_chat_inkling2_split_media(auto_params.prompt);
|
||||
}
|
||||
const auto & arena = auto_params.parser;
|
||||
LOG_DBG("%s: generated parser:\n%s\n\nparser generation prompt: %s\n", __func__, arena.dump(arena.root()).c_str(), auto_params.generation_prompt.c_str());
|
||||
return auto_params;
|
||||
@@ -1962,5 +1982,6 @@ common_json common_chat_templates_get_caps(const common_chat_templates * chat_te
|
||||
: chat_templates->template_default->caps;
|
||||
common_json result = caps.to_map();
|
||||
result["reasoning_effort_levels"] = caps.reasoning_effort_levels;
|
||||
result["reasoning_effort_rejected"] = caps.reasoning_effort_rejected;
|
||||
return result;
|
||||
}
|
||||
|
||||
+8
-2
@@ -491,10 +491,16 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
|
||||
struct common_chat_prompt_preset {
|
||||
std::string system;
|
||||
std::string user;
|
||||
bool has_language = false; // the preset already names the requested language
|
||||
bool has_language = false; // the preset already names the requested language
|
||||
bool has_context = false; // the user text already holds the context
|
||||
bool media_first = false; // the media goes before the user text
|
||||
bool needs_audio_length = false; // the user text depends on the audio length, call again with it
|
||||
};
|
||||
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "");
|
||||
// audio_length: in seconds, 0 if unknown
|
||||
// context: extra info from the user (e.g. names, hotwords), only used by some presets, see has_context
|
||||
common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "",
|
||||
double audio_length = 0.0, const std::string & context = "");
|
||||
|
||||
// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
|
||||
// language and voice are optional, the template defines their defaults
|
||||
|
||||
@@ -796,6 +796,7 @@ struct common_params {
|
||||
std::string tts_lang = "";
|
||||
std::string tts_voice = "";
|
||||
std::string tts_speaker_file = "";
|
||||
std::string tts_speaker_text = "";
|
||||
|
||||
bool is_gen_docs = false; // whether we are running inside llama-gen-docs
|
||||
};
|
||||
|
||||
+11
-1
@@ -627,9 +627,14 @@ caps caps_get(jinja::program & prog) {
|
||||
// levels sharing a render are aliases and the highest one is kept
|
||||
const auto unknown = render("REASONING_EFFORT_UNKNOWN");
|
||||
std::vector<std::string> renders;
|
||||
std::vector<std::string> rejected;
|
||||
for (const auto & level : candidates) {
|
||||
const auto rendered = render(level);
|
||||
if (!rendered || rendered == unknown) {
|
||||
if (!rendered) {
|
||||
rejected.push_back(level);
|
||||
continue;
|
||||
}
|
||||
if (rendered == unknown) {
|
||||
continue;
|
||||
}
|
||||
const auto it = std::find(renders.begin(), renders.end(), *rendered);
|
||||
@@ -640,6 +645,11 @@ caps caps_get(jinja::program & prog) {
|
||||
result.reasoning_effort_levels.push_back(level);
|
||||
}
|
||||
}
|
||||
|
||||
// a template that renders no level fails the probe for another reason, so its failures say nothing about the levels
|
||||
if (!result.reasoning_effort_levels.empty()) {
|
||||
result.reasoning_effort_rejected = rejected;
|
||||
}
|
||||
}
|
||||
|
||||
JJ_DEBUG("%s\n", result.to_string().c_str());
|
||||
|
||||
@@ -23,6 +23,9 @@ struct caps {
|
||||
// reasoning effort levels the template renders distinctly, from lowest to highest
|
||||
std::vector<std::string> reasoning_effort_levels;
|
||||
|
||||
// reasoning effort levels the template raises on
|
||||
std::vector<std::string> reasoning_effort_rejected;
|
||||
|
||||
// one of the 2 content capabilities must be true
|
||||
bool supports_string_content = true;
|
||||
bool supports_typed_content = false;
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
#include <charconv>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
@@ -264,47 +263,6 @@ lexer_result lexer::tokenize(const std::string & source) {
|
||||
|
||||
char ch = src[pos];
|
||||
|
||||
bool is_closing_block = ch == '-' && next_pos_is( {'%', '}'} );
|
||||
|
||||
// Check for unary operators
|
||||
if (!is_closing_block && (ch == '-' || ch == '+')) {
|
||||
start_pos = pos;
|
||||
token::type last_token_type = tokens.empty() ? token::eof : tokens.back().t;
|
||||
if (last_token_type == token::text || last_token_type == token::eof) {
|
||||
throw lexer_exception(std::string("unexpected character: ") + ch, source, pos);
|
||||
}
|
||||
// keyword followed by an operand, e.g. `x if c else -1`; `a.else - 1` is still binary
|
||||
static const std::set<std::string> operand_keywords = { "if", "else", "and", "or", "not", "in", "is" };
|
||||
const bool is_attr = tokens.size() >= 2 && tokens[tokens.size() - 2].t == token::dot;
|
||||
if (last_token_type == token::identifier && !is_attr && operand_keywords.count(tokens.back().value)) {
|
||||
last_token_type = token::eof; // handle as unary below
|
||||
}
|
||||
switch (last_token_type) {
|
||||
case token::identifier:
|
||||
case token::numeric_literal:
|
||||
case token::string_literal:
|
||||
case token::close_paren:
|
||||
case token::close_square_bracket:
|
||||
// Part of a binary operator
|
||||
// a - 1, 1 - 1, true - 1, "apple" - 1, (1) - 1, a[1] - 1
|
||||
// Continue parsing normally
|
||||
break;
|
||||
default: {
|
||||
// Is part of a unary operator
|
||||
// (-1), [-1], (1 + -1), not -1, -apple
|
||||
++pos; // Consume the operator
|
||||
|
||||
// Check for numbers following the unary operator
|
||||
std::string num = consume_numeric();
|
||||
std::string value = std::string(1, ch) + num;
|
||||
token::type t = num.empty() ? token::unary_operator : token::numeric_literal;
|
||||
// JJ_DEBUG("consumed unary operator or numeric literal: '%s'", value.c_str());
|
||||
tokens.push_back({t, value, start_pos});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try to match one of the tokens in the mapping table
|
||||
bool matched = false;
|
||||
for (const auto & [seq, typ] : ordered_mapping_table) {
|
||||
|
||||
@@ -39,7 +39,6 @@ struct token {
|
||||
multiplicative_binary_operator, // * / %
|
||||
power_operator, // **
|
||||
comparison_binary_operator, // < > <= >= == !=
|
||||
unary_operator, // ! - +
|
||||
comment, // {# ... #}
|
||||
};
|
||||
type t;
|
||||
@@ -74,7 +73,6 @@ static std::string type_to_string(token::type t) {
|
||||
case token::multiplicative_binary_operator: return "multiplicative_binary_operator";
|
||||
case token::power_operator: return "power_operator";
|
||||
case token::comparison_binary_operator: return "comparison_binary_operator";
|
||||
case token::unary_operator: return "unary_operator";
|
||||
case token::comment: return "comment";
|
||||
default: return "unknown";
|
||||
}
|
||||
|
||||
@@ -482,7 +482,8 @@ private:
|
||||
}
|
||||
|
||||
statement_ptr parse_unary_expression() {
|
||||
if (is(token::unary_operator)) {
|
||||
// a sign where an operand is expected is unary: -x, 1 + -x, x if c else -1
|
||||
if (is(token::additive_binary_operator) && peek().value != "~") {
|
||||
depth_guard guard(*this);
|
||||
size_t start_pos = current;
|
||||
auto op = next();
|
||||
|
||||
@@ -811,13 +811,14 @@ value call_statement::execute_impl(context & ctx) const {
|
||||
}
|
||||
auto * callee_func = cast_val<value_func>(callee_val);
|
||||
|
||||
context caller_ctx(ctx); // new scope for caller execution
|
||||
// caller() can be saved and called after this scope ended, it owns a snapshot of the variables
|
||||
auto caller_ctx = std::make_shared<context>(ctx.snapshot());
|
||||
|
||||
const func_handler func = [this, caller_ctx = std::move(caller_ctx)](const func_args & args) -> value {
|
||||
const func_handler func = [this, caller_ctx](const func_args & args) -> value {
|
||||
if (args.ctx.macro_depth >= JINJA_RECURSIVE_LIMIT) {
|
||||
throw std::runtime_error("macro calls are nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
|
||||
}
|
||||
context block_ctx(caller_ctx); // new scope for block execution
|
||||
context block_ctx(*caller_ctx); // new scope for block execution
|
||||
block_ctx.macro_depth = args.ctx.macro_depth + 1;
|
||||
|
||||
bind_parameters("caller", this->caller_args, args, block_ctx);
|
||||
|
||||
+30
-3
@@ -81,12 +81,38 @@ struct context {
|
||||
current_time(parent.current_time),
|
||||
is_get_stats(parent.is_get_stats),
|
||||
macro_depth(parent.macro_depth),
|
||||
// inherit variables (for example, when entering a new scope)
|
||||
env(mk_val<value_object>(*parent.env)) {}
|
||||
// child scope: own variables only, lookups fall back to the parent (which must outlive this scope)
|
||||
parent(&parent),
|
||||
env(mk_val<value_object>()) {
|
||||
env->has_builtins = false;
|
||||
}
|
||||
|
||||
context(context &&) = default;
|
||||
|
||||
// copy with all visible variables flattened in, for a scope that can outlive its parents (caller() of a call block)
|
||||
context snapshot() const {
|
||||
std::vector<const context *> chain;
|
||||
for (const context * c = this; c; c = c->parent) {
|
||||
chain.push_back(c);
|
||||
}
|
||||
context res(*this);
|
||||
res.parent = nullptr;
|
||||
for (auto it = chain.rbegin(); it != chain.rend(); ++it) {
|
||||
for (const auto & kv : (*it)->env->as_ordered_object()) {
|
||||
res.env->insert(kv.first, kv.second);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
// returns nullptr if not found
|
||||
value find_val(const value & key) {
|
||||
return env->find(key);
|
||||
for (const context * c = this; c; c = c->parent) {
|
||||
if (value v = c->env->find(key)) {
|
||||
return v;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
value get_val(const std::string & name) {
|
||||
@@ -107,6 +133,7 @@ struct context {
|
||||
}
|
||||
|
||||
private:
|
||||
const context * parent = nullptr;
|
||||
value_object env;
|
||||
};
|
||||
|
||||
|
||||
+23
-14
@@ -281,14 +281,12 @@ static value tojson(const func_args & args) {
|
||||
if (is_val<value_int>(val_indent)) {
|
||||
indent = static_cast<int>(val_indent->as_int());
|
||||
}
|
||||
if (val_sort->as_bool()) { // undefined == false
|
||||
throw not_implemented_exception("tojson sort_keys=true not implemented");
|
||||
}
|
||||
const bool sort_keys = val_sort->as_bool(); // undefined == false
|
||||
const bool ensure_ascii = val_ascii->as_bool(); // undefined == false
|
||||
auto separators = (is_val<value_array>(val_separators) ? val_separators : mk_val<value_array>())->as_array();
|
||||
std::string item_sep = separators.size() > 0 ? separators[0]->as_string().str() : (indent < 0 ? ", " : ",");
|
||||
std::string key_sep = separators.size() > 1 ? separators[1]->as_string().str() : ": ";
|
||||
std::string json_str = value_to_json(args.get_pos(0), indent, item_sep, key_sep);
|
||||
std::string json_str = value_to_json(args.get_pos(0), indent, item_sep, key_sep, sort_keys);
|
||||
if (ensure_ascii) {
|
||||
json_str = json_ensure_ascii_preserving_format(json_str);
|
||||
}
|
||||
@@ -1113,6 +1111,9 @@ static value array_sum(const func_args & args) {
|
||||
double sum = start->is_undefined() ? 0.0 : start->as_float();
|
||||
for (const auto & item : args.get_pos(0)->as_array()) {
|
||||
value v = attribute->is_undefined() ? item : get_attribute(item, attribute, undef);
|
||||
if (v->is_undefined() && !attribute->is_undefined()) {
|
||||
throw raised_exception("sum: " + item->type() + " has no attribute '" + attribute->as_string().str() + "'");
|
||||
}
|
||||
if (!is_val<value_int>(v) && !is_val<value_float>(v)) {
|
||||
throw raised_exception("sum: unsupported operand type " + v->type());
|
||||
}
|
||||
@@ -1758,7 +1759,7 @@ static std::string float_json(double val) {
|
||||
|
||||
// recursively convert value to JSON string
|
||||
// TODO: avoid circular references
|
||||
static void value_to_json_internal(std::ostringstream & oss, const value & val, int curr_lvl, int indent, const std::string_view item_sep, const std::string_view key_sep) {
|
||||
static void value_to_json_internal(std::ostringstream & oss, const value & val, int curr_lvl, int indent, const std::string_view item_sep, const std::string_view key_sep, bool sort_keys) {
|
||||
auto indent_str = [indent, curr_lvl]() -> std::string {
|
||||
return (indent > 0) ? std::string(curr_lvl * indent, ' ') : "";
|
||||
};
|
||||
@@ -1805,7 +1806,7 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
|
||||
oss << newline();
|
||||
for (size_t i = 0; i < arr.size(); ++i) {
|
||||
oss << indent_str() << (indent > 0 ? std::string(indent, ' ') : "");
|
||||
value_to_json_internal(oss, arr[i], curr_lvl + 1, indent, item_sep, key_sep);
|
||||
value_to_json_internal(oss, arr[i], curr_lvl + 1, indent, item_sep, key_sep, sort_keys);
|
||||
if (i < arr.size() - 1) {
|
||||
oss << item_sep;
|
||||
}
|
||||
@@ -1816,17 +1817,25 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
|
||||
oss << "]";
|
||||
} else if (is_val<value_object>(val)) {
|
||||
const auto & obj = val->as_ordered_object(); // IMPORTANT: need to keep exact order
|
||||
std::vector<std::pair<std::string, value>> items;
|
||||
items.reserve(obj.size());
|
||||
for (const auto & pair : obj) {
|
||||
items.emplace_back(is_val<value_float>(pair.first) ? float_json(pair.first->as_float()) : pair.first->as_string().str(), pair.second);
|
||||
}
|
||||
if (sort_keys) {
|
||||
// same order as python json.dumps(sort_keys=True): utf-8 byte order is code point order
|
||||
std::stable_sort(items.begin(), items.end(), [](const auto & a, const auto & b) { return a.first < b.first; });
|
||||
}
|
||||
oss << "{";
|
||||
if (!obj.empty()) {
|
||||
if (!items.empty()) {
|
||||
oss << newline();
|
||||
size_t i = 0;
|
||||
for (const auto & pair : obj) {
|
||||
for (const auto & [key, item] : items) {
|
||||
oss << indent_str() << (indent > 0 ? std::string(indent, ' ') : "");
|
||||
const std::string key = is_val<value_float>(pair.first) ? float_json(pair.first->as_float()) : pair.first->as_string().str();
|
||||
value_to_json_internal(oss, mk_val<value_string>(key), curr_lvl + 1, indent, item_sep, key_sep);
|
||||
value_to_json_internal(oss, mk_val<value_string>(key), curr_lvl + 1, indent, item_sep, key_sep, sort_keys);
|
||||
oss << key_sep;
|
||||
value_to_json_internal(oss, pair.second, curr_lvl + 1, indent, item_sep, key_sep);
|
||||
if (i < obj.size() - 1) {
|
||||
value_to_json_internal(oss, item, curr_lvl + 1, indent, item_sep, key_sep, sort_keys);
|
||||
if (i < items.size() - 1) {
|
||||
oss << item_sep;
|
||||
}
|
||||
oss << newline();
|
||||
@@ -1840,9 +1849,9 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
|
||||
}
|
||||
}
|
||||
|
||||
std::string value_to_json(const value & val, int indent, const std::string_view item_sep, const std::string_view key_sep) {
|
||||
std::string value_to_json(const value & val, int indent, const std::string_view item_sep, const std::string_view key_sep, bool sort_keys) {
|
||||
std::ostringstream oss;
|
||||
value_to_json_internal(oss, val, 0, indent, item_sep, key_sep);
|
||||
value_to_json_internal(oss, val, 0, indent, item_sep, key_sep, sort_keys);
|
||||
JJ_DEBUG("value_to_json: result=%s", oss.str().c_str());
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
@@ -199,7 +199,7 @@ value find_builtin_func(context & ctx, const std::string & name, value & input);
|
||||
value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false);
|
||||
value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args);
|
||||
|
||||
std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": ");
|
||||
std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": ", bool sort_keys = false);
|
||||
|
||||
// Note: only used for debugging purposes
|
||||
std::string value_to_string_repr(const value & val);
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
#include "parsers.h"
|
||||
|
||||
bool is_inkling_template(const std::string & src) {
|
||||
return src.find("<|message_user|>") != std::string::npos &&
|
||||
src.find("<|content_image|>") != std::string::npos;
|
||||
}
|
||||
|
||||
// the template makes one message block per content part, but a media marker comes inside the text of a message
|
||||
// <header><|content_text|>{media}text<|end_message|> --> <header>{media}<|end_message|><header><|content_text|>text<|end_message|>
|
||||
static void inkling_split_media(common_chat_prompt & prompt) {
|
||||
const std::string TEXT = "<|content_text|>";
|
||||
const std::string END = "<|end_message|>";
|
||||
const std::string MEDIA_BEG = "<__media_";
|
||||
const std::string MEDIA_END = "__>";
|
||||
|
||||
std::vector<common_chat_prompt_part> out;
|
||||
for (size_t i = 0; i < prompt.parts.size(); i++) {
|
||||
const auto & part = prompt.parts[i];
|
||||
const bool in_text_block = part.is_input && part.text.find(MEDIA_BEG) != std::string::npos &&
|
||||
!out.empty() && !out.back().is_input && string_ends_with(out.back().text, TEXT) &&
|
||||
i + 1 < prompt.parts.size() && !prompt.parts[i + 1].is_input && string_starts_with(prompt.parts[i + 1].text, END);
|
||||
const size_t header_pos = in_text_block ? out.back().text.rfind("<|message_") : std::string::npos;
|
||||
if (header_pos == std::string::npos) {
|
||||
out.push_back(part);
|
||||
continue;
|
||||
}
|
||||
const std::string header = out.back().text.substr(header_pos, out.back().text.size() - TEXT.size() - header_pos);
|
||||
|
||||
std::string rest = part.text;
|
||||
for (bool first = true; !rest.empty(); first = false) {
|
||||
const size_t beg = rest.find(MEDIA_BEG);
|
||||
const size_t end = beg == std::string::npos ? beg : rest.find(MEDIA_END, beg);
|
||||
const bool is_media = beg == 0 && end != std::string::npos;
|
||||
const size_t len = is_media ? end + MEDIA_END.size() : (end == std::string::npos ? rest.size() : beg);
|
||||
if (!first) {
|
||||
out.push_back({ END + header + (is_media ? "" : TEXT), false });
|
||||
} else if (is_media) {
|
||||
// mtmd opens the media block
|
||||
out.back().text.resize(out.back().text.size() - TEXT.size());
|
||||
}
|
||||
out.push_back({ rest.substr(0, len), true });
|
||||
rest.erase(0, len);
|
||||
}
|
||||
}
|
||||
prompt.parts = std::move(out);
|
||||
}
|
||||
|
||||
// Inkling 1 and 2 - the assistant turn is a list of blocks, all closed by <|end_message|>:
|
||||
// <|message_model|><|content_thinking|>...<|end_message|>
|
||||
// <|message_model|><|content_text|>...<|end_message|>
|
||||
// <|message_model|>NAME<|content_invoke_tool_json|>{"name":"NAME","args":{...}}<|end_message|>
|
||||
common_chat_params common_chat_params_init_inkling(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
inkling_split_media(data.prompt);
|
||||
|
||||
const std::string HEADER = "<|message_model|>";
|
||||
const std::string THINK = "<|content_thinking|>";
|
||||
const std::string TEXT = "<|content_text|>";
|
||||
const std::string TOOL_JSON = "<|content_invoke_tool_json|>";
|
||||
const std::string END = "<|end_message|>";
|
||||
|
||||
data.preserved_tokens = { HEADER, THINK, TEXT, TOOL_JSON, END };
|
||||
|
||||
data.thinking_start_tag = THINK;
|
||||
data.thinking_end_tags = { END };
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, HEADER },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|message_user|>" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|message_system|>" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|message_tool|>" },
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt.clear();
|
||||
if (!msg.reasoning_content.empty()) {
|
||||
data.generation_prompt = HEADER + THINK + msg.reasoning_content;
|
||||
}
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
if (!data.generation_prompt.empty()) {
|
||||
data.generation_prompt += END;
|
||||
}
|
||||
data.generation_prompt += HEADER + TEXT + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto use_tools = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto header = p.literal(HEADER);
|
||||
auto end = p.literal(END);
|
||||
|
||||
auto think = p.literal(THINK);
|
||||
auto thinking = p.rule("thinking", extract_reasoning ? think + p.reasoning(p.until(END)) + end :
|
||||
p.atomic(p.content(think)) + p.content(p.until(END) + end));
|
||||
auto text = p.rule("text", p.literal(TEXT) + p.content(p.until(END)) + end);
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_format = p.rule("response-format", p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
|
||||
return p.zero_or_more(header + thinking) + header + p.literal(TEXT) + response_format + end;
|
||||
}
|
||||
|
||||
auto blocks = p.zero_or_more(header + (thinking | text));
|
||||
|
||||
if (!use_tools) {
|
||||
return blocks;
|
||||
}
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
|
||||
auto open = p.tool_name(p.literal(name)) + p.literal(TOOL_JSON) + p.literal("{\"name\":" + json(name).dump() + ",\"args\":");
|
||||
auto args = p.tool_args(p.json());
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), p.tool(p.tool_open(open) + args + p.tool_close(p.literal("}")) + end));
|
||||
});
|
||||
|
||||
// tool calls are the last blocks of the turn
|
||||
auto calls = inputs.parallel_tool_calls ? tool_choice + p.zero_or_more(header + tool_choice) : tool_choice;
|
||||
auto tool_calls = header + p.trigger_rule("tool-calls", calls);
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return blocks + tool_calls;
|
||||
}
|
||||
|
||||
return blocks + p.optional(tool_calls);
|
||||
});
|
||||
|
||||
if (has_response_format || use_tools) {
|
||||
data.grammar_lazy = !has_response_format && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
if (data.grammar_lazy) {
|
||||
// a header followed by text and not by a content token is a tool call, the grammar starts at the name
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|message_model\\|>([^<])" },
|
||||
};
|
||||
// Inkling 1: the first header is in the generation prompt, and a lazy grammar does not get the prompt.
|
||||
// an output that starts with no content token is a tool call. not for a continuation, it starts with text
|
||||
if (!inputs.has_continuation()) {
|
||||
data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL, "[^<]+" });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
#include "parsers.h"
|
||||
|
||||
bool is_inkling2_template(const std::string & src) {
|
||||
return src.find("<|message_user|>") != std::string::npos &&
|
||||
src.find("<|content_image|>") != std::string::npos;
|
||||
}
|
||||
|
||||
// the template makes one message block per content part, but a media marker comes inside the text of a message
|
||||
// <header><|content_text|>{media}text<|end_message|> --> <header>{media}<|end_message|><header><|content_text|>text<|end_message|>
|
||||
void common_chat_inkling2_split_media(common_chat_prompt & prompt) {
|
||||
const std::string TEXT = "<|content_text|>";
|
||||
const std::string END = "<|end_message|>";
|
||||
const std::string MEDIA_BEG = "<__media_";
|
||||
const std::string MEDIA_END = "__>";
|
||||
|
||||
std::vector<common_chat_prompt_part> out;
|
||||
for (size_t i = 0; i < prompt.parts.size(); i++) {
|
||||
const auto & part = prompt.parts[i];
|
||||
const bool in_text_block = part.is_input && part.text.find(MEDIA_BEG) != std::string::npos &&
|
||||
!out.empty() && !out.back().is_input && string_ends_with(out.back().text, TEXT) &&
|
||||
i + 1 < prompt.parts.size() && !prompt.parts[i + 1].is_input && string_starts_with(prompt.parts[i + 1].text, END);
|
||||
const size_t header_pos = in_text_block ? out.back().text.rfind("<|message_") : std::string::npos;
|
||||
if (header_pos == std::string::npos) {
|
||||
out.push_back(part);
|
||||
continue;
|
||||
}
|
||||
const std::string header = out.back().text.substr(header_pos, out.back().text.size() - TEXT.size() - header_pos);
|
||||
|
||||
std::string rest = part.text;
|
||||
for (bool first = true; !rest.empty(); first = false) {
|
||||
const size_t beg = rest.find(MEDIA_BEG);
|
||||
const size_t end = beg == std::string::npos ? beg : rest.find(MEDIA_END, beg);
|
||||
const bool is_media = beg == 0 && end != std::string::npos;
|
||||
const size_t len = is_media ? end + MEDIA_END.size() : (end == std::string::npos ? rest.size() : beg);
|
||||
if (!first) {
|
||||
out.push_back({ END + header + (is_media ? "" : TEXT), false });
|
||||
} else if (is_media) {
|
||||
// mtmd opens the media block
|
||||
out.back().text.resize(out.back().text.size() - TEXT.size());
|
||||
}
|
||||
out.push_back({ rest.substr(0, len), true });
|
||||
rest.erase(0, len);
|
||||
}
|
||||
}
|
||||
prompt.parts = std::move(out);
|
||||
}
|
||||
@@ -41,10 +41,7 @@ std::string common_chat_template_generation_prompt_impl(
|
||||
|
||||
bool is_lfm2_template(const std::string & src);
|
||||
|
||||
bool is_inkling2_template(const std::string & src);
|
||||
|
||||
// give each media marker of a text block its own message block
|
||||
void common_chat_inkling2_split_media(common_chat_prompt & prompt);
|
||||
bool is_inkling_template(const std::string & src);
|
||||
|
||||
common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
@@ -60,6 +57,8 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
|
||||
|
||||
common_chat_params common_chat_params_init_harmony_xml(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_inkling(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_k2_horizon(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
@@ -10,7 +10,7 @@ set(LLAMA_CHAT_PARSERS_SOURCES
|
||||
${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/harmony-xml.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/inkling2.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/inkling.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/k2-horizon.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp
|
||||
|
||||
@@ -138,6 +138,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Grok1ForCausalLM": "grok",
|
||||
"GrokForCausalLM": "grok",
|
||||
"GroveMoeForCausalLM": "grovemoe",
|
||||
"HiggsAudio3Model": "higgs_audio",
|
||||
"HunYuanDenseV1ForCausalLM": "hunyuan",
|
||||
"HunYuanMoEV1ForCausalLM": "hunyuan",
|
||||
"HunYuanVLForConditionalGeneration": "hunyuan",
|
||||
@@ -282,6 +283,8 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Qwen3NextForCausalLM": "qwen",
|
||||
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
|
||||
"PocketTTSModel": "pockettts",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
|
||||
@@ -324,6 +327,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"UMT5Model": "t5",
|
||||
"UltravoxModel": "ultravox",
|
||||
"UnlimitedOCRForCausalLM": "deepseek",
|
||||
"VibeVoiceAsrForConditionalGeneration": "vibevoice_asr",
|
||||
"VLlama3ForCausalLM": "llama",
|
||||
"VllmDecision2Model": "vllm_decision2",
|
||||
"VoxtralForConditionalGeneration": "llama",
|
||||
@@ -333,6 +337,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"XverseForCausalLM": "xverse",
|
||||
"YoutuForCausalLM": "deepseek",
|
||||
"YoutuVLForConditionalGeneration": "deepseek",
|
||||
"fish_qwen3_omni": "fish_s2",
|
||||
"modeling_grove_moe.GroveMoeForCausalLM": "grovemoe",
|
||||
"modeling_sarvam_moe.SarvamMoEForCausalLM": "bailingmoe",
|
||||
}
|
||||
@@ -356,6 +361,8 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"EmbeddingGemma2Model": "gemma",
|
||||
"Exaone4_5_ForConditionalGeneration": "exaone",
|
||||
"FishOmniModel": "drama3",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"Gemma3ForConditionalGeneration": "gemma",
|
||||
"Gemma3nForConditionalGeneration": "gemma",
|
||||
"Gemma4ForConditionalGeneration": "gemma",
|
||||
@@ -370,6 +377,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"Granite4VisionForConditionalGeneration": "granite",
|
||||
"GraniteSpeechForConditionalGeneration": "granite",
|
||||
"GraniteSpeechPlusForConditionalGeneration": "granite",
|
||||
"HiggsAudio3Model": "higgs_audio",
|
||||
"HunYuanVLForConditionalGeneration": "hunyuan",
|
||||
"Idefics3ForConditionalGeneration": "smolvlm",
|
||||
"Inkling2ForConditionalGeneration": "inkling2",
|
||||
@@ -429,9 +437,11 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"Step3p7ForConditionalGeneration": "step3",
|
||||
"UltravoxModel": "ultravox",
|
||||
"UnlimitedOCRForCausalLM": "deepseek",
|
||||
"VibeVoiceAsrForConditionalGeneration": "vibevoice_asr",
|
||||
"VoxtralForConditionalGeneration": "ultravox",
|
||||
"VoxtralRealtimeForConditionalGeneration": "ultravox",
|
||||
"YoutuVLForConditionalGeneration": "youtuvl",
|
||||
"fish_qwen3_omni": "fish_s2",
|
||||
}
|
||||
|
||||
|
||||
|
||||
+8
-3
@@ -1986,8 +1986,10 @@ class TextModel(ModelBase):
|
||||
logger.info(f"gguf: rope theta swa = {local_rope_theta}")
|
||||
# Add rope_pattern from no_rope_layers
|
||||
if no_rope_layers := self.hparams.get("no_rope_layers"):
|
||||
# Convert 1/0 to bool (1 = use RoPE, 0 = NoPE)
|
||||
rope_pattern = [bool(x) for x in no_rope_layers]
|
||||
if len(no_rope_layers) == self.block_count: # "no_rope_layers": [0, 0, 0, 1]
|
||||
rope_pattern = [bool(x) for x in no_rope_layers]
|
||||
else: # "no_rope_layers": [3]
|
||||
rope_pattern = [i not in no_rope_layers for i in range(self.block_count)]
|
||||
self.gguf_writer.add_rope_pattern(rope_pattern)
|
||||
logger.info(f"gguf: rope_pattern = {sum(rope_pattern)} RoPE layers / {len(rope_pattern)} total")
|
||||
if (f_rms_eps := self.find_hparam(["rms_norm_eps", "norm_eps"], optional=True)) is not None:
|
||||
@@ -3475,7 +3477,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
|
||||
# Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older
|
||||
# Kimi-Linear-48B architecture and cannot load K3 (no attention residuals,
|
||||
# latent MoE, situ, ...). Route on the top-level architecture instead.
|
||||
if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration"):
|
||||
if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "KimiK31ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration", "HiggsAudio3Model"):
|
||||
return arch
|
||||
|
||||
# if "architectures" is found in the sub-config, use that instead
|
||||
@@ -3486,6 +3488,9 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
|
||||
# some models (llava_onevision2) only declare the architecture inside text_config
|
||||
if arch is None and text_config.get("architectures") is not None:
|
||||
arch = text_config["architectures"][0]
|
||||
# some configs only have a model_type (fishaudio/s2-pro)
|
||||
if arch is None:
|
||||
arch = hparams.get("model_type")
|
||||
if arch is None:
|
||||
raise ValueError("Failed to detect model architecture")
|
||||
return arch
|
||||
|
||||
+32
-19
@@ -18,6 +18,30 @@ _TOWER_SHARED = "audio_decoder_mtp_shared.layers."
|
||||
_TOWER_RESIDUAL = "audio_decoder_mtp.layers."
|
||||
|
||||
|
||||
def load_codec(dir_model) -> dict[str, Tensor]:
|
||||
sd = torch.load(dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
|
||||
return sd.get("state_dict", sd)
|
||||
|
||||
|
||||
def fold_conv(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# fold weight norm
|
||||
if prefix + ".conv.weight" in sd:
|
||||
return sd[prefix + ".conv.weight"].float()
|
||||
g = sd[prefix + ".conv.parametrizations.weight.original0"].float()
|
||||
v = sd[prefix + ".conv.parametrizations.weight.original1"].float()
|
||||
return g * v / v.norm(dim=(1, 2), keepdim=True)
|
||||
|
||||
|
||||
def fold_wn(g: Tensor, v: Tensor) -> Tensor:
|
||||
# fold old-style weight norm of a 1x1 conv, returns [out, in]
|
||||
return (g.float() * v.float() / v.float().norm(dim=(1, 2), keepdim=True)).squeeze(-1)
|
||||
|
||||
|
||||
def fold_snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
|
||||
alpha = alpha.float().flatten()
|
||||
return alpha, 1.0 / (alpha + 1e-9)
|
||||
|
||||
|
||||
@ModelBase.register("FishOmniModel")
|
||||
@ModelBase.example("fishaudio/drama-3")
|
||||
class Drama3TextModel(Qwen3_5MoeTextModel):
|
||||
@@ -36,6 +60,7 @@ class Drama3TextModel(Qwen3_5MoeTextModel):
|
||||
class Drama3MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
gen_projector_type = gguf.VisionProjectorType.DRAMA3_GEN
|
||||
|
||||
_LAYER_MAP = {
|
||||
"enorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ENORM, True),
|
||||
@@ -69,7 +94,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
cfg = self.hparams
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.DRAMA3_GEN)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(self.gen_projector_type)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(cfg["dim"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(cfg["intermediate_size"])
|
||||
@@ -153,32 +178,20 @@ class Drama3MmprojModel(MmprojModel):
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# only the DAC decoder, the encoder is for voice cloning (not supported yet)
|
||||
sd = torch.load(self.dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
|
||||
sd = sd.get("state_dict", sd)
|
||||
sd = load_codec(self.dir_model)
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
def get(name: str) -> Tensor:
|
||||
return sd[name].float()
|
||||
|
||||
def conv(prefix: str) -> Tensor:
|
||||
# fold weight norm
|
||||
if prefix + ".conv.weight" in sd:
|
||||
return get(prefix + ".conv.weight")
|
||||
g = get(prefix + ".conv.parametrizations.weight.original0")
|
||||
v = get(prefix + ".conv.parametrizations.weight.original1")
|
||||
return g * v / v.norm(dim=(1, 2), keepdim=True)
|
||||
return fold_conv(sd, prefix)
|
||||
|
||||
def quant_table(prefix: str) -> Tensor:
|
||||
# fold out_proj into the codebook, so decoding is a row lookup
|
||||
g = get(prefix + ".out_proj.weight_g")
|
||||
v = get(prefix + ".out_proj.weight_v")
|
||||
w = (g * v / v.norm(dim=(1, 2), keepdim=True)).squeeze(-1)
|
||||
w = fold_wn(sd[prefix + ".out_proj.weight_g"], sd[prefix + ".out_proj.weight_v"])
|
||||
return get(prefix + ".codebook.weight") @ w.T + get(prefix + ".out_proj.bias")
|
||||
|
||||
def snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
|
||||
alpha = alpha.flatten()
|
||||
return alpha, 1.0 / (alpha + 1e-9)
|
||||
|
||||
q = "quantizer."
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), quant_table(q + "semantic_quantizer.quantizers.0"))
|
||||
n_rest = sum(1 for k in sd if k.startswith(q + "quantizer.quantizers.") and k.endswith(".codebook.weight"))
|
||||
@@ -228,7 +241,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get(d + "0.conv.bias"))
|
||||
for bid in range(4):
|
||||
p = f"{d}{bid + 1}.block."
|
||||
a, b = snake(get(p + "0.alpha"))
|
||||
a, b = fold_snake(get(p + "0.alpha"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), conv(p + "1"))
|
||||
@@ -236,7 +249,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
for xid in range(3):
|
||||
r = f"{p}{xid + 2}.block."
|
||||
for act, tensor in (("0", T.A_GEN_WAV_DAC_RES_ACT1), ("2", T.A_GEN_WAV_DAC_RES_ACT2)):
|
||||
a, b = snake(get(r + act + ".alpha"))
|
||||
a, b = fold_snake(get(r + act + ".alpha"))
|
||||
name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
|
||||
yield (name + ".alpha", a)
|
||||
yield (name + ".beta", b)
|
||||
@@ -244,7 +257,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
|
||||
yield (name + ".weight", conv(r + cv))
|
||||
yield (name + ".bias", get(r + cv + ".conv.bias"))
|
||||
a, b = snake(get(d + "5.alpha"))
|
||||
a, b = fold_snake(get(d + "5.alpha"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6"))
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, gguf
|
||||
from .drama3 import Drama3MmprojModel, fold_conv, fold_snake, fold_wn, load_codec
|
||||
from .qwen import Qwen3Model
|
||||
|
||||
# Fish Audio S2 Pro (dual-AR)
|
||||
# text model: Qwen3 backbone (slow AR), samples the semantic code of each frame
|
||||
# mmproj: fast AR code predictor (other codebooks) and Fish DAC decoder (codec.pth, same as drama3)
|
||||
|
||||
# residual codebook size of codec.pth, the fast AR head is cut to it
|
||||
_N_RESIDUAL_CODES = 1024
|
||||
|
||||
# codec encoder strides (encoder_rates of the codec config), the quantizer then downsamples 2x2
|
||||
_CODEC_ENC_STRIDES = (2, 4, 8, 8)
|
||||
|
||||
# codec encoder transformers, both 1024 wide with 16 heads of 64
|
||||
# generic audio layers a.blk.0..3: last encoder block, a.blk.4..11: quantizer pre_module
|
||||
_N_ENC_TFM_LAYERS = 4
|
||||
_N_PRE_TFM_LAYERS = 8
|
||||
_CODEC_HEAD_DIM = 64
|
||||
|
||||
|
||||
def _load_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
|
||||
cfg = hparams["text_config"]
|
||||
hparams["text_config"] = {
|
||||
"hidden_size": cfg["dim"],
|
||||
"num_hidden_layers": cfg["n_layer"],
|
||||
"num_attention_heads": cfg["n_head"],
|
||||
"num_key_value_heads": cfg["n_local_heads"],
|
||||
"head_dim": cfg["head_dim"],
|
||||
"intermediate_size": cfg["intermediate_size"],
|
||||
"rms_norm_eps": cfg["norm_eps"],
|
||||
"rope_theta": cfg["rope_base"],
|
||||
"max_position_embeddings": cfg["max_seq_len"],
|
||||
"vocab_size": cfg["vocab_size"],
|
||||
"tie_word_embeddings": cfg["tie_word_embeddings"],
|
||||
}
|
||||
return hparams
|
||||
|
||||
|
||||
def _neox_rows(w: Tensor, head_dim: int) -> Tensor:
|
||||
# checkpoint uses interleaved RoPE, reorder the rows of each head so NEOX RoPE gives the same result
|
||||
order = torch.cat([torch.arange(0, head_dim, 2), torch.arange(1, head_dim, 2)])
|
||||
return w.reshape(-1, head_dim, *w.shape[1:])[:, order].reshape(w.shape)
|
||||
|
||||
|
||||
@ModelBase.register("fish_qwen3_omni")
|
||||
@ModelBase.example("fishaudio/s2-pro")
|
||||
class FishS2TextModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# the rest is mapped by tensor_mapping.py
|
||||
name = name.removeprefix("text_model.")
|
||||
T = gguf.MODEL_TENSOR
|
||||
head_dim = self.hparams["head_dim"]
|
||||
if name.endswith(".attention.wqkv.weight"):
|
||||
n_q = self.hparams["num_attention_heads"] * head_dim
|
||||
n_kv = self.hparams["num_key_value_heads"] * head_dim
|
||||
q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
|
||||
yield (self.format_tensor_name(T.ATTN_Q, bid), _neox_rows(q, head_dim))
|
||||
yield (self.format_tensor_name(T.ATTN_K, bid), _neox_rows(k, head_dim))
|
||||
yield (self.format_tensor_name(T.ATTN_V, bid), v)
|
||||
return
|
||||
if name.endswith(".attention.q_norm.weight"):
|
||||
yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), _neox_rows(data_torch, head_dim))
|
||||
return
|
||||
if name.endswith(".attention.k_norm.weight"):
|
||||
yield (self.format_tensor_name(T.ATTN_K_NORM, bid), _neox_rows(data_torch, head_dim))
|
||||
return
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("fish_qwen3_omni")
|
||||
@ModelBase.example("fishaudio/s2-pro")
|
||||
class FishS2MmprojModel(Drama3MmprojModel):
|
||||
has_audio_encoder = True # codec encoder, for voice cloning
|
||||
gen_projector_type = gguf.VisionProjectorType.FISH_S2_GEN
|
||||
|
||||
_FAST_AR_MAP = {
|
||||
"attention_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
|
||||
"attention.wo.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
|
||||
"ffn_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
|
||||
"feed_forward.w1.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
|
||||
"feed_forward.w3.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP,
|
||||
"feed_forward.w2.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
|
||||
}
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
cfg = dict(self.global_config["audio_decoder_config"])
|
||||
cfg["num_hidden_layers"] = cfg["n_layer"]
|
||||
return cfg
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
# codec encoder: raw 44.1 kHz waveform -> backbone input rows, other hparams are hardcoded in tools/mtmd
|
||||
self.gguf_writer.add_clip_has_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.FISH_S2_SPKENC)
|
||||
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_audio_num_mel_bins(1)
|
||||
self.gguf_writer.add_audio_block_count(_N_ENC_TFM_LAYERS + _N_PRE_TFM_LAYERS)
|
||||
self.gguf_writer.add_audio_embedding_length(1024)
|
||||
self.gguf_writer.add_audio_feed_forward_length(3072)
|
||||
self.gguf_writer.add_audio_head_count(1024 // _CODEC_HEAD_DIM)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# the VQ picks the nearest codebook entry, small errors in the encoder change the codes
|
||||
if new_name.startswith("a.codec."):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# codec transformers: F16 gives codes closer to the reference than BF16
|
||||
if new_name.startswith("a.blk.") and n_dims == 2:
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name = item[0]
|
||||
if name.startswith("audio_decoder.") or name == "text_model.model.embeddings.weight":
|
||||
return item
|
||||
return None
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
T = gguf.MODEL_TENSOR
|
||||
cfg = self.hparams
|
||||
|
||||
# already named by generate_extra_tensors()
|
||||
if name.startswith("a."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
if name == "text_model.model.embeddings.weight":
|
||||
# semantic band of the backbone embeddings
|
||||
begin = self.global_config["semantic_start_token_id"]
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch[begin:begin + cfg["vocab_size"]])
|
||||
return
|
||||
|
||||
if name == "audio_decoder.codebook_embeddings.weight":
|
||||
# backbone input of a speech row, one table per codebook
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), data_torch.reshape(cfg["num_codebooks"], -1, data_torch.shape[-1]))
|
||||
return
|
||||
|
||||
if name == "audio_decoder.embeddings.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_TOK_EMBD), data_torch)
|
||||
return
|
||||
|
||||
if name == "audio_decoder.output.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch[:_N_RESIDUAL_CODES])
|
||||
return
|
||||
|
||||
if name == "audio_decoder.norm.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_OUTPUT_NORM), data_torch)
|
||||
return
|
||||
|
||||
assert bid is not None and name.startswith(f"audio_decoder.layers.{bid}.")
|
||||
key = name.removeprefix(f"audio_decoder.layers.{bid}.")
|
||||
if key == "attention.wqkv.weight":
|
||||
head_dim = cfg["head_dim"]
|
||||
n_q = cfg["n_head"] * head_dim
|
||||
n_kv = cfg["n_local_heads"] * head_dim
|
||||
q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, bid), _neox_rows(q, head_dim))
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, bid), _neox_rows(k, head_dim))
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, bid), v)
|
||||
return
|
||||
yield (self.format_tensor_name(self._FAST_AR_MAP[key], bid), data_torch)
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
yield from super().generate_extra_tensors()
|
||||
|
||||
sd = load_codec(self.dir_model)
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
def get(name: str) -> Tensor:
|
||||
return sd[name].float()
|
||||
|
||||
def tn(tensor: gguf.MODEL_TENSOR, bid: int | None = None, xid: int | None = None) -> str:
|
||||
return gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
|
||||
|
||||
def conv_wb(tensor: gguf.MODEL_TENSOR, prefix: str, bid: int | None = None, xid: int | None = None, stride: int = 1):
|
||||
# taps-major [K / stride, OC, stride * IC], see conv() in tools/mtmd/models/fish-s2-spkenc.cpp
|
||||
w = fold_conv(sd, prefix)
|
||||
n_out, n_in, k = w.shape
|
||||
w = w.reshape(n_out, n_in, k // stride, stride).permute(2, 0, 3, 1).reshape(k // stride, n_out, stride * n_in)
|
||||
yield (tn(tensor, bid, xid) + ".weight", w)
|
||||
yield (tn(tensor, bid, xid) + ".bias", get(prefix + ".conv.bias"))
|
||||
|
||||
def snake_ab(tensor: gguf.MODEL_TENSOR, name: str, bid: int | None = None, xid: int | None = None):
|
||||
a, b = fold_snake(get(name))
|
||||
yield (tn(tensor, bid, xid) + ".alpha", a)
|
||||
yield (tn(tensor, bid, xid) + ".beta", b)
|
||||
|
||||
def tfm_layers(prefix: str, n_layer: int, offset: int):
|
||||
for il in range(n_layer):
|
||||
p = f"{prefix}layers.{il}."
|
||||
bid = offset + il
|
||||
wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0)
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_Q, bid), _neox_rows(wq, _CODEC_HEAD_DIM))
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_K, bid), _neox_rows(wk, _CODEC_HEAD_DIM))
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_V, bid), wv)
|
||||
yield (self.format_tensor_name(T.A_ENC_OUTPUT, bid), get(p + "attention.wo.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_INPUT_NORM, bid), get(p + "attention_norm.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_SCALE, bid), get(p + "attention_layer_scale.gamma"))
|
||||
yield (self.format_tensor_name(T.A_ENC_OUTPUT_NORM, bid), get(p + "ffn_norm.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_GATE, bid), get(p + "feed_forward.w1.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_UP, bid), get(p + "feed_forward.w3.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_DOWN, bid), get(p + "feed_forward.w2.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_SCALE_LS, bid), get(p + "ffn_layer_scale.gamma"))
|
||||
|
||||
# encoder: entry conv, 4x (3 residual units, Snake, strided conv), Snake, post conv
|
||||
e = "encoder.block."
|
||||
yield from conv_wb(T.A_CODEC_ENC_ENTRY, e + "0")
|
||||
for bid, stride in enumerate(_CODEC_ENC_STRIDES):
|
||||
p = f"{e}{bid + 1}.block."
|
||||
for xid in range(3):
|
||||
r = f"{p}{xid}.block."
|
||||
yield from snake_ab(T.A_CODEC_ENC_RES_ACT1, r + "0.alpha", bid, xid)
|
||||
yield from conv_wb(T.A_CODEC_ENC_RES_CONV1, r + "1", bid, xid)
|
||||
yield from snake_ab(T.A_CODEC_ENC_RES_ACT2, r + "2.alpha", bid, xid)
|
||||
yield from conv_wb(T.A_CODEC_ENC_RES_CONV2, r + "3", bid, xid)
|
||||
yield from snake_ab(T.A_CODEC_ENC_SNAKE, p + "3.alpha", bid)
|
||||
yield from conv_wb(T.A_CODEC_ENC_CONV, p + "4", bid, stride=stride)
|
||||
# the last block ends with a windowed transformer
|
||||
yield from tfm_layers(f"{e}4.block.5.", _N_ENC_TFM_LAYERS, 0)
|
||||
yield (self.format_tensor_name(T.A_POST_NORM), get(f"{e}4.block.5.norm.weight"))
|
||||
yield from snake_ab(T.A_CODEC_ENC_POST_SNAKE, e + "5.alpha")
|
||||
yield from conv_wb(T.A_CODEC_ENC_POST_CONV, e + "6")
|
||||
|
||||
# quantizer: 2x (strided conv + ConvNeXt), windowed transformer, then semantic + residual VQ
|
||||
q = "quantizer."
|
||||
for bid in range(2):
|
||||
p = f"{q}downsample.{bid}."
|
||||
yield from conv_wb(T.A_CODEC_DOWN_CONV, p + "0", bid, stride=2)
|
||||
# depthwise: one [C] row per tap
|
||||
yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".weight", fold_conv(sd, p + "1.dwconv").squeeze(1).T.contiguous())
|
||||
yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".bias", get(p + "1.dwconv.conv.bias"))
|
||||
for key, tensor in (("norm", T.A_CODEC_DOWN_NORM), ("pwconv1", T.A_CODEC_DOWN_PW1), ("pwconv2", T.A_CODEC_DOWN_PW2)):
|
||||
yield (tn(tensor, bid) + ".weight", get(f"{p}1.{key}.weight"))
|
||||
yield (tn(tensor, bid) + ".bias", get(f"{p}1.{key}.bias"))
|
||||
yield (tn(T.A_CODEC_DOWN_GAMMA, bid), get(p + "1.gamma"))
|
||||
yield from tfm_layers(q + "pre_module.", _N_PRE_TFM_LAYERS, _N_ENC_TFM_LAYERS)
|
||||
yield (self.format_tensor_name(T.A_CODEC_PRE_NORM), get(q + "pre_module.norm.weight"))
|
||||
|
||||
# nearest codebook entry by cosine similarity, see VectorQuantize.decode_latents()
|
||||
def vq(prefix: str) -> tuple[Tensor, Tensor, Tensor]:
|
||||
w = fold_wn(sd[prefix + ".in_proj.weight_g"], sd[prefix + ".in_proj.weight_v"])
|
||||
cb = torch.nn.functional.normalize(get(prefix + ".codebook.weight"), dim=-1)
|
||||
return w, get(prefix + ".in_proj.bias"), cb
|
||||
|
||||
w, b, cb = vq(q + "semantic_quantizer.quantizers.0")
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".weight"), w)
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".bias"), b)
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_CB), cb)
|
||||
n_rest = self.hparams["num_codebooks"] - 1
|
||||
rest = [vq(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)]
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".weight"), torch.stack([r[0] for r in rest]))
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".bias"), torch.stack([r[1] for r in rest]))
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_CB), torch.stack([r[2] for r in rest]))
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, gguf
|
||||
from .qwen import Qwen3Model
|
||||
from .ultravox import WhisperEncoderModel
|
||||
|
||||
|
||||
# plain ChatML as built by transcribe.py upstream, the Qwen3 template renders past assistant turns differently from the last one
|
||||
HIGGS_AUDIO_CHAT_TEMPLATE = (
|
||||
"{%- for message in messages -%}"
|
||||
"{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}"
|
||||
"{%- endfor -%}"
|
||||
"{%- if add_generation_prompt -%}"
|
||||
"{{- '<|im_start|>assistant\\n' -}}"
|
||||
"{%- if enable_thinking is defined and enable_thinking is false -%}"
|
||||
"{{- '<think>\\n\\n</think>\\n\\n' -}}"
|
||||
"{%- endif -%}"
|
||||
"{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
@ModelBase.register("HiggsAudio3Model")
|
||||
@ModelBase.example("bosonai/higgs-audio-v3-stt")
|
||||
class HiggsAudioModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
def set_vocab(self):
|
||||
super().set_vocab()
|
||||
self.gguf_writer.add_chat_template(HIGGS_AUDIO_CHAT_TEMPLATE)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
# the audio_* tensors are dropped by TextModel, rename the text head before that
|
||||
if name == "audio_decoder_proj.text_lm_head.weight":
|
||||
name = "lm_head.weight"
|
||||
else:
|
||||
name = "model." + name
|
||||
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
|
||||
@ModelBase.register("HiggsAudio3Model")
|
||||
class HiggsAudioWhisperEncoderModel(WhisperEncoderModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
return self.global_config["audio_encoder_config"]
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# depthwise conv kernel, same as the whisper convs
|
||||
if name == "audio_encoder_proj.temporal.weight":
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
assert self.global_config["projector_type"] == "mlp" and self.global_config["projector_temporal_downsample"] == 2
|
||||
assert self.global_config["audio_adapter_type"] == "stack" and not self.global_config.get("duplicate_audio_triplet", False)
|
||||
super().set_gguf_parameters()
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.HIGGS_AUDIO)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not name.startswith(("audio_tower.", "audio_encoder_proj.")):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
@@ -302,6 +302,10 @@ class Inkling2VisionModel(MmprojModel):
|
||||
w.add_vision_attention_layernorm_eps(config["layer_norm_eps"])
|
||||
w.add_vision_spatial_merge_size(config["spatial_merge_size"])
|
||||
|
||||
def dequant_model(self):
|
||||
# vision tensors are BF16, only the text experts are quantized
|
||||
return
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
name, gen = item
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import MmprojModel, ModelBase, gguf
|
||||
from .qwen import Qwen2Model
|
||||
|
||||
|
||||
# the upstream chat_template.jinja is a processor template (list content, audio parts), use plain ChatML with its default system prompt
|
||||
VIBEVOICE_ASR_CHAT_TEMPLATE = (
|
||||
"{%- if messages[0]['role'] == 'system' -%}"
|
||||
"{{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' -}}"
|
||||
"{%- set loop_messages = messages[1:] -%}"
|
||||
"{%- else -%}"
|
||||
"{{- '<|im_start|>system\\nYou are a helpful assistant that transcribes audio input into text output in JSON format.<|im_end|>\\n' -}}"
|
||||
"{%- set loop_messages = messages -%}"
|
||||
"{%- endif -%}"
|
||||
"{%- for message in loop_messages -%}"
|
||||
"{{- '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>\\n' -}}"
|
||||
"{%- endfor -%}"
|
||||
"{%- if add_generation_prompt -%}"
|
||||
"{{- '<|im_start|>assistant\\n' -}}"
|
||||
"{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
@ModelBase.register("VibeVoiceAsrForConditionalGeneration")
|
||||
@ModelBase.example("microsoft/VibeVoice-ASR-HF")
|
||||
class VibeVoiceAsrModel(Qwen2Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN2
|
||||
|
||||
def set_vocab(self):
|
||||
super().set_vocab()
|
||||
self.gguf_writer.add_chat_template(VIBEVOICE_ASR_CHAT_TEMPLATE)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not name.startswith("language_model."):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
|
||||
@ModelBase.register("VibeVoiceAsrForConditionalGeneration")
|
||||
class VibeVoiceAsrAudioModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
|
||||
# values hard-coded in tools/mtmd/models/vibevoice-asr.cpp
|
||||
_encoder_consts = {"channels": 1, "kernel_size": 7, "num_filters": 32, "ffn_expansion": 4, "hidden_act": "gelu", "rms_norm_eps": 1e-5}
|
||||
_encoders = ("acoustic_tokenizer_encoder", "semantic_tokenizer_encoder")
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
c = self.global_config["acoustic_tokenizer_encoder_config"]
|
||||
return {
|
||||
**c,
|
||||
"intermediate_size": c["ffn_expansion"] * c["hidden_size"],
|
||||
"num_attention_heads": 1,
|
||||
"num_hidden_layers": sum(c["depths"]),
|
||||
}
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
for enc in self._encoders:
|
||||
c = self.global_config[f"{enc}_config"]
|
||||
for k, v in self._encoder_consts.items():
|
||||
assert c[k] == v, f"{enc}.{k} = {c[k]}, expected {v}"
|
||||
assert c["depths"] == self.hparams["depths"] and c["downsampling_ratios"] == self.hparams["downsampling_ratios"]
|
||||
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.VIBEVOICE_ASR)
|
||||
# the input is the raw waveform, one channel takes the place of the mel bins
|
||||
self.gguf_writer.add_audio_num_mel_bins(1)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams["rms_norm_eps"])
|
||||
self.gguf_writer.add_audio_conv_kernel_size(self.hparams["kernel_size"])
|
||||
# stride of each stage, the stem has none
|
||||
self.gguf_writer.add_audio_patch_sizes([1] + [int(r) for r in self.hparams["downsampling_ratios"]])
|
||||
self.gguf_writer.add_audio_depths([int(d) for d in self.hparams["depths"]])
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
if ".dw." in new_name:
|
||||
return gguf.GGMLQuantizationType.F32 # ggml_ssm_conv
|
||||
if ".conv." in new_name and new_name.endswith(".weight"):
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if not name.startswith(cls._encoders + ("multi_modal_projector.",)):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
_block_tensors = {
|
||||
"norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_NORM,
|
||||
"mixer.conv.weight": gguf.MODEL_TENSOR.A_ENC_CNX_DW,
|
||||
"mixer.conv.bias": gguf.MODEL_TENSOR.A_ENC_CNX_DW,
|
||||
"gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS1,
|
||||
"ffn_norm.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_NORM,
|
||||
"ffn.linear1.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP,
|
||||
"ffn.linear1.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_UP,
|
||||
"ffn.linear2.weight": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
|
||||
"ffn.linear2.bias": gguf.MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
|
||||
"ffn_gamma": gguf.MODEL_TENSOR.A_ENC_CNX_LS2,
|
||||
}
|
||||
|
||||
# mm.a.mlp.{0,1} + mm.a.mlp_norm.0 for the acoustic connector, {2,3} + 2 for the semantic one
|
||||
_proj_tensors = {
|
||||
"acoustic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 0),
|
||||
"acoustic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 0),
|
||||
"acoustic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 1),
|
||||
"semantic_linear_1": (gguf.MODEL_TENSOR.A_MMPROJ, 2),
|
||||
"semantic_norm": (gguf.MODEL_TENSOR.A_MMPROJ_NORM, 2),
|
||||
"semantic_linear_2": (gguf.MODEL_TENSOR.A_MMPROJ, 3),
|
||||
}
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# the layer scales (gamma) have no suffix upstream
|
||||
suffix = "." + name.rsplit(".", 1)[-1] if name.endswith((".weight", ".bias")) else ".weight"
|
||||
|
||||
if name.startswith("multi_modal_projector."):
|
||||
key, idx = self._proj_tensors[name.split(".")[1]]
|
||||
yield (self.format_tensor_name(key, idx, suffix=suffix), data_torch)
|
||||
return
|
||||
|
||||
enc, _, rest = name.partition(".")
|
||||
enc_id = self._encoders.index(enc)
|
||||
parts = rest.split(".")
|
||||
depths = [int(d) for d in self.hparams["depths"]]
|
||||
|
||||
# causal convs of the stages: 0 = stem, 1..6 = downsample, 7 = head
|
||||
if rest.startswith("stem.conv."):
|
||||
conv_id = 0
|
||||
elif parts[0] == "conv_layers" and parts[2] == "conv":
|
||||
conv_id = int(parts[1]) + 1
|
||||
elif parts[0] == "head":
|
||||
conv_id = len(depths)
|
||||
else:
|
||||
conv_id = None
|
||||
if conv_id is not None:
|
||||
yield (gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_CNX_CONV].format(bid=enc_id, xid=conv_id) + suffix, data_torch)
|
||||
return
|
||||
|
||||
# ConvNeXt blocks are numbered per stage upstream and flat here
|
||||
if parts[0] == "stem":
|
||||
stage, blk, tail = 0, int(parts[2]), ".".join(parts[3:])
|
||||
else:
|
||||
assert parts[0] == "conv_layers" and parts[2] == "stage", name
|
||||
stage, blk, tail = int(parts[1]) + 1, int(parts[3]), ".".join(parts[4:])
|
||||
xid = sum(depths[:stage]) + blk
|
||||
|
||||
key = self._block_tensors[tail]
|
||||
if key == gguf.MODEL_TENSOR.A_ENC_CNX_DW and tail.endswith(".weight"):
|
||||
data_torch = data_torch.squeeze(1) # [C, 1, K] -> [C, K] for ggml_ssm_conv
|
||||
yield (gguf.TENSOR_NAMES[key].format(bid=enc_id, xid=xid) + suffix, data_torch)
|
||||
@@ -0,0 +1,234 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, MmprojModel, gguf, logger
|
||||
from .qwen import QwenModel, Qwen3Model
|
||||
|
||||
# token ids of the yue2_infer protocol (protocol.py)
|
||||
EOD = 151643
|
||||
ABC_START, ABC_END = 151847, 151848
|
||||
MUSIC_START, MUSIC_END = 151851, 151852
|
||||
CODEC_OFFSET, CODEC_SIZE = 151853, 32768
|
||||
LATENT_START = 184621
|
||||
|
||||
# the VAE is a separate repo (m-a-p/YuE2-Vae), downloaded into this subdir of the model dir
|
||||
VAE_DIR = "vae"
|
||||
|
||||
INSTRUCTIONS = {
|
||||
"off": "Generate music with codec tokens from the given conditions.",
|
||||
"melody": "Generate a melody-only ABC transcription without chord symbols, then generate music with codec tokens from the given conditions.",
|
||||
"full": "Generate a chord-annotated ABC transcription, then generate music with codec tokens from the given conditions.",
|
||||
}
|
||||
|
||||
# the text is "[Tags]\n<style>\n[Lyrics]\n<lyrics>", optionally followed by "\n[ABC]\n<score>" (cover mode)
|
||||
# voice picks the planning mode: full (default), melody or off
|
||||
TTS_TEMPLATE = (
|
||||
"{%- set text = messages[0]['content'] -%}"
|
||||
"{%- set mode = (voice | lower) if voice is defined and voice and (voice | lower) in ['full', 'melody', 'off'] else 'full' -%}"
|
||||
"{%- set parts = text.split('\\n[ABC]\\n') -%}"
|
||||
"{%- set instructions = {"
|
||||
+ ", ".join(f"'{k}': '{v}'" for k, v in INSTRUCTIONS.items())
|
||||
+ "} -%}"
|
||||
"{{- '<|endoftext|>' + instructions[mode] + '\\n' + parts[0].rstrip() + '\\n' -}}"
|
||||
"{%- if mode == 'off' -%}"
|
||||
"{{- '<abc></abc><|music_start|>' -}}"
|
||||
"{%- elif parts | length > 1 -%}"
|
||||
"{{- '<abc>' + parts[1] + '</abc><|music_start|>' -}}"
|
||||
"{%- else -%}"
|
||||
"{{- '<abc>' -}}"
|
||||
"{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
def _is_nar_tensor(name: str) -> bool:
|
||||
return ".nar_" in name or name.startswith(("llm2vae.", "vae2llm.", "time_embedder.", "latent_pos_embed."))
|
||||
|
||||
|
||||
@ModelBase.register("YuE2ForCausalLM")
|
||||
class YuE2TextModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
# the NAR path goes to the mmproj
|
||||
if _is_nar_tensor(item[0]):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
# the codes are sampled with classifier-free guidance, the uncond prompt runs as a second sequence
|
||||
self.gguf_writer.add_classifier_free_guidance(True)
|
||||
|
||||
def set_vocab(self):
|
||||
# checkpoint-native tiktoken file, see tokenization_yue2.py
|
||||
ranks: dict[bytes, int] = {}
|
||||
with open(self.dir_model / "qwen.tiktoken", "rb") as f:
|
||||
for line in f.read().splitlines():
|
||||
if line:
|
||||
tok, rank = line.split()
|
||||
ranks[base64.b64decode(tok)] = int(rank)
|
||||
if len(ranks) != EOD:
|
||||
raise ValueError(f"YuE2: expected {EOD} BPE tokens, got {len(ranks)}")
|
||||
|
||||
vocab_size = self.hparams["vocab_size"]
|
||||
tokens: list[str] = [""] * vocab_size
|
||||
toktypes: list[int] = [gguf.TokenType.UNUSED] * vocab_size
|
||||
merges: list[str] = []
|
||||
for tok, rank in ranks.items():
|
||||
tokens[rank] = QwenModel.token_bytes_to_string(tok)
|
||||
toktypes[rank] = gguf.TokenType.NORMAL
|
||||
if len(tok) > 1:
|
||||
merged = QwenModel.bpe(ranks, tok, max_rank=rank)
|
||||
assert len(merged) == 2
|
||||
merges.append(" ".join(map(QwenModel.token_bytes_to_string, merged)))
|
||||
|
||||
specials = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<R>", "<S>", "<X>", "<mask>", "<sep>"]
|
||||
specials += [f"<extra_{i}>" for i in range(200)]
|
||||
specials[204:206] = ["<abc>", "</abc>"]
|
||||
named = {EOD + i: s for i, s in enumerate(specials)}
|
||||
named[MUSIC_START] = "<|music_start|>"
|
||||
named[MUSIC_END] = "<|music_end|>"
|
||||
for i in range(CODEC_SIZE):
|
||||
named[CODEC_OFFSET + i] = f"<|codec_{i}|>"
|
||||
for i, s in enumerate(["<|latent_start|>", "<|latent_end|>", "<|latent_pad|>"]):
|
||||
named[LATENT_START + i] = s
|
||||
for i in range(EOD, vocab_size):
|
||||
tokens[i] = named.get(i, f"[PAD{i}]")
|
||||
toktypes[i] = gguf.TokenType.CONTROL if i in named else gguf.TokenType.UNUSED
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre("qwen2")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
self.gguf_writer.add_token_merges(merges)
|
||||
self.gguf_writer.add_bos_token_id(EOD)
|
||||
self.gguf_writer.add_eos_token_id(MUSIC_END)
|
||||
self.gguf_writer.add_pad_token_id(EOD)
|
||||
self.gguf_writer.add_add_bos_token(False)
|
||||
self.gguf_writer.add_chat_template([{"name": "tts", "template": TTS_TEMPLATE}])
|
||||
|
||||
|
||||
@ModelBase.register("YuE2ForCausalLM")
|
||||
class YuE2MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
# the NAR path: same shapes as the backbone layers
|
||||
return {k: self.global_config[k] for k in ("num_hidden_layers", "hidden_size", "intermediate_size",
|
||||
"num_attention_heads", "num_key_value_heads", "rms_norm_eps")}
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name = item[0]
|
||||
if name == "latent_pos_embed.pe":
|
||||
return None # sinusoidal, computed at runtime
|
||||
if _is_nar_tensor(name) or name == "model.norm.weight":
|
||||
return item
|
||||
return None
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.YUE2_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(hp["hidden_size"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(hp["intermediate_size"])
|
||||
self.gguf_writer.add_gen_audio_block_count(hp["num_hidden_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(hp["num_attention_heads"])
|
||||
self.gguf_writer.add_gen_audio_head_count_kv(hp["num_key_value_heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(hp["rms_norm_eps"])
|
||||
# rope theta, the ODE solver and the VAE hparams are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# read on the host, or small convs and projections of the flow / VAE graphs
|
||||
if not new_name.startswith("a.gen.nar.blk.") or n_dims < 2:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@staticmethod
|
||||
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
|
||||
return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
from safetensors.torch import load_file
|
||||
path = self.dir_model / VAE_DIR / "model.safetensors"
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"YuE2: VAE not found at {path}, download m-a-p/YuE2-Vae into it")
|
||||
logger.info(f"gguf: loading VAE decoder from '{path}'")
|
||||
sd = {k[len("decoder.layers."):]: v for k, v in load_file(str(path)).items() if k.startswith("decoder.layers.")}
|
||||
p = "a.gen.vae"
|
||||
|
||||
# SnakeBeta with log-scale params: x + 1/(exp(beta) + 1e-9) * sin(exp(alpha) * x)^2
|
||||
def snake(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
yield (f"{dst}.alpha", torch.exp(sd[src + ".alpha"].float()))
|
||||
yield (f"{dst}.beta", 1.0 / (torch.exp(sd[src + ".beta"].float()) + 1e-9))
|
||||
|
||||
yield (f"{p}.conv_in.weight", self._wn(sd, "0"))
|
||||
yield (f"{p}.conv_in.bias", sd["0.bias"])
|
||||
n_blk = sum(1 for k in sd if k.endswith(".layers.1.weight_g") and k.count(".") == 3)
|
||||
for b in range(n_blk):
|
||||
s = f"{b + 1}.layers"
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.0")
|
||||
yield (f"{p}.blk.{b}.up.weight", self._wn(sd, f"{s}.1"))
|
||||
yield (f"{p}.blk.{b}.up.bias", sd[f"{s}.1.bias"])
|
||||
for r in range(3):
|
||||
rs = f"{s}.{r + 2}.layers"
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake1", f"{rs}.0")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.weight", self._wn(sd, f"{rs}.1"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.bias", sd[f"{rs}.1.bias"])
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake2", f"{rs}.2")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.weight", self._wn(sd, f"{rs}.3"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.bias", sd[f"{rs}.3.bias"])
|
||||
yield from snake(f"{p}.post_snake", f"{n_blk + 1}")
|
||||
yield (f"{p}.conv_out.weight", self._wn(sd, f"{n_blk + 2}"))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name.startswith("a.gen.vae."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
p = "a.gen.nar"
|
||||
aux = {
|
||||
"model.norm.weight": f"{p}.output_norm.weight",
|
||||
"vae2llm.weight": f"{p}.inp.weight",
|
||||
"vae2llm.bias": f"{p}.inp.bias",
|
||||
"llm2vae.weight": f"{p}.out.weight",
|
||||
"llm2vae.bias": f"{p}.out.bias",
|
||||
"time_embedder.mlp.0.weight": f"{p}.time.0.weight",
|
||||
"time_embedder.mlp.0.bias": f"{p}.time.0.bias",
|
||||
"time_embedder.mlp.2.weight": f"{p}.time.2.weight",
|
||||
"time_embedder.mlp.2.bias": f"{p}.time.2.bias",
|
||||
}
|
||||
if name in aux:
|
||||
yield (aux[name], data_torch)
|
||||
return
|
||||
|
||||
# the NAR layers keep the libllama names, mtmd loads them as a text model
|
||||
assert bid is not None, name
|
||||
layer = {
|
||||
"nar_input_layernorm.weight": "attn_norm.weight",
|
||||
"nar_self_attn.q_proj.weight": "attn_q.weight",
|
||||
"nar_self_attn.k_proj.weight": "attn_k.weight",
|
||||
"nar_self_attn.v_proj.weight": "attn_v.weight",
|
||||
"nar_self_attn.o_proj.weight": "attn_output.weight",
|
||||
"nar_self_attn.q_norm.weight": "attn_q_norm.weight",
|
||||
"nar_self_attn.k_norm.weight": "attn_k_norm.weight",
|
||||
"nar_pre_mlp_layernorm.weight": "ffn_norm.weight",
|
||||
"nar_mlp.gate_proj.weight": "ffn_gate.weight",
|
||||
"nar_mlp.up_proj.weight": "ffn_up.weight",
|
||||
"nar_mlp.down_proj.weight": "ffn_down.weight",
|
||||
}
|
||||
key = name.split(".", 3)[3] # strip "model.layers.{bid}."
|
||||
yield (f"{p}.blk.{bid}.{layer[key]}", data_torch)
|
||||
@@ -0,0 +1,575 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, MmprojModel, LazyTorchTensor, gguf, logger
|
||||
|
||||
LM_DIR = Path("lm") / "zonos3_mini"
|
||||
|
||||
# codec decoder: quantizers and semantic head from the generator, the rest from the head checkpoint
|
||||
CODEC_GENERATOR = Path("autoencoder") / "generator.pt"
|
||||
CODEC_VARIANT = "keep2_plain"
|
||||
CODEC_HEAD = Path("autoencoder") / "heads" / CODEC_VARIANT / "step_0050000.pt"
|
||||
CODEC_STAGES = [0, 2, 4, 6, 8] # transformer stages of the decoder
|
||||
|
||||
DEFAULT_VOICE = "Savannah_en-American_F_1"
|
||||
|
||||
# speaker encoder (ownmect_s2), MECT_B2 blocks: (cnn depth, out channels, (stride f, stride t), expansion)
|
||||
SPK_CKPT = Path("speaker_encoders") / "ownmect_s2" / "ckpt_0037.pth"
|
||||
SPK_STAGES = [
|
||||
[(3, 32, (1, 1), 4)] * 2,
|
||||
[(3, 64, (2, 1), 2)] + [(3, 64, (1, 2), 2)] * 3,
|
||||
[(3, 128, (2, 1), 2)] + [(3, 128, (1, 2), 2)] * 4,
|
||||
[(3, 256, (2, 1), 1)] + [(2, 256, (1, 2), 1)] * 2,
|
||||
]
|
||||
VOICE_MAX_FRAMES = 900 # reference_max_frames of the reference
|
||||
|
||||
# GDN geometry, hard-coded in the reference (zonos3.reference.lm)
|
||||
GDN_HEADS, GDN_HEAD_DIM, GDN_KERNEL = 4, 128, 4
|
||||
|
||||
# the text table keeps the tokenizer ids shifted by this offset, ids 2 and 3 are BOS and EOS
|
||||
TEXT_OFFSET = 4
|
||||
TEXT_BOS, TEXT_EOS = 2, 3
|
||||
|
||||
|
||||
def _round_up(value: float, multiple: int) -> int:
|
||||
return multiple * ((int(value) + multiple - 1) // multiple)
|
||||
|
||||
|
||||
def _load_params(dir_model: Path) -> dict[str, Any] | None:
|
||||
path = dir_model / LM_DIR / "params.json"
|
||||
if not path.is_file():
|
||||
return None
|
||||
with open(path, encoding="utf-8") as f:
|
||||
params = json.load(f).get("model", {})
|
||||
return params if params.get("model_type") == "zonos2" else None
|
||||
|
||||
|
||||
def _is_zonos3_checkpoint(dir_model: Path) -> bool:
|
||||
return _load_params(dir_model) is not None
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_zonos3_checkpoint)
|
||||
def _load_zonos3_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected ZONOS3 checkpoint")
|
||||
params = _load_params(dir_model)
|
||||
assert params is not None
|
||||
for key, want in (("attention_pattern", "gdn3_global1"), ("residual_type", "full_attnres"),
|
||||
("moe_router_type", "linear"), ("moe_router_use_eda", False), ("use_xsa", False),
|
||||
("moe_end_from_layer", 0), ("speaker_enabled", True)):
|
||||
if params.get(key) != want:
|
||||
raise NotImplementedError(f"ZONOS3: unsupported {key} = {params.get(key)!r}, expected {want!r}")
|
||||
text_config = dict(params, hidden_size=params["dim"])
|
||||
return {"architectures": ["Zonos3Model"], "text_config": text_config, "audio_config": {"num_hidden_layers": 0}}
|
||||
|
||||
|
||||
def _index_lm_tensors(model: ModelBase) -> dict[str, Callable[[], Tensor]]:
|
||||
path = model.dir_model / LM_DIR / "model.pth"
|
||||
logger.info(f"gguf: indexing model part '{LM_DIR / 'model.pth'}'")
|
||||
state = torch.load(str(path), map_location="cpu", mmap=True, weights_only=True)
|
||||
tensors: dict[str, Callable[[], Tensor]] = {}
|
||||
for name, data in state.items():
|
||||
gen = (lambda data=data: LazyTorchTensor.from_eager(data)) if model.lazy else (lambda data=data: data)
|
||||
titem = model.filter_tensors((name, gen))
|
||||
model._source_tensor_names[name] = titem[0] if titem else None
|
||||
if titem:
|
||||
tensors[titem[0]] = titem[1]
|
||||
return tensors
|
||||
|
||||
|
||||
@ModelBase.register("Zonos3Model")
|
||||
class Zonos3TextModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.ZONOS3
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
hp = self.hparams
|
||||
self.n_embd = hp["dim"]
|
||||
self.n_codebooks = hp["n_codebooks"]
|
||||
self.audio_vocab = hp["codebook_size"] + 2 # codes, EOA, pad
|
||||
self.audio_pad_id = hp["audio_pad_id"]
|
||||
self.text_rows = hp["text_vocab"] + 1 # + the padding row, used by the generated rows
|
||||
self.cond_counts = [n + 1 for n in hp["condition_bucket_counts"]] # + the unknown bucket
|
||||
self.gdn_period = int(hp["attention_pattern"][len("gdn"):].split("_")[0]) + 1
|
||||
self.n_ff = _round_up(hp["ffn_dim_multiplier"] * hp["dim"], hp["multiple_of"])
|
||||
self.n_ff_exp = _round_up(hp["moe_ffn_dim_multiplier"] * hp["dim"], hp["moe_multiple_of"])
|
||||
self.n_latent = int(hp["dim"] * hp["moe_alpha"])
|
||||
self._embd: dict[str, Tensor] = {}
|
||||
self._router: dict[int, dict[str, Tensor]] = {}
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
return _index_lm_tensors(self)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
# audio heads and speaker projection go to the mmproj
|
||||
if name.startswith(("multi_output.", "speaker_projection.")):
|
||||
return None
|
||||
return item
|
||||
|
||||
def set_vocab(self):
|
||||
with open(self.dir_model / LM_DIR / "tokenizer" / "superbpe8192.json", encoding="utf-8") as f:
|
||||
tok = json.load(f)
|
||||
model = tok["model"]
|
||||
if model["type"] != "BPE" or model.get("byte_fallback") or tok.get("normalizer") is not None:
|
||||
raise NotImplementedError("ZONOS3: unexpected tokenizer type")
|
||||
|
||||
vocab: dict[str, int] = model["vocab"]
|
||||
n_bpe = len(vocab)
|
||||
if n_bpe + TEXT_OFFSET != self.hparams["text_vocab"]:
|
||||
raise ValueError(f"ZONOS3: tokenizer has {n_bpe} tokens, text_vocab is {self.hparams['text_vocab']}")
|
||||
added = {t["id"] for t in tok.get("added_tokens", [])}
|
||||
|
||||
tokens: list[str] = ["<|unused_0|>", "<|unused_1|>", "<|bos|>", "<|eos|>"]
|
||||
toktypes: list[int] = [gguf.TokenType.CONTROL] * TEXT_OFFSET
|
||||
inv = {i: t for t, i in vocab.items()}
|
||||
for i in range(n_bpe):
|
||||
tokens.append(inv[i])
|
||||
toktypes.append(gguf.TokenType.CONTROL if i in added else gguf.TokenType.NORMAL)
|
||||
|
||||
# padding row of the text table (the text column of every generated row)
|
||||
tokens.append("<|text_pad|>")
|
||||
toktypes.append(gguf.TokenType.CONTROL)
|
||||
|
||||
# condition rows, the last bucket of each table is "unknown"
|
||||
for cond, n in zip(self.hparams["condition_names"], self.cond_counts):
|
||||
tokens += [f"<|{cond}_{i}|>" for i in range(n - 1)] + [f"<|{cond}_unk|>"]
|
||||
toktypes += [gguf.TokenType.CONTROL] * n
|
||||
|
||||
merges = [" ".join(m) if isinstance(m, list) else m for m in model["merges"]]
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre("zonos3")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
self.gguf_writer.add_token_merges(merges)
|
||||
self.gguf_writer.add_bos_token_id(TEXT_BOS)
|
||||
self.gguf_writer.add_eos_token_id(TEXT_EOS)
|
||||
self.gguf_writer.add_pad_token_id(TEXT_OFFSET + 1) # <|pad|> of the tokenizer
|
||||
# the reference wraps the text as [BOS] + ids + [EOS]
|
||||
self.gguf_writer.add_add_bos_token(True)
|
||||
self.gguf_writer.add_add_eos_token(True)
|
||||
# the reference normalizes the text with NeMo, the template covers the English numbers
|
||||
template_path = Path(__file__).parent.parent / "models" / "templates" / "Zyphra-ZONOS3-Mini-tts.jinja"
|
||||
with open(template_path, "r", encoding="utf-8") as f:
|
||||
self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}])
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
n_layer = self.block_count
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_block_count(n_layer)
|
||||
self.gguf_writer.add_context_length(hp["max_seqlen"])
|
||||
self.gguf_writer.add_embedding_length(self.n_embd)
|
||||
self.gguf_writer.add_feed_forward_length(self.n_ff)
|
||||
self.gguf_writer.add_head_count(hp["n_heads"])
|
||||
self.gguf_writer.add_head_count_kv(hp["n_kv_heads"])
|
||||
self.gguf_writer.add_key_length(hp["head_dim"])
|
||||
self.gguf_writer.add_value_length(hp["head_dim"])
|
||||
self.gguf_writer.add_rope_dimension_count(hp["head_dim"])
|
||||
self.gguf_writer.add_rope_freq_base(hp["rope_theta"])
|
||||
self.gguf_writer.add_layer_norm_rms_eps(hp["norm_eps"])
|
||||
self.gguf_writer.add_full_attention_interval(self.gdn_period)
|
||||
|
||||
# gated delta net
|
||||
self.gguf_writer.add_ssm_conv_kernel(GDN_KERNEL)
|
||||
self.gguf_writer.add_ssm_inner_size(GDN_HEADS * GDN_HEAD_DIM)
|
||||
self.gguf_writer.add_ssm_state_size(GDN_HEAD_DIM)
|
||||
self.gguf_writer.add_ssm_time_step_rank(GDN_HEADS)
|
||||
self.gguf_writer.add_ssm_group_count(GDN_HEADS)
|
||||
|
||||
# latent MoE, the gating (softmax over the selected logits) is hard-coded in libllama
|
||||
self.gguf_writer.add_leading_dense_block_count(hp["moe_start_from_layer"])
|
||||
self.gguf_writer.add_expert_count(hp["moe_n_experts"])
|
||||
self.gguf_writer.add_expert_used_count(hp["moe_router_topk"])
|
||||
self.gguf_writer.add_expert_feed_forward_length(self.n_ff_exp)
|
||||
self.gguf_writer.add_moe_latent_size(self.n_latent)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# router logits pick the experts, keep them exact
|
||||
if new_name.endswith(("ffn_gate_inp.weight", "exp_probs_b.bias")):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def _token_embd(self) -> Tensor:
|
||||
n_cb = self.n_codebooks
|
||||
pad = sum(self._embd[f"multi_embedder.embedders.{cb}.weight"][self.audio_pad_id].float() for cb in range(n_cb))
|
||||
text = self._embd[f"multi_embedder.embedders.{n_cb}.weight"].float()
|
||||
if text.shape[0] != self.text_rows:
|
||||
raise ValueError(f"ZONOS3: text table has {text.shape[0]} rows, expected {self.text_rows}")
|
||||
conds = [self._embd[f"condition_embeddings.{i}.weight"].float() for i in range(len(self.cond_counts))]
|
||||
for t, n in zip(conds, self.cond_counts):
|
||||
assert t.shape[0] == n
|
||||
# a text row is [pad x n_codebooks, text_id], condition rows replace the summed row
|
||||
return torch.cat([text + pad] + conds, dim=0)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
T = gguf.MODEL_TENSOR
|
||||
hp = self.hparams
|
||||
n_layer = self.block_count
|
||||
|
||||
if name.startswith(("multi_embedder.", "condition_embeddings.")):
|
||||
self._embd[name] = data_torch
|
||||
if len(self._embd) == self.n_codebooks + 1 + len(self.cond_counts):
|
||||
yield (self.format_tensor_name(T.TOKEN_EMBD), self._token_embd())
|
||||
self._embd.clear()
|
||||
return
|
||||
|
||||
if name == "block_attnres_queries":
|
||||
# one query per routing point: 2*il before the mixer, 2*il+1 before the FFN, the last one before out_norm
|
||||
assert data_torch.shape[0] == 2 * n_layer + 1
|
||||
for il in range(n_layer):
|
||||
yield (self.format_tensor_name(T.ATTN_RES_SCORE, il), data_torch[2 * il])
|
||||
yield (self.format_tensor_name(T.FFN_RES_SCORE, il), data_torch[2 * il + 1])
|
||||
yield (self.format_tensor_name(T.OUTPUT_RES_SCORE), data_torch[2 * n_layer])
|
||||
return
|
||||
|
||||
if name == "out_norm.weight":
|
||||
yield (self.format_tensor_name(T.OUTPUT_NORM), data_torch)
|
||||
return
|
||||
|
||||
assert bid is not None, name
|
||||
key = name.split(".", 2)[2] # strip "layers.{bid}."
|
||||
|
||||
if key == "attention_norm.weight":
|
||||
yield (self.format_tensor_name(T.ATTN_NORM, bid), data_torch)
|
||||
return
|
||||
if key == "ffn_norm.weight":
|
||||
yield (self.format_tensor_name(T.FFN_NORM, bid), data_torch)
|
||||
return
|
||||
|
||||
# gated delta net
|
||||
if key == "attention.in_proj.weight":
|
||||
# split the fused [qkv | z | a | b] weight into the qwen35 GDN tensors
|
||||
# attn_gate holds z, per-head output gate in the attention layers
|
||||
inner = GDN_HEADS * GDN_HEAD_DIM
|
||||
qkv, z, a, b = data_torch.split([3 * inner, inner, GDN_HEADS, GDN_HEADS], dim=0)
|
||||
yield (self.format_tensor_name(T.ATTN_QKV, bid), qkv)
|
||||
yield (self.format_tensor_name(T.ATTN_GATE, bid), z)
|
||||
yield (self.format_tensor_name(T.SSM_ALPHA, bid), a)
|
||||
yield (self.format_tensor_name(T.SSM_BETA, bid), b)
|
||||
return
|
||||
if key == "attention.conv.weight":
|
||||
yield (self.format_tensor_name(T.SSM_CONV1D, bid), data_torch.squeeze(1))
|
||||
return
|
||||
if key == "attention.A_log":
|
||||
yield (self.format_tensor_name(T.SSM_A, bid, suffix=""), -torch.exp(data_torch.float()))
|
||||
return
|
||||
if key == "attention.dt_bias":
|
||||
yield (self.format_tensor_name(T.SSM_DT, bid, suffix=".bias"), data_torch)
|
||||
return
|
||||
if key == "attention.out_weight":
|
||||
yield (self.format_tensor_name(T.SSM_NORM, bid), data_torch)
|
||||
return
|
||||
if key == "attention.wo.weight" and (bid + 1) % self.gdn_period != 0:
|
||||
yield (self.format_tensor_name(T.SSM_OUT, bid), data_torch)
|
||||
return
|
||||
|
||||
# global attention
|
||||
if key == "attention.wq.weight":
|
||||
yield (self.format_tensor_name(T.ATTN_Q, bid), data_torch)
|
||||
return
|
||||
if key == "attention.wkv.weight":
|
||||
yield (self.format_tensor_name(T.ATTN_K, bid), data_torch[0])
|
||||
yield (self.format_tensor_name(T.ATTN_V, bid), data_torch[1])
|
||||
return
|
||||
if key == "attention.wo.weight":
|
||||
yield (self.format_tensor_name(T.ATTN_OUT, bid), data_torch)
|
||||
return
|
||||
if key == "attention.gater.weight":
|
||||
# one sigmoid gate per head, same tensor name as the GDN z gate
|
||||
yield (self.format_tensor_name(T.ATTN_GATE, bid), data_torch)
|
||||
return
|
||||
if key == "attention.temp":
|
||||
# Q = rms_norm(q) * |temp|, the per-head scale is stored as the Q norm weight
|
||||
temp = data_torch.float().abs().reshape(hp["n_heads"], 1).expand(hp["n_heads"], hp["head_dim"])
|
||||
yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), temp.contiguous())
|
||||
return
|
||||
|
||||
# dense FFN (layer 0): out = w_out(w_in[0](x) * silu(w_in[1](x)))
|
||||
if key == "feed_forward.w_in.weight":
|
||||
yield (self.format_tensor_name(T.FFN_UP, bid), data_torch[0])
|
||||
yield (self.format_tensor_name(T.FFN_GATE, bid), data_torch[1])
|
||||
return
|
||||
if key == "feed_forward.w_out.weight":
|
||||
yield (self.format_tensor_name(T.FFN_DOWN, bid), data_torch)
|
||||
return
|
||||
|
||||
# latent MoE
|
||||
if key in ("feed_forward.router.router_linear.weight", "feed_forward.router.rmsnorm_eda.weight"):
|
||||
# logits = router_linear(rms_norm(x) * w), the norm weight is folded into the router columns
|
||||
r = self._router.setdefault(bid, {})
|
||||
r[key] = data_torch
|
||||
if len(r) == 2:
|
||||
w = r["feed_forward.router.router_linear.weight"].float() * r["feed_forward.router.rmsnorm_eda.weight"].float()[None, :]
|
||||
yield (self.format_tensor_name(T.FFN_GATE_INP, bid), w)
|
||||
del self._router[bid]
|
||||
return
|
||||
if key == "feed_forward.router.balancing_biases":
|
||||
# the reference selects on logits - bias
|
||||
yield (self.format_tensor_name(T.FFN_EXP_PROBS_B, bid, suffix=".bias"), -data_torch.float())
|
||||
return
|
||||
if key == "feed_forward.latent_down.weight":
|
||||
yield (self.format_tensor_name(T.MOE_LATENT_DOWN, bid), data_torch)
|
||||
return
|
||||
if key == "feed_forward.latent_up.weight":
|
||||
yield (self.format_tensor_name(T.MOE_LATENT_UP, bid), data_torch)
|
||||
return
|
||||
if key == "feed_forward.experts.gate_up_proj":
|
||||
gate, up = data_torch.chunk(2, dim=1)
|
||||
yield (self.format_tensor_name(T.FFN_GATE_EXP, bid), gate)
|
||||
yield (self.format_tensor_name(T.FFN_UP_EXP, bid), up)
|
||||
return
|
||||
if key == "feed_forward.experts.down_proj":
|
||||
yield (self.format_tensor_name(T.FFN_DOWN_EXP, bid), data_torch)
|
||||
return
|
||||
|
||||
raise ValueError(f"ZONOS3: unexpected tensor {name}")
|
||||
|
||||
|
||||
@ModelBase.register("Zonos3Model")
|
||||
class Zonos3MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
hp = self.global_config["text_config"]
|
||||
self.n_codebooks = hp["n_codebooks"]
|
||||
self.text_vocab = hp["text_vocab"]
|
||||
self._embd: dict[str, Tensor] = {}
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
return _index_lm_tensors(self)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
if name.startswith(("multi_output.", "multi_embedder.", "speaker_projection.")):
|
||||
return item
|
||||
return None
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
|
||||
# speaker encoder for voice cloning, the front-end hparams are hard-coded in tools/mtmd
|
||||
self.gguf_writer.add_clip_has_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.ZONOS3_SPKENC)
|
||||
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_audio_num_mel_bins(80)
|
||||
self.gguf_writer.add_audio_block_count(0)
|
||||
self.gguf_writer.add_audio_embedding_length(192)
|
||||
self.gguf_writer.add_audio_head_count(1)
|
||||
self.gguf_writer.add_audio_feed_forward_length(192)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
|
||||
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.ZONOS3_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(0)
|
||||
self.gguf_writer.add_gen_audio_block_count(0)
|
||||
self.gguf_writer.add_gen_audio_head_count(0)
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(self.global_config["text_config"]["norm_eps"])
|
||||
self.gguf_writer.add_gen_audio_model_variant(CODEC_VARIANT)
|
||||
if any(self.dir_model.glob(f"gguf/voices/*/{DEFAULT_VOICE}.codes.npy")):
|
||||
self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE)
|
||||
# softcap and the codec hparams are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# read on the host, or used by get_rows / ggml_ssm_conv / the conv ops of the speaker encoder
|
||||
if new_name.startswith(("voice.", "a.spk.")) or new_name.endswith((".table", ".dw.weight", ".mixer.weight", ".resample.weight")):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@staticmethod
|
||||
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
|
||||
return (g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))).squeeze(-1)
|
||||
|
||||
def _speaker_encoder(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# same layout as the reference ggml runtime (tools/convert_gguf.py speaker-encoder)
|
||||
sd = torch.load(self.dir_model / SPK_CKPT, map_location="cpu", weights_only=True)["modules"]["spk_model"]
|
||||
sd = {k[len("model."):]: v.double() for k, v in sd.items() if not k.endswith("num_batches_tracked")}
|
||||
p = "a.spk"
|
||||
|
||||
def bn(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
scale = sd[src + ".weight"] / torch.sqrt(sd[src + ".running_var"] + 1e-5)
|
||||
yield (f"{p}.{dst}.scale", scale)
|
||||
yield (f"{p}.{dst}.shift", sd[src + ".bias"] - sd[src + ".running_mean"] * scale)
|
||||
|
||||
def put(dst: str, t: Tensor) -> tuple[str, Tensor]:
|
||||
return (f"{p}.{dst}", t)
|
||||
|
||||
fe = "feature_extractor."
|
||||
yield put("fe.head.conv.weight", sd[fe + "head.0.weight"])
|
||||
yield put("fe.head.conv.bias", sd[fe + "head.0.bias"])
|
||||
yield from bn("fe.head.bn", fe + "head.1")
|
||||
c_in, f_in = 32, 80
|
||||
for si, stage in enumerate(SPK_STAGES):
|
||||
for k, (depth, out_ch, (s_f, s_t), expansion) in enumerate(stage):
|
||||
src, dst = f"{fe}mect_block{si + 1}.{k}.", f"fe.b{si + 1}.{k}."
|
||||
for r in range(depth):
|
||||
cs, cd = f"{src}cnn_block.{r}.", f"{dst}cnn.{r}."
|
||||
# grouped 3x3 convs, one depthwise kernel per input slot of a group
|
||||
for conv in ("conv1", "conv2"):
|
||||
w = sd[cs + conv + ".weight"]
|
||||
for j in range(w.shape[1]):
|
||||
yield put(f"{cd}{conv}.w{j}", w[:, j:j + 1])
|
||||
for i in (1, 2):
|
||||
yield put(f"{cd}pw{i}.weight", sd[f"{cs}conv_pw{i}.weight"])
|
||||
yield put(f"{cd}pw{i}.bias", sd[f"{cs}conv_pw{i}.bias"])
|
||||
yield from bn(f"{cd}bn{i}", f"{cs}bn{i}")
|
||||
if cs + "shortcut.0.weight" in sd:
|
||||
yield put(cd + "sc.weight", sd[cs + "shortcut.0.weight"])
|
||||
yield from bn(cd + "sc.bn", cs + "shortcut.1")
|
||||
if expansion > 1:
|
||||
yield put(dst + "reduce.weight", sd[src + "reduce_conv.0.weight"])
|
||||
yield put(dst + "reduce.bias", sd[src + "reduce_conv.0.bias"])
|
||||
yield from bn(dst + "reduce.bn", src + "reduce_conv.1")
|
||||
yield put(dst + "rproj.weight", sd[src + "reduce_project.0.weight"].squeeze(-1))
|
||||
yield put(dst + "rproj.bias", sd[src + "reduce_project.0.bias"])
|
||||
yield put(dst + "rproj.ln.weight", sd[src + "reduce_project.1.weight"])
|
||||
yield put(dst + "rproj.ln.bias", sd[src + "reduce_project.1.bias"])
|
||||
yield put(dst + "pos.dw.weight", sd[src + "pos_conv.pos_conv.0.weight"].squeeze(1))
|
||||
yield put(dst + "pos.dw.bias", sd[src + "pos_conv.pos_conv.0.bias"])
|
||||
yield from bn(dst + "pos.bn", src + "pos_conv.pos_conv.1")
|
||||
yield put(dst + "pos.pw.weight", sd[src + "pos_conv.pos_conv.3.weight"].squeeze(-1))
|
||||
yield put(dst + "pos.pw.bias", sd[src + "pos_conv.pos_conv.3.bias"])
|
||||
tr = src + "transformer."
|
||||
for a, b in (("q", "q_proj"), ("k", "k_proj"), ("v", "v_proj"), ("o", "out_proj")):
|
||||
yield put(f"{dst}attn.{a}.weight", sd[f"{tr}attention.{b}.weight"])
|
||||
yield put(f"{dst}attn.{a}.bias", sd[f"{tr}attention.{b}.bias"])
|
||||
yield put(dst + "ln1.weight", sd[tr + "layer_norm.weight"])
|
||||
yield put(dst + "ln1.bias", sd[tr + "layer_norm.bias"])
|
||||
moe = tr + "feed_forward.intermediate_dense.token_moe."
|
||||
yield put(dst + "moe.gate.weight", sd[moe + "gate.weight"])
|
||||
yield put(dst + "moe.experts.weight", sd[moe + "experts.weight"])
|
||||
yield put(dst + "ffn_out.weight", sd[tr + "feed_forward.output_dense.weight"])
|
||||
yield put(dst + "ffn_out.bias", sd[tr + "feed_forward.output_dense.bias"])
|
||||
yield put(dst + "ln2.weight", sd[tr + "final_layer_norm.weight"])
|
||||
yield put(dst + "ln2.bias", sd[tr + "final_layer_norm.bias"])
|
||||
yield put(dst + "eproj.weight", sd[src + "expansion_project.0.weight"].squeeze(-1))
|
||||
yield put(dst + "eproj.bias", sd[src + "expansion_project.0.bias"])
|
||||
c_in, f_in = c_in * s_f, f_in // s_f
|
||||
# the stage outputs are mixed by softmax weights
|
||||
yield put("fe.dense_w", torch.softmax(sd[fe + "dense3.inputs_weights"], dim=1)[0, :, :, 0])
|
||||
yield put("pool.l1.weight", sd["encoding_layers.linear1.weight"].squeeze(-1))
|
||||
yield put("pool.l1.bias", sd["encoding_layers.linear1.bias"])
|
||||
yield put("pool.l2.weight", sd["encoding_layers.linear2.weight"].squeeze(-1))
|
||||
yield put("pool.l2.bias", sd["encoding_layers.linear2.bias"])
|
||||
yield from bn("bn", "bn")
|
||||
yield put("linear.weight", sd["linear.weight"])
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
yield from self._speaker_encoder()
|
||||
logger.info(f"gguf: loading codec decoder from '{CODEC_GENERATOR}' and '{CODEC_HEAD}'")
|
||||
gen = torch.load(self.dir_model / CODEC_GENERATOR, map_location="cpu", mmap=True, weights_only=True)["generator"]
|
||||
head = torch.load(self.dir_model / CODEC_HEAD, map_location="cpu", mmap=True, weights_only=True)["trainable"]
|
||||
p = "a.gen.wav"
|
||||
|
||||
def convnext(dst: str, src: dict[str, Tensor], prefix: str) -> Iterable[tuple[str, Tensor]]:
|
||||
yield (f"{dst}.dw.weight", src[f"{prefix}.dwconv.weight"].squeeze(1))
|
||||
yield (f"{dst}.dw.bias", src[f"{prefix}.dwconv.bias"])
|
||||
yield (f"{dst}.norm.weight", src[f"{prefix}.norm.weight"])
|
||||
yield (f"{dst}.norm.bias", src[f"{prefix}.norm.bias"])
|
||||
yield (f"{dst}.pw1.weight", src[f"{prefix}.pwconv1.weight"])
|
||||
yield (f"{dst}.pw1.bias", src[f"{prefix}.pwconv1.bias"])
|
||||
yield (f"{dst}.pw2.weight", src[f"{prefix}.pwconv2.weight"])
|
||||
yield (f"{dst}.pw2.bias", src[f"{prefix}.pwconv2.bias"])
|
||||
|
||||
# RVQ codebooks folded through their out_proj, the biases are added once
|
||||
def rvq(prefix: str) -> tuple[Tensor, Tensor]:
|
||||
return gen[prefix + ".codebook.weight"].float() @ self._wn(gen, prefix + ".out_proj").T, gen[prefix + ".out_proj.bias"].float()
|
||||
|
||||
t, b = rvq("quantizer.semantic_quantizer.quantizers.0")
|
||||
yield (f"{p}.sem.table", t)
|
||||
yield (f"{p}.sem.bias", b)
|
||||
res = [rvq(f"quantizer.quantizer.quantizers.{i}") for i in range(self.n_codebooks - 1)]
|
||||
yield (f"{p}.res.table", torch.cat([t for t, _ in res], dim=0))
|
||||
# one bias per residual codebook, a frame decoded before all its codes are known leaves the missing ones out
|
||||
yield (f"{p}.res.bias", torch.stack([b for _, b in res]))
|
||||
|
||||
n_sem = sum(1 for k in gen if k.startswith("semantic_head.") and k.endswith(".dwconv.weight"))
|
||||
for i in range(n_sem):
|
||||
yield from convnext(f"{p}.sem.{i}", gen, f"semantic_head.{i}")
|
||||
yield (f"{p}.sem.out.weight", self._wn(gen, f"semantic_head.{n_sem}"))
|
||||
yield (f"{p}.sem.out.bias", gen[f"semantic_head.{n_sem}.bias"])
|
||||
|
||||
names = {"norm1.weight": "norm1.weight", "norm1.bias": "norm1.bias", "qkv.weight": "qkv.weight",
|
||||
"attention_out.weight": "wo.weight", "attention_scale": "attn_scale", "norm2.weight": "norm2.weight",
|
||||
"norm2.bias": "norm2.bias", "ffn.0.weight": "ffn1.weight", "ffn.3.weight": "ffn2.weight", "ffn_scale": "ffn_scale"}
|
||||
for si, s in enumerate(CODEC_STAGES):
|
||||
yield (f"{p}.stage.{si}.in.weight", head[f"stages.{s}.input_proj.weight"])
|
||||
if f"stages.{s}.output_proj.weight" in head:
|
||||
yield (f"{p}.stage.{si}.out.weight", head[f"stages.{s}.output_proj.weight"])
|
||||
n_layer = sum(1 for k in head if k.startswith(f"stages.{s}.layers.") and k.endswith(".qkv.weight"))
|
||||
for il in range(n_layer):
|
||||
for src, dst in names.items():
|
||||
yield (f"{p}.stage.{si}.blk.{il}.{dst}", head[f"stages.{s}.layers.{il}.{src}"])
|
||||
|
||||
n_head = sum(1 for k in head if k.startswith("head.blocks.") and k.endswith(".dwconv.weight"))
|
||||
for i in range(n_head):
|
||||
yield from convnext(f"{p}.head.{i}", head, f"head.blocks.{i}")
|
||||
yield (f"{p}.head.norm.weight", head["head.norm.weight"])
|
||||
yield (f"{p}.head.norm.bias", head["head.norm.bias"])
|
||||
# both iSTFT branches in one projection
|
||||
yield (f"{p}.head.out.weight", torch.cat([head["head.outs.0.weight"], head["head.outs.1.weight"]], dim=0))
|
||||
yield (f"{p}.head.out.bias", torch.cat([head["head.outs.0.bias"], head["head.outs.1.bias"]], dim=0))
|
||||
# read on the host
|
||||
yield (f"{p}.mixer.weight", head["head.mixer.weight"].squeeze(0))
|
||||
yield (f"{p}.resample.weight", gen["to_external.kernel"].squeeze(1))
|
||||
|
||||
# "reference" speaker mode, the default of the reference: the voice is a prefix of delayed code rows
|
||||
def _voices(self, tables: list[Tensor]) -> Iterable[tuple[str, Tensor]]:
|
||||
n_cb, pad = self.n_codebooks, self.global_config["text_config"]["audio_pad_id"]
|
||||
for path in sorted((self.dir_model / "gguf" / "voices").glob("*/*.codes.npy")):
|
||||
codes = torch.from_numpy(np.load(path, allow_pickle=False).astype(np.int64))
|
||||
if codes.ndim != 2 or codes.shape[1] != n_cb or codes.min() < 0 or codes.max() >= pad - 1:
|
||||
logger.warning(f"voice {path.name}: skipped, expected [frames, {n_cb}] codes")
|
||||
continue
|
||||
codes = codes[:VOICE_MAX_FRAMES]
|
||||
n_frames = codes.shape[0]
|
||||
# codebook cb of frame t is in row t + cb, the other cells are pad
|
||||
rows = torch.full((n_frames + n_cb - 1, n_cb), pad, dtype=torch.int64)
|
||||
for cb in range(n_cb):
|
||||
rows[cb:cb + n_frames, cb] = codes[:, cb]
|
||||
embd = sum(tables[cb][rows[:, cb]] for cb in range(n_cb))
|
||||
yield (f"voice.{path.name[:-len('.codes.npy')]}", embd)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
if name.startswith(("a.gen.wav.", "a.spk.")):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
# the speaker encoder ends with the speaker projection of the backbone
|
||||
if name.startswith("speaker_projection."):
|
||||
yield ("a.spk.proj." + name.split(".")[-1], data_torch)
|
||||
return
|
||||
|
||||
if name == "multi_output.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch)
|
||||
return
|
||||
|
||||
assert name.startswith("multi_embedder.")
|
||||
self._embd[name] = data_torch
|
||||
if len(self._embd) < self.n_codebooks + 1:
|
||||
return
|
||||
# every row built by the mmproj has text id = text_vocab, so that row is folded into codebook 0
|
||||
tables = [self._embd[f"multi_embedder.embedders.{cb}.weight"].float() for cb in range(self.n_codebooks)]
|
||||
tables[0] = tables[0] + self._embd[f"multi_embedder.embedders.{self.n_codebooks}.weight"][self.text_vocab].float()
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), torch.cat(tables, dim=0))
|
||||
yield from self._voices([LazyTorchTensor.to_eager(t) for t in tables])
|
||||
self._embd.clear()
|
||||
@@ -692,7 +692,7 @@ llama-openvino:server --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_
|
||||
# Or Using llama-server executable
|
||||
./build/ReleaseOV/bin/llama-server -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --port 8080 -c 1024
|
||||
|
||||
# Option 1: Open your browser to http://localhost:8080 to access the web UI for the llama.cpp server.
|
||||
# Option 1: Open your browser to http://localhost:8080 to open Llama Chat, the built-in chat app of the llama.cpp server.
|
||||
# Option 2: In a NEW terminal, test the server with curl
|
||||
|
||||
# If you are behind a proxy, make sure to set NO_PROXY to avoid proxy for localhost
|
||||
|
||||
@@ -11,6 +11,9 @@ Supported models:
|
||||
- [LFM3-Audio](#lfm3-audio)
|
||||
- [Chatterbox](#chatterbox)
|
||||
- [Drama 3](#drama-3)
|
||||
- [ZONOS3](#zonos3)
|
||||
- [Fish Audio S2 Pro](#fish-audio-s2-pro)
|
||||
- [YuE2](#yue2) (songs)
|
||||
|
||||
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
|
||||
|
||||
@@ -180,3 +183,126 @@ The routed experts of the checkpoint are FP8, they are dequantized at conversion
|
||||
python convert_hf_to_gguf.py path/to/drama-3 --outtype q8_0 --outfile drama3.gguf
|
||||
python convert_hf_to_gguf.py path/to/drama-3 --mmproj --outtype bf16 --outfile mmproj-drama3.gguf
|
||||
```
|
||||
|
||||
## ZONOS3
|
||||
|
||||
[Zyphra ZONOS3 Mini](https://huggingface.co/Zyphra/ZONOS3-Mini): a hybrid Gated DeltaNet / attention MoE backbone (800M, 100M active) predicts the 10 codebooks of each frame with an inter-codebook delay, and the ZAC codec decodes them to 44.1 kHz audio.
|
||||
|
||||
Available params:
|
||||
- `--tts-voice` picks one of the predefined voices of the release (`Savannah_en-American_F_1` by default); `llama-server` lists them in `GET /props`. A voice is the prompt prefix of its reference codes, the default speaker mode of the reference
|
||||
- The language is the one of the text; `--tts-lang` (`lang` in `llama-server`) only selects the text normalization
|
||||
- Numbers are spelled out by the `tts` chat template of the GGUF: cardinals, decimals, money, percents, clock times, common ordinal forms (`1st`, `1er`, `1º`), English and German years. A 4-digit number is read as a year only after a word like `in`, `since`, a month name, or `im Jahr`, `seit` in German (`in 1990` -> "in nineteen ninety", `1234 people` -> "one thousand two hundred thirty-four people"). Languages: `en`, `fr`, `de`, `es`, `it`, `pt`, `ja` and `zh` (or `cmn`), the region is ignored (`fr_fr`, `pt_br`); English when the language is empty, other languages are left as written. Set the language to the one of the text: the separators differ, e.g. `1.234` is a thousand in German
|
||||
- Note: the reference uses NeMo text normalization, which also covers units, dates, abbreviations and Korean; supply normalized text for them
|
||||
- `--tts-speaker-file` clones the voice of a clip (the first 60 s are used) with the speaker encoder of the release, the "embedding" speaker mode of the reference
|
||||
- `llama-server` also takes `speed` (relative to 15 UTF-8 bytes per second) and `loudness` (LUFS), see the `/tts` endpoint; they pick the speaking-rate and loudness condition buckets of the checkpoint, the defaults leave them unknown
|
||||
- Note: `loudness` has a strong effect with a cloned voice; a predefined voice mostly keeps the loudness of its reference clip
|
||||
- Note: `--top-k` and `--top-p` are not used, frames are sampled with the settings of the reference: temperature 0.75 (`--temp` changes it), top-k 80, top-p 0.69, repetition penalty 2.4 over the last 96 frames of codebooks 0 to 5
|
||||
- Each frame is 2940 samples (about 67 ms), generation stops at the end-of-audio code
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m zonos3-mini.gguf \
|
||||
-mm mmproj-zonos3-mini.gguf \
|
||||
-p "The old lighthouse keeper climbed the spiral stairs every evening." \
|
||||
--tts-voice Imogen_en-British_F_1 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
Convert the root of the release repository, not the `lm/` subdirectory. The mmproj holds the codebook heads, the predefined voices, the speaker encoder and the `keep2_plain` codec decoder, it is best kept in F32:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/ZONOS3-Mini --outtype q8_0 --outfile zonos3-mini.gguf
|
||||
python convert_hf_to_gguf.py path/to/ZONOS3-Mini --mmproj --outtype f32 --outfile mmproj-zonos3-mini.gguf
|
||||
```
|
||||
|
||||
## Fish Audio S2 Pro
|
||||
|
||||
[Fish Audio S2 Pro](https://huggingface.co/fishaudio/s2-pro): a Qwen3 backbone (4B) samples each frame's semantic token, a 4-layer fast AR predicts the 9 other codes, and the Fish DAC (same as Drama 3) decodes them to 44.1 kHz audio.
|
||||
|
||||
Available params:
|
||||
- Speaker tags `<|speaker:0|>`, `<|speaker:1|>`, ... in the text give each character its own voice, a text without a tag starts with `<|speaker:0|>`
|
||||
- Inline instructions in brackets control the delivery, for example `[laughing]`, `[whisper]` or `[pause]`
|
||||
- The language is the one of the text, `--tts-lang` is not used
|
||||
- Voice cloning: `--tts-speaker-file` with the reference clip, optionally `--tts-speaker-text` with its transcript (it is put in the prompt as the reference does). The clip is encoded by the codec encoder half of `codec.pth`, it is truncated to the first 30 s
|
||||
- Note: `--temp`, `--top-k` and `--top-p` are not used (only `--seed`), frames are sampled with the settings of the reference (semantic token: repetition penalty 1.1 over the last 16 frames, top-k 30, top-p 0.8, temperature 0.8, or top-p 0.9 and temperature 1.0 when the last 4 frames repeat a token; other codes: greedy)
|
||||
- Each frame is 2048 samples (about 46 ms), generation stops at the end-of-speech token
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m s2-pro.gguf \
|
||||
-mm mmproj-s2-pro.gguf \
|
||||
-p "<|speaker:0|>Did you remember to buy the bread? <|speaker:1|>Of course I did [laughing] it is on the kitchen table." \
|
||||
-n 1000 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Voice cloning (`--tts-speaker-text` can be left out):
|
||||
|
||||
```sh
|
||||
llama-tts -m s2-pro.gguf \
|
||||
-mm mmproj-s2-pro.gguf \
|
||||
--tts-speaker-file reference.wav \
|
||||
--tts-speaker-text "The words spoken in reference.wav." \
|
||||
-p "Hello, this is a cloned voice." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The mmproj holds the fast AR and `codec.pth` (the encoder half for voice cloning, the decoder half for the audio), it is best kept in BF16. The codec encoder is always stored in F16/F32, small errors in it change the codes picked by its VQ:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/s2-pro --outtype q8_0 --outfile s2-pro.gguf
|
||||
python convert_hf_to_gguf.py path/to/s2-pro --mmproj --outtype bf16 --outfile mmproj-s2-pro.gguf
|
||||
```
|
||||
|
||||
## YuE2
|
||||
|
||||
[m-a-p/YuE2-3B](https://huggingface.co/m-a-p/YuE2-3B) generates songs (vocals and accompaniment) from a style prompt and lyrics, as 48 kHz stereo. The backbone (a Qwen3 text model) writes an ABC score, then one semantic code per 40 ms frame; the NAR half of the checkpoint turns the codes into VAE latents by flow matching, attending to the keys and values of the backbone, and the [YuE2-Vae](https://huggingface.co/m-a-p/YuE2-Vae) decoder makes the audio.
|
||||
|
||||
The prompt is the text of the reference request:
|
||||
|
||||
```
|
||||
[Tags]
|
||||
City Pop, upbeat, danceable, groovy bass, electric guitar, synth
|
||||
[Lyrics]
|
||||
[Verse]
|
||||
...
|
||||
```
|
||||
|
||||
Available params:
|
||||
- `--tts-voice` picks the planning mode of the reference (`cot`): `full` (default, a chord-annotated score), `melody` (a melody-only score) or `off` (no score)
|
||||
- Cover mode: append the score to the prompt as `\n[ABC]\n<score>`, with `--tts-voice melody` or `full`; the model then goes straight to the codes
|
||||
- The generation stops at the limits of the reference (4096 score tokens, then 9000 codes, 6 minutes); `-n` sets a lower limit on the score tokens and the codes together
|
||||
- The context must keep the default size (24576 positions, the one of the reference): a smaller `-c` is rejected when the model is loaded
|
||||
- Note: `--top-k` and `--top-p` are not used, both phases are sampled with the settings of the reference (`yue2_generation_config.json`); `--temp` changes the temperature of the codes
|
||||
- Note: the backbone context must use an F16 KV cache (default): the NAR pass loads its keys and values
|
||||
- The codes are sampled with classifier-free guidance like the reference: scale 1.01 with `--tts-voice off`, off (1.0) otherwise. `llama-server` takes `cfg_scale` to change it, the reference suggests 1.2 for a stronger text guidance. The uncond prompt runs as a second sequence, so the context must also hold it (the instruction, the score and the codes once more)
|
||||
- The NAR pass solves the flow-matching ODE in 32 midpoint steps like the reference; `llama-server` takes `flow_steps` to change it. The NAR time is about proportional to it: 16 steps are about 35% faster for the NAR and the VAE together, with a small deviation from the reference (about 0.2 dB of spectral difference); it grows with fewer steps (about 0.4 dB at 8, 0.8 dB at 4)
|
||||
- The NAR pass runs on chunks of codes like the reference: one chunk for a song that fits in 24576 positions, so the audio arrives after the last code. When streaming, each 30 s of codes is a chunk, processed during the generation, so the audio arrives in 30 s steps; the chunks are independent (each sees the prompt, the score and its own codes), which may be audible at the boundaries
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m yue2-3b.gguf \
|
||||
-mm mmproj-yue2-3b.gguf \
|
||||
-f song.txt \
|
||||
--output song.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The VAE is a separate repository, download it into the `vae` subdirectory of the model:
|
||||
|
||||
```sh
|
||||
hf download m-a-p/YuE2-3B --local-dir YuE2-3B
|
||||
hf download m-a-p/YuE2-Vae --local-dir YuE2-3B/vae
|
||||
python convert_hf_to_gguf.py YuE2-3B --outfile yue2-3b.gguf
|
||||
python convert_hf_to_gguf.py YuE2-3B --mmproj --outfile mmproj-yue2-3b.gguf
|
||||
```
|
||||
|
||||
The mmproj holds the NAR transformer layers (loaded with libllama as a Qwen3 text model, in the `--outtype` type), and the latent projections and the VAE decoder, always in F32.
|
||||
|
||||
@@ -87,36 +87,32 @@ for id: llama_token in tokens {
|
||||
|
||||
print("\n")
|
||||
|
||||
var batch = llama_batch_init(max(Int32(tokens.count), Int32(n_parallel)), 0, 1)
|
||||
let batch = llama_batch_ext_init(context)
|
||||
defer {
|
||||
llama_batch_free(batch)
|
||||
llama_batch_ext_free(batch)
|
||||
}
|
||||
|
||||
// evaluate the initial prompt
|
||||
batch.n_tokens = Int32(tokens.count)
|
||||
|
||||
for (i, token) in tokens.enumerated() {
|
||||
batch.token[i] = token
|
||||
batch.pos[i] = Int32(i)
|
||||
batch.n_seq_id[i] = 1
|
||||
// batch.seq_id[i][0] = 0
|
||||
// TODO: is this the proper way to do this?
|
||||
if let seq_id = batch.seq_id[i] {
|
||||
seq_id[0] = 0
|
||||
let idx = llama_batch_ext_add_token(batch, 0, token)
|
||||
if idx < 0 {
|
||||
print("llama_batch_ext_add_token() failed: \(idx)")
|
||||
exit(1)
|
||||
}
|
||||
batch.logits[i] = 0
|
||||
var pos = llama_pos(i)
|
||||
llama_batch_ext_set_pos(batch, idx, &pos)
|
||||
}
|
||||
|
||||
// llama_decode will output logits only for the last token of the prompt
|
||||
batch.logits[Int(batch.n_tokens) - 1] = 1
|
||||
// llama_process will output logits only for the last token of the prompt
|
||||
llama_batch_ext_set_output_logits(batch, Int32(tokens.count) - 1, true)
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed")
|
||||
exit(1)
|
||||
}
|
||||
|
||||
for i in 1 ..< n_parallel {
|
||||
llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, batch.n_tokens)
|
||||
llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, Int32(tokens.count))
|
||||
}
|
||||
|
||||
if n_parallel > 1 {
|
||||
@@ -125,16 +121,17 @@ if n_parallel > 1 {
|
||||
|
||||
var streams: [String] = .init(repeating: "", count: n_parallel)
|
||||
var streamBuffers: [[CChar]] = .init(repeating: [], count: n_parallel)
|
||||
var i_batch = [Int32](repeating: batch.n_tokens - 1, count: n_parallel)
|
||||
var i_batch = [Int32](repeating: Int32(tokens.count) - 1, count: n_parallel)
|
||||
|
||||
var n_cur = batch.n_tokens
|
||||
var n_cur = Int32(tokens.count)
|
||||
var n_decode = 0
|
||||
|
||||
let t_main_start = ggml_time_us()
|
||||
|
||||
while n_cur <= n_len {
|
||||
// prepare the next batch
|
||||
batch.n_tokens = 0
|
||||
llama_batch_ext_clear(batch)
|
||||
var n_tokens: Int32 = 0
|
||||
|
||||
// sample the next token for each parallel sequence / stream
|
||||
for i in 0 ..< n_parallel {
|
||||
@@ -165,31 +162,32 @@ while n_cur <= n_len {
|
||||
streams[i] += nextStringPiece
|
||||
|
||||
// push this new token for next evaluation
|
||||
batch.token[Int(batch.n_tokens)] = new_token_id
|
||||
batch.pos[Int(batch.n_tokens)] = n_cur
|
||||
batch.n_seq_id[Int(batch.n_tokens)] = 1
|
||||
if let seq_id = batch.seq_id[Int(batch.n_tokens)] {
|
||||
seq_id[0] = Int32(i)
|
||||
let idx = llama_batch_ext_add_token(batch, Int32(i), new_token_id)
|
||||
if idx < 0 {
|
||||
print("llama_batch_ext_add_token() failed: \(idx)")
|
||||
exit(1)
|
||||
}
|
||||
batch.logits[Int(batch.n_tokens)] = 1
|
||||
var pos = n_cur
|
||||
llama_batch_ext_set_pos(batch, idx, &pos)
|
||||
llama_batch_ext_set_output_logits(batch, idx, true)
|
||||
|
||||
i_batch[i] = batch.n_tokens
|
||||
i_batch[i] = idx
|
||||
|
||||
batch.n_tokens += 1
|
||||
n_tokens += 1
|
||||
|
||||
n_decode += 1
|
||||
}
|
||||
|
||||
// all streams are finished
|
||||
if batch.n_tokens == 0 {
|
||||
if n_tokens == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
n_cur += 1
|
||||
|
||||
// evaluate the current batch with the transformer model
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed")
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,20 +5,16 @@ enum LlamaError: Error {
|
||||
case couldNotInitializeContext
|
||||
}
|
||||
|
||||
func llama_batch_clear(_ batch: inout llama_batch) {
|
||||
batch.n_tokens = 0
|
||||
}
|
||||
|
||||
func llama_batch_add(_ batch: inout llama_batch, _ id: llama_token, _ pos: llama_pos, _ seq_ids: [llama_seq_id], _ logits: Bool) {
|
||||
batch.token [Int(batch.n_tokens)] = id
|
||||
batch.pos [Int(batch.n_tokens)] = pos
|
||||
batch.n_seq_id[Int(batch.n_tokens)] = Int32(seq_ids.count)
|
||||
for i in 0..<seq_ids.count {
|
||||
batch.seq_id[Int(batch.n_tokens)]![Int(i)] = seq_ids[i]
|
||||
@discardableResult
|
||||
func llama_batch_add(_ batch: OpaquePointer, _ id: llama_token, _ pos: llama_pos, _ seq_id: llama_seq_id, _ logits: Bool) -> Int32 {
|
||||
let idx = llama_batch_ext_add_token(batch, seq_id, id)
|
||||
if idx < 0 {
|
||||
return idx
|
||||
}
|
||||
batch.logits [Int(batch.n_tokens)] = logits ? 1 : 0
|
||||
|
||||
batch.n_tokens += 1
|
||||
var pos = pos
|
||||
llama_batch_ext_set_pos(batch, idx, &pos)
|
||||
llama_batch_ext_set_output_logits(batch, idx, logits)
|
||||
return idx
|
||||
}
|
||||
|
||||
actor LlamaContext {
|
||||
@@ -26,7 +22,7 @@ actor LlamaContext {
|
||||
private var context: OpaquePointer
|
||||
private var vocab: OpaquePointer
|
||||
private var sampling: UnsafeMutablePointer<llama_sampler>
|
||||
private var batch: llama_batch
|
||||
private var batch: OpaquePointer
|
||||
private var tokens_list: [llama_token]
|
||||
var is_done: Bool = false
|
||||
|
||||
@@ -42,7 +38,7 @@ actor LlamaContext {
|
||||
self.model = model
|
||||
self.context = context
|
||||
self.tokens_list = []
|
||||
self.batch = llama_batch_init(512, 0, 1)
|
||||
self.batch = llama_batch_ext_init(context)
|
||||
self.temporary_invalid_cchars = []
|
||||
let sparams = llama_sampler_chain_default_params()
|
||||
self.sampling = llama_sampler_chain_init(sparams)
|
||||
@@ -53,7 +49,7 @@ actor LlamaContext {
|
||||
|
||||
deinit {
|
||||
llama_sampler_free(sampling)
|
||||
llama_batch_free(batch)
|
||||
llama_batch_ext_free(batch)
|
||||
llama_model_free(model)
|
||||
llama_free(context)
|
||||
llama_backend_free()
|
||||
@@ -110,10 +106,6 @@ actor LlamaContext {
|
||||
return SwiftString
|
||||
}
|
||||
|
||||
func get_n_tokens() -> Int32 {
|
||||
return batch.n_tokens;
|
||||
}
|
||||
|
||||
func completion_init(text: String) {
|
||||
print("attempting to complete \"\(text)\"")
|
||||
|
||||
@@ -133,25 +125,27 @@ actor LlamaContext {
|
||||
print(String(cString: token_to_piece(token: id) + [0]))
|
||||
}
|
||||
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_ext_clear(batch)
|
||||
|
||||
for i1 in 0..<tokens_list.count {
|
||||
let i = Int(i1)
|
||||
llama_batch_add(&batch, tokens_list[i], Int32(i), [0], false)
|
||||
}
|
||||
batch.logits[Int(batch.n_tokens) - 1] = 1 // true
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed")
|
||||
let idx = llama_batch_add(batch, tokens_list[i], Int32(i), 0, i == tokens_list.count - 1)
|
||||
if idx < 0 {
|
||||
print("llama_batch_add() failed: \(idx)")
|
||||
}
|
||||
}
|
||||
|
||||
n_cur = batch.n_tokens
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed")
|
||||
}
|
||||
|
||||
n_cur = Int32(tokens_list.count)
|
||||
}
|
||||
|
||||
func completion_loop() -> String {
|
||||
var new_token_id: llama_token = 0
|
||||
|
||||
new_token_id = llama_sampler_sample(sampling, context, batch.n_tokens - 1)
|
||||
new_token_id = llama_sampler_sample(sampling, context, -1)
|
||||
|
||||
if llama_vocab_is_eog(vocab, new_token_id) || n_cur == n_len {
|
||||
print("\n")
|
||||
@@ -178,13 +172,13 @@ actor LlamaContext {
|
||||
print(new_token_str)
|
||||
// tokens_list.append(new_token_id)
|
||||
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_add(&batch, new_token_id, n_cur, [0], true)
|
||||
llama_batch_ext_clear(batch)
|
||||
llama_batch_add(batch, new_token_id, n_cur, 0, true)
|
||||
|
||||
n_decode += 1
|
||||
n_cur += 1
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("failed to evaluate llama!")
|
||||
}
|
||||
|
||||
@@ -201,21 +195,20 @@ actor LlamaContext {
|
||||
for _ in 0..<nr {
|
||||
// bench prompt processing
|
||||
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_ext_clear(batch)
|
||||
|
||||
let n_tokens = pp
|
||||
|
||||
for i in 0..<n_tokens {
|
||||
llama_batch_add(&batch, 0, Int32(i), [0], false)
|
||||
llama_batch_add(batch, 0, Int32(i), 0, i == n_tokens - 1)
|
||||
}
|
||||
batch.logits[Int(batch.n_tokens) - 1] = 1 // true
|
||||
|
||||
llama_memory_clear(llama_get_memory(context), false)
|
||||
|
||||
let t_pp_start = DispatchTime.now().uptimeNanoseconds / 1000;
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed during prompt")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed during prompt")
|
||||
}
|
||||
llama_synchronize(context)
|
||||
|
||||
@@ -228,14 +221,14 @@ actor LlamaContext {
|
||||
let t_tg_start = DispatchTime.now().uptimeNanoseconds / 1000;
|
||||
|
||||
for i in 0..<tg {
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_ext_clear(batch)
|
||||
|
||||
for j in 0..<pl {
|
||||
llama_batch_add(&batch, 0, Int32(i), [Int32(j)], true)
|
||||
llama_batch_add(batch, 0, Int32(i), Int32(j), true)
|
||||
}
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed during text generation")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed during text generation")
|
||||
}
|
||||
llama_synchronize(context)
|
||||
}
|
||||
|
||||
@@ -9,11 +9,16 @@ constexpr int D = 128;
|
||||
using bf16 = nv_bfloat16;
|
||||
|
||||
template <int R, int C> struct matrix {
|
||||
bf16 hi[R * C], lo[R * C];
|
||||
#ifdef GGML_USE_HIP
|
||||
static constexpr int alloc = C + 4;
|
||||
#else
|
||||
static constexpr int alloc = C;
|
||||
#endif // GGML_USE_HIP
|
||||
bf16 hi[R * alloc], lo[R * alloc];
|
||||
|
||||
static __device__ __forceinline__ int index(int r, int c) {
|
||||
#ifdef GGML_USE_HIP
|
||||
return r * C + c;
|
||||
return r * alloc + c;
|
||||
#else
|
||||
return 2 * ggml_cuda_mma::swizzle<C / 2, nv_bfloat162>(r * (C / 2) + c / 2, r) + c % 2;
|
||||
#endif // GGML_USE_HIP
|
||||
@@ -56,11 +61,11 @@ template <int C, int V> struct alignas(128) shared {
|
||||
static __device__ __forceinline__ int lower_index(int r, int c) { return r * C + c; }
|
||||
};
|
||||
|
||||
#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
|
||||
#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
|
||||
using namespace ggml_cuda_mma;
|
||||
#ifdef GGML_USE_HIP
|
||||
using acc = tile<16, 16, float, DATA_LAYOUT_J_MAJOR>;
|
||||
using a_tile = tile<16, 8, nv_bfloat162, DATA_LAYOUT_I_MAJOR_MIRRORED>;
|
||||
using a_tile = tile<16, 8, nv_bfloat162, get_input_data_layout()>;
|
||||
using b_tile = a_tile;
|
||||
#else
|
||||
using acc = tile<16, 8, float>;
|
||||
@@ -73,6 +78,23 @@ __device__ __forceinline__ tile_t load(const matrix<R, C> & m, int row, int col,
|
||||
tile_t t;
|
||||
const bf16 * data = low ? m.lo : m.hi;
|
||||
#ifdef GGML_USE_HIP
|
||||
#ifdef AMD_MFMA_AVAILABLE
|
||||
if constexpr (!TRANS) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < tile_t::ne; ++i) {
|
||||
const int r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i);
|
||||
ggml_cuda_memcpy_1<4>(t.x + i, data + m.index(r, c));
|
||||
}
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < tile_t::ne; ++i) {
|
||||
const int r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i);
|
||||
const bf16 x = data[m.index(c, r)];
|
||||
const bf16 y = data[m.index(c + 1, r)];
|
||||
t.x[i] = __float22bfloat162_rn(make_float2(__bfloat162float(x), __bfloat162float(y)));
|
||||
}
|
||||
}
|
||||
#else
|
||||
#pragma unroll
|
||||
for (int i = 0; i < tile_t::ne; ++i) {
|
||||
const int r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i);
|
||||
@@ -80,6 +102,7 @@ __device__ __forceinline__ tile_t load(const matrix<R, C> & m, int row, int col,
|
||||
const bf16 y = data[TRANS ? m.index(c + 1, r) : m.index(r, c + 1)];
|
||||
t.x[i] = __float22bfloat162_rn(make_float2(__bfloat162float(x), __bfloat162float(y)));
|
||||
}
|
||||
#endif // AMD_MFMA_AVAILABLE
|
||||
#else
|
||||
const auto * packed = reinterpret_cast<const nv_bfloat162 *>(data);
|
||||
if constexpr (TRANS) {
|
||||
@@ -118,15 +141,17 @@ template <int R, int C> __device__ __forceinline__ void store(matrix<R, C> & m,
|
||||
}
|
||||
#endif // GGML_USE_HIP
|
||||
}
|
||||
#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
|
||||
#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
|
||||
|
||||
// Each block owns V independent value columns and advances by C tokens.
|
||||
// Keep recurrent state in FP32; pack high/residual BF16 operands inside the block.
|
||||
template <int C, int V, int BLOCKS = 1>
|
||||
static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_mma_args a) {
|
||||
#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
|
||||
static_assert(ggml_cuda_get_physical_warp_size() == 32, "GDN MMA requires 32-lane warps");
|
||||
constexpr int WARPS = 8;
|
||||
#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
static_assert(256 % warp_size == 0, "block must span a whole number of waves");
|
||||
constexpr int warps = 256 / warp_size;
|
||||
constexpr int input_rows = warp_size / 32; // input rows a wave stages per step
|
||||
#ifdef GGML_USE_HIP
|
||||
constexpr int N = 16;
|
||||
#else
|
||||
@@ -134,34 +159,37 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
#endif // GGML_USE_HIP
|
||||
static_assert(C >= 16 && (C & (C - 1)) == 0 && V % N == 0 && D % V == 0, "invalid GDN tile");
|
||||
extern __shared__ __align__(128) unsigned char bytes[];
|
||||
auto & s = *reinterpret_cast<shared<C, V> *>(bytes);
|
||||
const int lane = threadIdx.x, warp = threadIdx.y, tid = warp * 32 + lane;
|
||||
const int slice = blockIdx.x % (D / V);
|
||||
const int h = (blockIdx.x / (D / V)) % a.H;
|
||||
const int seq = blockIdx.x / ((D / V) * a.H);
|
||||
const int v0 = slice * V;
|
||||
const int64_t soff = ((int64_t) seq * a.H + h) * D * D;
|
||||
const int64_t qoff = (seq / a.rq3) * a.sq3 + (h % a.H_k) * a.sq1;
|
||||
const int64_t voff = seq * a.sv3 + h * a.sv1 + v0;
|
||||
const int64_t goff = seq * a.sb3 + h * a.sb1;
|
||||
float * out = a.dst + ((int64_t) seq * a.n_tokens * a.H + h) * D + v0;
|
||||
constexpr int STATE_TILES = (D / 16) * (V / N);
|
||||
constexpr int PER_WARP = STATE_TILES / WARPS;
|
||||
static_assert(STATE_TILES % WARPS == 0, "state must divide across warps");
|
||||
auto & s = *reinterpret_cast<shared<C, V> *>(bytes);
|
||||
const int lane = threadIdx.x;
|
||||
const int warp = threadIdx.y;
|
||||
const int tid = warp * warp_size + lane;
|
||||
const int sub = warp_size > 32 ? lane / 32 : 0;
|
||||
const int slane = warp_size > 32 ? lane % 32 : lane;
|
||||
const int slice = blockIdx.x % (D / V);
|
||||
const int h = (blockIdx.x / (D / V)) % a.H;
|
||||
const int seq = blockIdx.x / ((D / V) * a.H);
|
||||
const int v0 = slice * V;
|
||||
const int64_t soff = ((int64_t) seq * a.H + h) * D * D;
|
||||
const int64_t qoff = (seq / a.rq3) * a.sq3 + (h % a.H_k) * a.sq1;
|
||||
const int64_t voff = seq * a.sv3 + h * a.sv1 + v0;
|
||||
const int64_t goff = seq * a.sb3 + h * a.sb1;
|
||||
float * out = a.dst + ((int64_t) seq * a.n_tokens * a.H + h) * D + v0;
|
||||
constexpr int state_tiles = (D / 16) * (V / N);
|
||||
constexpr int PER_WARP = state_tiles / warps;
|
||||
static_assert(state_tiles % warps == 0, "state must divide across warps");
|
||||
acc state[PER_WARP];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < PER_WARP; ++j) {
|
||||
const int tile_index = warp + j * WARPS, r = (tile_index / (V / N)) * 16, c = (tile_index % (V / N)) * N;
|
||||
const int tile_index = warp + j * warps, r = (tile_index / (V / N)) * 16, c = (tile_index % (V / N)) * N;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
state[j].x[i] = a.state[soff + (v0 + c + acc::get_j(i)) * D + r + acc::get_i(i)];
|
||||
}
|
||||
}
|
||||
auto read_qk = [&](int64_t t, float4 & q, float4 & k) {
|
||||
t += warp;
|
||||
q = k = make_float4(0, 0, 0, 0);
|
||||
if (t < a.n_tokens) {
|
||||
const int64_t off = qoff + t * a.sq2 + 4 * lane;
|
||||
const int64_t off = qoff + t * a.sq2 + 4 * slane;
|
||||
if ((((uintptr_t) (a.q + off) | (uintptr_t) (a.k + off)) & 15) == 0) {
|
||||
q = *reinterpret_cast<const float4 *>(a.q + off);
|
||||
k = *reinterpret_cast<const float4 *>(a.k + off);
|
||||
@@ -172,22 +200,26 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
}
|
||||
};
|
||||
// Carry the first eight input rows across chunks to hide global-load latency.
|
||||
constexpr int QK_ROWS = warps * input_rows;
|
||||
constexpr bool PREFETCH = C == 16 && V == 32;
|
||||
float4 next_q, next_k;
|
||||
float4 next_q;
|
||||
float4 next_k;
|
||||
if constexpr (PREFETCH) {
|
||||
read_qk(0, next_q, next_k);
|
||||
read_qk(warp * input_rows + sub, next_q, next_k);
|
||||
}
|
||||
for (int64_t t0 = 0; t0 < a.n_tokens; t0 += C) {
|
||||
const int valid = min((int64_t) C, a.n_tokens - t0);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < C / 8; ++j) {
|
||||
const int t = warp + 8 * j, d = 4 * lane;
|
||||
float4 q, k;
|
||||
for (int j = 0; j < C / QK_ROWS; ++j) {
|
||||
const int t = warp * input_rows + sub + QK_ROWS * j;
|
||||
const int d = 4 * slane;
|
||||
float4 q;
|
||||
float4 k;
|
||||
if (PREFETCH && j == 0) {
|
||||
q = next_q;
|
||||
k = next_k;
|
||||
} else {
|
||||
read_qk(t0 + 8 * j, q, k);
|
||||
read_qk(t0 + t, q, k);
|
||||
}
|
||||
s.q.store2(t, d, make_float2(q.x * a.scale, q.y * a.scale));
|
||||
s.q.store2(t, d + 2, make_float2(q.z * a.scale, q.w * a.scale));
|
||||
@@ -197,13 +229,13 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
// Load gates in parallel before the FP32 prefix scan.
|
||||
if (warp == 0) {
|
||||
float carry = 0.f;
|
||||
for (int base = 0; base < C; base += 32) {
|
||||
for (int base = 0; base < C; base += warp_size) {
|
||||
const int t = base + lane;
|
||||
float g = t < valid ? a.g[goff + (t0 + t) * a.sb2] : 0.f;
|
||||
const float beta = t < valid ? a.beta[goff + (t0 + t) * a.sb2] : 0.f;
|
||||
#pragma unroll
|
||||
for (int offset = 1; offset < 32; offset *= 2) {
|
||||
const float previous = ggml_cuda_shfl_up(g, offset, 32);
|
||||
for (int offset = 1; offset < warp_size; offset *= 2) {
|
||||
const float previous = ggml_cuda_shfl_up(g, offset, warp_size);
|
||||
if (lane >= offset) {
|
||||
g += previous;
|
||||
}
|
||||
@@ -213,23 +245,23 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
s.prefix[t] = g;
|
||||
s.beta[t] = beta;
|
||||
}
|
||||
carry = ggml_cuda_shfl(g, 31, 32);
|
||||
carry = ggml_cuda_shfl(g, warp_size - 1, warp_size);
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < PER_WARP; ++j) {
|
||||
const int ti = warp + j * WARPS;
|
||||
const int ti = warp + j * warps;
|
||||
store(s.scratch.state, state[j], (ti / (V / N)) * 16, (ti % (V / N)) * N);
|
||||
}
|
||||
__syncthreads();
|
||||
if constexpr (PREFETCH) {
|
||||
if (t0 + C < a.n_tokens) {
|
||||
read_qk(t0 + C, next_q, next_k);
|
||||
read_qk(t0 + C + warp * input_rows + sub, next_q, next_k);
|
||||
}
|
||||
}
|
||||
// V32 overlaps Gram work with one output tile per projection warp.
|
||||
constexpr bool SPLIT_PROJECTION = C == 16 && V == 32;
|
||||
acc base_output[((C / 16) * (V / N) + WARPS - 1) / WARPS];
|
||||
acc base_output[((C / 16) * (V / N) + warps - 1) / warps];
|
||||
// delta = beta * (v - decay * K S), base output = decay * Q S
|
||||
auto project = [&]() {
|
||||
if constexpr (V == 32) {
|
||||
@@ -237,23 +269,26 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
return;
|
||||
}
|
||||
}
|
||||
for (int ti = (V == 32 ? warp - 4 : warp); ti < (C / 16) * (V / N); ti += (V == 32 ? 4 : WARPS)) {
|
||||
const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
acc qs, ks;
|
||||
for (int ti = (V == 32 ? warp - 4 : warp); ti < (C / 16) * (V / N); ti += (V == 32 ? 4 : warps)) {
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
acc qs;
|
||||
acc ks;
|
||||
product<D, false, true>(qs, s.q, s.scratch.state, r, c);
|
||||
product<D, false, true>(ks, s.k, s.scratch.state, r, c);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
const int t = r + acc::get_i(i), v = c + acc::get_j(i);
|
||||
const int t = r + acc::get_i(i);
|
||||
const int v = c + acc::get_j(i);
|
||||
const float decay = expf(s.prefix[t]);
|
||||
const float vv = t < valid ? a.v[voff + (t0 + t) * a.sv2 + v] : 0.f;
|
||||
s.delta.store(t, v, s.beta[t] * (vv - decay * ks.x[i]));
|
||||
base_output[ti / WARPS].x[i] = decay * qs.x[i];
|
||||
base_output[ti / warps].x[i] = decay * qs.x[i];
|
||||
}
|
||||
}
|
||||
};
|
||||
// A = beta * strict_tril(K K^T * decay), P = tril(Q K^T * decay)
|
||||
for (int ti = warp; ti < 2 * (C / 16) * (C / N) && (V != 32 || warp < 4); ti += (V == 32 ? 4 : WARPS)) {
|
||||
for (int ti = warp; ti < 2 * (C / 16) * (C / N) && (V != 32 || warp < 4); ti += (V == 32 ? 4 : warps)) {
|
||||
const bool gram = ti < (C / 16) * (C / N);
|
||||
const int t = ti % ((C / 16) * (C / N)), r = (t / (C / N)) * 16, c = (t % (C / N)) * N;
|
||||
acc x;
|
||||
@@ -280,27 +315,27 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
}
|
||||
__syncthreads();
|
||||
// inverse = (I + A)^-1 by forward substitution
|
||||
for (int row = warp; row < C; row += WARPS) {
|
||||
float x[(C + 31) / 32];
|
||||
for (int row = warp; row < C; row += warps) {
|
||||
float x[(C + warp_size - 1) / warp_size];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < (C + 31) / 32; ++j) {
|
||||
const int col = lane + 32 * j;
|
||||
x[j] = col == row ? 1.f : (col < row ? -s.lower[s.lower_index(row, col)] : 0.f);
|
||||
for (int j = 0; j < (C + warp_size - 1) / warp_size; ++j) {
|
||||
const int col = lane + warp_size * j;
|
||||
x[j] = col == row ? 1.f : (col < row ? -s.lower[s.lower_index(row, col)] : 0.f);
|
||||
}
|
||||
for (int pivot = row - 1; pivot > 0; --pivot) {
|
||||
const float xp = ggml_cuda_shfl(x[pivot / 32], pivot % 32, 32);
|
||||
const float xp = ggml_cuda_shfl(x[pivot / warp_size], pivot % warp_size, warp_size);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < (C + 31) / 32; ++j) {
|
||||
const int col = lane + 32 * j;
|
||||
for (int j = 0; j < (C + warp_size - 1) / warp_size; ++j) {
|
||||
const int col = lane + warp_size * j;
|
||||
if (col < pivot) {
|
||||
x[j] -= xp * s.lower[s.lower_index(pivot, col)];
|
||||
}
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < (C + 31) / 32; ++j) {
|
||||
if (lane + 32 * j < C) {
|
||||
s.inverse.store(row, lane + 32 * j, x[j]);
|
||||
for (int j = 0; j < (C + warp_size - 1) / warp_size; ++j) {
|
||||
if (lane + warp_size * j < C) {
|
||||
s.inverse.store(row, lane + warp_size * j, x[j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -309,8 +344,9 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
}
|
||||
__syncthreads();
|
||||
// solved = inverse * delta
|
||||
for (int ti = warp; ti < (C / 16) * (V / N); ti += WARPS) {
|
||||
const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
for (int ti = warp; ti < (C / 16) * (V / N); ti += warps) {
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
acc x;
|
||||
product<C, false, true>(x, s.inverse, s.delta, r, c);
|
||||
store(s.scratch.solved, x, r, c);
|
||||
@@ -318,21 +354,25 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
__syncthreads();
|
||||
// out = base output + P * solved
|
||||
for (int ti = (SPLIT_PROJECTION ? warp - 4 : warp); ti < (C / 16) * (V / N) && (!SPLIT_PROJECTION || warp >= 4);
|
||||
ti += WARPS) {
|
||||
const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
ti += warps) {
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
acc x;
|
||||
product<C, false, true>(x, s.p, s.scratch.solved, r, c);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
const int t = r + acc::get_i(i), v = c + acc::get_j(i);
|
||||
const int t = r + acc::get_i(i);
|
||||
const int v = c + acc::get_j(i);
|
||||
if (t < valid) {
|
||||
out[(t0 + t) * a.H * D + v] = base_output[ti / WARPS].x[i] + x.x[i];
|
||||
out[(t0 + t) * a.H * D + v] = base_output[ti / warps].x[i] + x.x[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
// S = decay(chunk) * S + K^T * (solved * decay to chunk end)
|
||||
for (int i = tid; i < C * V; i += 256) {
|
||||
const int t = i / V, v = i % V, j = s.scratch.solved.index(t, v);
|
||||
for (int i = tid; i < C * V; i += warps * warp_size) {
|
||||
const int t = i / V;
|
||||
const int v = i % V;
|
||||
const int j = s.scratch.solved.index(t, v);
|
||||
const float x = __bfloat162float(s.scratch.solved.hi[j]) + __bfloat162float(s.scratch.solved.lo[j]);
|
||||
s.delta.store(t, v, x * expf(s.prefix[valid - 1] - s.prefix[t]));
|
||||
}
|
||||
@@ -340,7 +380,9 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
const float decay = expf(s.prefix[valid - 1]);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < PER_WARP; ++j) {
|
||||
const int ti = warp + j * WARPS, r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
const int ti = warp + j * warps;
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
state[j].x[i] *= decay;
|
||||
@@ -351,7 +393,9 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < PER_WARP; ++j) {
|
||||
const int ti = warp + j * WARPS, r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
const int ti = warp + j * warps;
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
a.state_out[soff + (v0 + c + acc::get_j(i)) * D + r + acc::get_i(i)] = state[j].x[i];
|
||||
@@ -360,10 +404,11 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
#else
|
||||
GGML_UNUSED_VARS(a);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
|
||||
#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
|
||||
}
|
||||
|
||||
template <int C, int V, int BLOCKS = 1> bool init(int device) {
|
||||
template <int C, int V, int BLOCKS = 1>
|
||||
static bool init(int device) {
|
||||
static std::once_flag once[GGML_CUDA_MAX_DEVICES];
|
||||
static bool available[GGML_CUDA_MAX_DEVICES] = {};
|
||||
std::call_once(once[device], [device] {
|
||||
@@ -381,7 +426,13 @@ template <int C, int V, int BLOCKS = 1> bool init(int device) {
|
||||
return available[device];
|
||||
}
|
||||
|
||||
enum class gdn_path { ar, slice32_one_block, slice32_two_blocks, slice64, whole_head };
|
||||
enum class gdn_path {
|
||||
ar,
|
||||
slice32_one_block,
|
||||
slice32_two_blocks,
|
||||
slice64,
|
||||
whole_head
|
||||
};
|
||||
|
||||
static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a) {
|
||||
GGML_ASSERT(device >= 0 && device < GGML_CUDA_MAX_DEVICES);
|
||||
@@ -391,10 +442,25 @@ static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a
|
||||
}
|
||||
const auto & info = ggml_cuda_info().devices[device];
|
||||
#ifdef GGML_USE_HIP
|
||||
if (!GGML_CUDA_CC_IS_RDNA3_5(info.cc) || a.n_tokens < 2048) {
|
||||
return gdn_path::ar;
|
||||
if (GGML_CUDA_CC_IS_RDNA3_5(info.cc)) {
|
||||
if (a.n_tokens < 2048) {
|
||||
return gdn_path::ar;
|
||||
}
|
||||
return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar;
|
||||
}
|
||||
return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar;
|
||||
if (GGML_CUDA_CC_IS_CDNA(info.cc)) {
|
||||
if (!amd_mfma_available(info.cc)) {
|
||||
return gdn_path::ar;
|
||||
}
|
||||
const int64_t blocks = a.H * a.n_seqs * 2;
|
||||
const int64_t blocks_nwaves = (blocks + info.nsm - 1) / info.nsm;
|
||||
const int64_t blocks_efficiency = 100 * blocks / (info.nsm * blocks_nwaves);
|
||||
if (blocks_nwaves > 1 && blocks_efficiency < 60) {
|
||||
return gdn_path::ar;
|
||||
}
|
||||
return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar;
|
||||
}
|
||||
return gdn_path::ar;
|
||||
#elif defined(GGML_USE_MUSA)
|
||||
GGML_UNUSED_VARS(info);
|
||||
return gdn_path::ar;
|
||||
@@ -428,7 +494,8 @@ bool ggml_cuda_gdn_mma_launch(int device, const ggml_cuda_gdn_mma_args & a, cuda
|
||||
return false;
|
||||
}
|
||||
#ifdef GGML_USE_HIP
|
||||
gdn_single<16, 64><<<a.H * a.n_seqs * 2, dim3(32, 8), sizeof(shared<16, 64>), stream>>>(a);
|
||||
const int warp_size = ggml_cuda_info().devices[device].warp_size;
|
||||
gdn_single<16, 64><<<a.H * a.n_seqs * 2, dim3(warp_size, 256 / warp_size), sizeof(shared<16, 64>), stream>>>(a);
|
||||
#else
|
||||
if (path == gdn_path::slice32_two_blocks) {
|
||||
gdn_single<16, 32, 2><<<a.H * a.n_seqs * 4, dim3(32, 8), sizeof(shared<16, 32>), stream>>>(a);
|
||||
|
||||
@@ -2526,7 +2526,7 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c
|
||||
} else if (src0->type == GGML_TYPE_F8_E4M3 &&
|
||||
ggml_cuda_should_use_mmvf(src0, cc, warp_size, dst->ne[2])) {
|
||||
return false;
|
||||
} else if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) {
|
||||
} else if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc) && src0->ne[0] % 2 == 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -2570,7 +2570,8 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
|
||||
ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst);
|
||||
return;
|
||||
} else {
|
||||
if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) {
|
||||
// MMVF reads src0 in pairs of values, so it needs an even number of columns
|
||||
if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc) && ne00 % 2 == 0) {
|
||||
ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1079,8 +1079,14 @@ static __global__ void mul_mat_q(
|
||||
const uint2 tmp2 = fast_div_modulo(blockIdx.z, nchannels_y);
|
||||
const int wt = tmp2.x;
|
||||
int zt = tmp2.y;
|
||||
#if defined(GGML_USE_HIP)
|
||||
// Row tiles innermost for MoE, see launch_mul_mat_q.
|
||||
const int jt = ids_dst ? blockIdx.y : blockIdx.x;
|
||||
const int it = ids_dst ? blockIdx.x : blockIdx.y;
|
||||
#else
|
||||
const int jt = blockIdx.x;
|
||||
const int it = blockIdx.y;
|
||||
#endif // defined(GGML_USE_HIP)
|
||||
|
||||
// Defaults for regular matrix multiplication:
|
||||
int col_low = jt*J;
|
||||
@@ -1483,8 +1489,15 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
|
||||
const int I_dst = has_fusion ? config.I/2 : config.I; // Rows of dst per tile, see mul_mat_q.
|
||||
const int nti = (args.nrows_x + I_dst - 1) / I_dst;
|
||||
const int ntzw = args.nchannels_y * args.nsamples_y;
|
||||
// Column tiles innermost so that the concurrently running blocks share the same tile of x and read it from L2.
|
||||
const dim3 block_nums_xy_tiling(args.ntj, nti, ntzw);
|
||||
#if defined(GGML_USE_HIP)
|
||||
// For MoE the column tiles are tied to experts, so the column-innermost order does not improve the reuse of x in L2.
|
||||
// It also groups blocks with very different runtimes which schedules poorly on AMD GPUs, see #31599.
|
||||
// Therefore the row tiles are innermost for MoE. The column-innermost order is kept for dense multiplication.
|
||||
const bool rows_innermost = args.ids_dst != nullptr;
|
||||
#else
|
||||
constexpr bool rows_innermost = false;
|
||||
#endif // defined(GGML_USE_HIP)
|
||||
const dim3 block_nums_xy_tiling = rows_innermost ? dim3(nti, args.ntj, ntzw) : dim3(args.ntj, nti, ntzw);
|
||||
|
||||
GGML_ASSERT(args.ids_dst || args.nchannels_y % args.nchannels_x == 0);
|
||||
GGML_ASSERT( args.nsamples_y % args.nsamples_x == 0);
|
||||
@@ -1499,7 +1512,7 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
|
||||
const uint3 sample_ratio_fd = init_fastdiv_values(sample_ratio);
|
||||
|
||||
if (has_fusion || !config.stream_k) {
|
||||
GGML_ASSERT(nti <= 65535); // gridDim.y limit
|
||||
GGML_ASSERT((rows_innermost ? args.ntj : nti) <= 65535); // gridDim.y limit
|
||||
mul_mat_q<type, J, fallback, prec_src1, has_fusion><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
|
||||
(args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.tile_spec, args.dst, nullptr,
|
||||
args.y_scale, args.fusion.glu_op, args.fusion.glu_limit, (const float *) args.fusion.x_scale, (const float *) args.fusion.gate_scale,
|
||||
|
||||
@@ -378,6 +378,14 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
|
||||
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
||||
}
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_RDNA3_5(cc)) {
|
||||
switch (type) { // tuned on Radeon 8060S (gfx1151)
|
||||
case GGML_TYPE_Q8_0:
|
||||
return ne11 <= 6;
|
||||
default:
|
||||
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
||||
}
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_CDNA(cc)) {
|
||||
if (GGML_CUDA_CC_IS_CDNA1(cc)) {
|
||||
switch (type) {
|
||||
|
||||
Vendored
+2
-2
@@ -240,9 +240,9 @@ typedef uint64_t ggml_cuda_thread_mask_t;
|
||||
#define RDNA2
|
||||
#endif
|
||||
|
||||
#if defined(__gfx1010__) || defined(__gfx1012__)
|
||||
#if defined(__gfx1010__) || defined(__gfx1012__) || defined(__gfx1013__)
|
||||
#define RDNA1
|
||||
#endif // defined(__gfx1010__) || defined(__gfx1012__)
|
||||
#endif // defined(__gfx1010__) || defined(__gfx1012__) || defined(__gfx1013__)
|
||||
|
||||
#if defined(RDNA4) || defined(RDNA4_M) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)
|
||||
#define RDNA // For the entire family
|
||||
|
||||
@@ -712,6 +712,9 @@ struct ggml_backend_opencl_context {
|
||||
size_t image_max_buffer_size;
|
||||
size_t image2d_max_width;
|
||||
size_t image2d_max_height;
|
||||
size_t image3d_max_width;
|
||||
size_t image3d_max_height;
|
||||
size_t image3d_max_depth;
|
||||
|
||||
cl_device_svm_capabilities svm_caps;
|
||||
|
||||
@@ -1431,6 +1434,7 @@ struct ggml_backend_opencl_context {
|
||||
cl_kernel kernel_gemv_noshuffle_q4_k_f32_mc3; // multi-column (N=3) verify GEMV
|
||||
cl_kernel kernel_gemm_noshuffle_q4_k_f32;
|
||||
cl_kernel kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin;
|
||||
cl_kernel kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin;
|
||||
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin;
|
||||
cl_kernel kernel_gemv_noshuffle_q4_k_f32_32b_trans;
|
||||
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM
|
||||
@@ -6701,6 +6705,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
|
||||
backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = nullptr;
|
||||
backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = nullptr;
|
||||
backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin = nullptr;
|
||||
backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = nullptr;
|
||||
{
|
||||
if (backend_ctx->has_vector_subgroup_broadcast) {
|
||||
@@ -6745,6 +6750,17 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
CL_CHECK(clReleaseProgram(bin_prog));
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_f32_32b_trans_ila_a7", &bin_size);
|
||||
if (kernel_bin && bin_size > 0) {
|
||||
cl_program bin_prog =
|
||||
build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size);
|
||||
|
||||
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin =
|
||||
clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7", &err), err));
|
||||
CL_CHECK(clReleaseProgram(bin_prog));
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9258,6 +9274,9 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE_MAX_BUFFER_SIZE, sizeof(size_t), &backend_ctx->image_max_buffer_size, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_WIDTH, sizeof(size_t), &backend_ctx->image2d_max_width, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_HEIGHT, sizeof(size_t), &backend_ctx->image2d_max_height, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE3D_MAX_WIDTH, sizeof(size_t), &backend_ctx->image3d_max_width, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE3D_MAX_HEIGHT, sizeof(size_t), &backend_ctx->image3d_max_height, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE3D_MAX_DEPTH, sizeof(size_t), &backend_ctx->image3d_max_depth, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(size_t), &backend_ctx->max_workgroup_size, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_LOCAL_MEM_SIZE, sizeof(cl_ulong), &backend_ctx->local_mem_size, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_SVM_CAPABILITIES, sizeof(cl_device_svm_capabilities), &backend_ctx->svm_caps, 0));
|
||||
@@ -11660,6 +11679,26 @@ static bool use_fa_bin_kernels_prefill(const ggml_backend_opencl_context * backe
|
||||
|
||||
constexpr bool prefill_only = true;
|
||||
|
||||
// The repacked Q/K/V are 3D images: Q {n_q, n_batch*n_head, dk/4}, K {dk, n_kv/4, n_batch*n_head_kv},
|
||||
// V {n_kv, dv/4, n_batch*n_head_kv}. Past the device's 3D image limits clCreateImage fails and the
|
||||
// call asserts (X2-90: V is n_kv wide, so any prefill past ~16k tokens of context), so decline those shapes.
|
||||
const size_t n_kv = (size_t) k->ne[1];
|
||||
const size_t n_bh = (size_t) q->ne[3] * (size_t) q->ne[2];
|
||||
const size_t n_bhkv = (size_t) q->ne[3] * (size_t) k->ne[2];
|
||||
if ((size_t) n_q > backend_ctx->image3d_max_width || n_bh > backend_ctx->image3d_max_height ||
|
||||
(size_t) dk / 4 > backend_ctx->image3d_max_depth ||
|
||||
(size_t) dk > backend_ctx->image3d_max_width || (n_kv + 3) / 4 > backend_ctx->image3d_max_height ||
|
||||
n_kv > backend_ctx->image3d_max_width || (size_t) dv / 4 > backend_ctx->image3d_max_height ||
|
||||
n_bhkv > backend_ctx->image3d_max_depth) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// q8_0 K/V at DK=512 gives wrong results here when V differs from K (f16 K/V is correct).
|
||||
// Decline it, as this backend does without the kernel library.
|
||||
if (is_q8_0 && dk == 512) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (backend_ctx->gpu_family == GPU_FAMILY::ADRENO &&
|
||||
(is_mixed || is_q8_0) && (dk == dv)
|
||||
&& (dk == 64 || dk == 128 || dk == 256 || dk == 512)
|
||||
@@ -11775,7 +11814,8 @@ inline bool use_q6_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
|
||||
inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
if (!backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans ||
|
||||
!backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin) {
|
||||
!(backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin ||
|
||||
backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin)) {
|
||||
return false;
|
||||
}
|
||||
// bin kernels require ne2 == 1 and ne3 == 1 for weights
|
||||
@@ -22794,6 +22834,7 @@ static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_te
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
|
||||
|
||||
CL_CHECK(clReleaseMemObject(mem_tex_matrixO_1dbuf));
|
||||
CL_CHECK(clReleaseMemObject(mem_tex_mask_fallback_1dbuf));
|
||||
CL_CHECK(clReleaseMemObject(img_q_wmm));
|
||||
CL_CHECK(clReleaseMemObject(img_k_wmm));
|
||||
CL_CHECK(clReleaseMemObject(img_v_wmm));
|
||||
@@ -27675,6 +27716,8 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const gg
|
||||
const int gemm_tile_n = 64;
|
||||
int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n;
|
||||
|
||||
const bool have_a8 = backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin != nullptr;
|
||||
|
||||
static const char * q4_k_bin_dp4a_env = getenv("GGML_OPENCL_Q4_K_BIN_DP4A");
|
||||
bool q4_k_bin_dp4a_on = q4_k_bin_dp4a_env
|
||||
? (atoi(q4_k_bin_dp4a_env) != 0)
|
||||
@@ -27744,6 +27787,74 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const gg
|
||||
return;
|
||||
}
|
||||
|
||||
if (!have_a8) {
|
||||
cl_mem b_src_sub = nullptr;
|
||||
cl_mem b_pad_sub = nullptr;
|
||||
cl_mem b_buf = nullptr;
|
||||
region.origin = offset1;
|
||||
region.size = (size_t)K * N * sizeof(float);
|
||||
CL_CHECK((b_src_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
|
||||
if (N_pad == N) {
|
||||
b_buf = b_src_sub;
|
||||
} else {
|
||||
size_t padded_bytes = (size_t)K * N_pad * sizeof(float);
|
||||
backend_ctx->prealloc_act_trans.allocate(context, padded_bytes);
|
||||
region.origin = 0;
|
||||
region.size = padded_bytes;
|
||||
CL_CHECK((b_pad_sub = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
|
||||
|
||||
const float b_pad_zero = 0.0f;
|
||||
CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_pad_sub, &b_pad_zero, sizeof(float),
|
||||
0, padded_bytes, 0, NULL, NULL));
|
||||
CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, b_src_sub, b_pad_sub, 0, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL));
|
||||
b_buf = b_pad_sub;
|
||||
}
|
||||
|
||||
img_fmt = { CL_R, CL_FLOAT };
|
||||
memset(&img_desc, 0, sizeof(img_desc));
|
||||
img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
|
||||
img_desc.image_width = (size_t)K * N_pad;
|
||||
img_desc.buffer = b_buf;
|
||||
cl_mem b_img_a7;
|
||||
CL_CHECK((b_img_a7 = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
|
||||
|
||||
const int c_offset_for_kernel = (int)offsetd;
|
||||
|
||||
int line_stride_B = K * (int)sizeof(float);
|
||||
int line_stride_C = M * (int)sizeof(float);
|
||||
|
||||
kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin;
|
||||
cl_uint k_arg = 0;
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->q_img));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->d));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->dm));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->s));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img_a7));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset1));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_C));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N));
|
||||
|
||||
size_t local_work_size[3] = { 64, 2, 4 };
|
||||
size_t m_tiles = (size_t)CEIL_DIV(M, 64);
|
||||
size_t global_y = ((m_tiles + 1) / 2) * 2;
|
||||
size_t global_work_size[3] = { 64, global_y, (size_t)CEIL_DIV(N_pad, 32) };
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
|
||||
|
||||
CL_CHECK(clReleaseMemObject(b_img_a7));
|
||||
if (b_src_sub) {
|
||||
CL_CHECK(clReleaseMemObject(b_src_sub));
|
||||
}
|
||||
if (b_pad_sub) {
|
||||
CL_CHECK(clReleaseMemObject(b_pad_sub));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
cl_mem b_sub_buf = nullptr;
|
||||
cl_mem b_padded = nullptr;
|
||||
cl_mem b_buf = nullptr;
|
||||
|
||||
@@ -335,7 +335,7 @@ static int ggml_vk_comm_sfd_export(vk_comm_device & d) {
|
||||
int fd = -1;
|
||||
VkSemaphoreGetFdInfoKHR gi{};
|
||||
gi.sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR;
|
||||
gi.semaphore = d.sfd.sem;
|
||||
gi.semaphore = static_cast<VkSemaphore>(d.sfd.sem);
|
||||
gi.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
|
||||
const VkResult result = d.sfd.get_fd((VkDevice) d.ctx->device->device, &gi, &fd);
|
||||
if (result != VK_SUCCESS) {
|
||||
@@ -348,7 +348,7 @@ static int ggml_vk_comm_sfd_export(vk_comm_device & d) {
|
||||
static void ggml_vk_comm_sfd_import(vk_comm_device & d, vk::Semaphore sem, int fd) {
|
||||
VkImportSemaphoreFdInfoKHR isi{};
|
||||
isi.sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR;
|
||||
isi.semaphore = sem;
|
||||
isi.semaphore = static_cast<VkSemaphore>(sem);
|
||||
isi.flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT;
|
||||
isi.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
|
||||
isi.fd = fd;
|
||||
|
||||
@@ -70,7 +70,7 @@ void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t siz
|
||||
vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size);
|
||||
|
||||
// pipelines
|
||||
uint64_t vk_tensor_offset(const ggml_tensor * tensor);
|
||||
size_t vk_tensor_offset(const ggml_tensor * tensor);
|
||||
uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t);
|
||||
void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx);
|
||||
void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx, vk_pipeline& pipeline, uint32_t n);
|
||||
|
||||
@@ -466,15 +466,15 @@ struct vk_conv2d_pipeline_state {
|
||||
|
||||
struct vk_conv3d_pipeline_state {
|
||||
vk_conv3d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t s2, uint32_t p0, uint32_t p1, uint32_t p2,
|
||||
uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned)
|
||||
: s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned) {}
|
||||
uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned, uint32_t split_k)
|
||||
: s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned), split_k(split_k) {}
|
||||
|
||||
uint32_t s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD;
|
||||
uint32_t aligned;
|
||||
uint32_t aligned, split_k;
|
||||
|
||||
bool operator<(const vk_conv3d_pipeline_state &b) const {
|
||||
return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned) <
|
||||
std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned);
|
||||
return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned, split_k) <
|
||||
std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned, b.split_k);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -259,7 +259,7 @@ void ggml_vk_print_device_lost_info(const vk_device& device) {
|
||||
}
|
||||
void * const vk_ptr_base = (void *)(uintptr_t) 0x1000; // NOLINT
|
||||
|
||||
uint64_t vk_tensor_offset(const ggml_tensor * tensor) {
|
||||
size_t vk_tensor_offset(const ggml_tensor * tensor) {
|
||||
if (tensor->view_src) {
|
||||
return (uint8_t *) tensor->view_src->data - (uint8_t *) vk_ptr_base;
|
||||
}
|
||||
@@ -1258,10 +1258,14 @@ static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device,
|
||||
|
||||
result.shmem_staging = (device->vendor_id == VK_VENDOR_ID_NVIDIA && hsk < 256 && hsv < 256) ? 1 : 0;
|
||||
|
||||
if (!reduce_block_rows && !ggml_vk_flash_attn_scalar_shmem_support(device, result, hsk, hsv, f32acc, k_type, v_type)) {
|
||||
if (!reduce_block_rows && result.block_rows > 1 && !ggml_vk_flash_attn_scalar_shmem_support(device, result, hsk, hsv, f32acc, k_type, v_type)) {
|
||||
result.block_rows /= 2;
|
||||
}
|
||||
|
||||
if (result.shmem_staging && !ggml_vk_flash_attn_scalar_shmem_support(device, result, hsk, hsv, f32acc, k_type, v_type)) {
|
||||
result.shmem_staging = 0;
|
||||
}
|
||||
|
||||
// With head sizes above 128, 16 rows per workgroup spill registers on Qualcomm and run about
|
||||
// 40x slower; at large KV a single dispatch can then exceed the Windows GPU timeout.
|
||||
if (device->vendor_id == VK_VENDOR_ID_QUALCOMM && (hsk > 128 || hsv > 128)) {
|
||||
@@ -3886,8 +3890,16 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
// variant (spec constant 1) additionally gathers the qwen4 indexer input on the fly.
|
||||
{
|
||||
const uint32_t BLOCK_SIZE = 1u << std::min(10u, device->max_workgroup_size_log2);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_f32, "topk_radix_f32", topk_radix_select_f32_len, topk_radix_select_f32_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 0}, 1, true);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_qsa, "topk_radix_qsa", topk_radix_select_f32_len, topk_radix_select_f32_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 1}, 1, true);
|
||||
// the bucket scan needs one invocation per radix bucket
|
||||
if (BLOCK_SIZE >= 256) {
|
||||
if (device->subgroup_arithmetic && device->subgroup_require_full_support) {
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_f32, "topk_radix_f32", topk_radix_select_f32_subgroup_len, topk_radix_select_f32_subgroup_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 0, device->subgroup_size}, 1, true, true, device->subgroup_size);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_qsa, "topk_radix_qsa", topk_radix_select_f32_subgroup_len, topk_radix_select_f32_subgroup_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 1, device->subgroup_size}, 1, true, true, device->subgroup_size);
|
||||
} else {
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_f32, "topk_radix_f32", topk_radix_select_f32_len, topk_radix_select_f32_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 0}, 1, true);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_qsa, "topk_radix_qsa", topk_radix_select_f32_len, topk_radix_select_f32_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 1}, 1, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ggml_vk_create_pipeline(device, device->pipeline_argmax_f32, "argmax_f32", argmax_f32_len, argmax_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1);
|
||||
@@ -4282,6 +4294,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
spec_constants_cpy.push_back(conv2d_csh_store); \
|
||||
spec_constants_cpy.push_back(conv2d_WM); \
|
||||
spec_constants_cpy.push_back(conv2d_WN); \
|
||||
spec_constants_cpy.push_back(state.split_k); \
|
||||
ggml_vk_create_pipeline( \
|
||||
device, c.second, "conv3d" #type_suffix, \
|
||||
conv3d##type_suffix##spv_suffix##_len, conv3d##type_suffix##spv_suffix##_data, "main", 3, \
|
||||
@@ -8520,7 +8533,9 @@ bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_f
|
||||
kblocksh_size = 0;
|
||||
}
|
||||
|
||||
const uint32_t total_size = tmpsh + tmpshv4 + masksh + iq_shmem + Qf + kvsh + kblocksh_size;
|
||||
const uint32_t occupancy_limiter = std::max(params.limit_occupancy_shmem, 1u) * 4 * sizeof(float);
|
||||
|
||||
const uint32_t total_size = tmpsh + tmpshv4 + masksh + iq_shmem + Qf + kvsh + kblocksh_size + occupancy_limiter;
|
||||
const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize;
|
||||
|
||||
VK_LOG_DEBUG("ggml_vk_flash_attn_scalar_shmem_support(HSK=" << hsk << ", HSV=" << hsv << ", mmq=" << mmq << ", total_size=" << total_size << ", supported=" << supported);
|
||||
@@ -9271,24 +9286,58 @@ static vk_conv_shapes ggml_vk_conv_select_shape(ggml_backend_vk_context * ctx, u
|
||||
}
|
||||
}
|
||||
|
||||
static vk_conv_shapes ggml_vk_conv3d_select_shape(ggml_backend_vk_context * ctx, uint32_t K, uint32_t NPQ) {
|
||||
const auto & block = vk_conv_block_sizes[CONV_SHAPE_256x128];
|
||||
static vk_conv_shapes ggml_vk_conv3d_select_shape(ggml_backend_vk_context * ctx, const ggml_tensor * kernel, const ggml_tensor * dst, uint32_t & split_k) {
|
||||
split_k = 1;
|
||||
const uint32_t K = ggml_get_op_params_i32(dst, 11);
|
||||
const uint32_t NPQ = ggml_get_op_params_i32(dst, 10) * dst->ne[0] * dst->ne[1] * dst->ne[2];
|
||||
const vk_conv_shapes fallback = ggml_vk_conv_select_shape(ctx, K, NPQ);
|
||||
const uint32_t shader_core_count = ctx->device->shader_core_count;
|
||||
// Slowdowns were observed on DGX Spark with drivers older than R615.
|
||||
const bool driver_supported = ctx->device->driver_id != vk::DriverId::eNvidiaProprietary ||
|
||||
(ctx->device->properties.driverVersion >> 22) >= 615;
|
||||
if (ctx->device->coopmat2 && shader_core_count > 0 && K % block.K == 0 && driver_supported) {
|
||||
const uint64_t n_tiles = (uint64_t)(K / block.K) * CEIL_DIV((uint64_t)NPQ, block.NPQ);
|
||||
// Use the largest cm2 padding and include the dummy Csh[8].
|
||||
const uint32_t pad = 8, csh_elems = 8;
|
||||
const uint32_t shmem = (block.K * (block.CRS + pad) + block.CRS * (block.NPQ + pad) + csh_elems) * sizeof(uint16_t);
|
||||
// Reuse each loaded input across 256 output channels instead of 128.
|
||||
// Require full channel tiles and at least as many workgroups as GPU cores.
|
||||
if (n_tiles >= shader_core_count && shmem <= ctx->device->properties.limits.maxComputeSharedMemorySize) {
|
||||
return CONV_SHAPE_256x128;
|
||||
}
|
||||
if (!ctx->device->coopmat2 || shader_core_count == 0 || K == 0 || NPQ == 0 || !driver_supported) {
|
||||
return fallback;
|
||||
}
|
||||
return ggml_vk_conv_select_shape(ctx, K, NPQ);
|
||||
const auto & block = vk_conv_block_sizes[CONV_SHAPE_256x128];
|
||||
const auto & limits = ctx->device->properties.limits;
|
||||
// Use the largest cm2 padding and include the dummy Csh[8].
|
||||
const uint32_t pad = 8, csh_elems = 8;
|
||||
const uint32_t shmem = (block.K * (block.CRS + pad) + block.CRS * (block.NPQ + pad) + csh_elems) * sizeof(uint16_t);
|
||||
if (shmem > limits.maxComputeSharedMemorySize) {
|
||||
return fallback;
|
||||
}
|
||||
const uint32_t channel_tiles = CEIL_DIV(K, block.K);
|
||||
const uint64_t n_tiles = (uint64_t)channel_tiles * CEIL_DIV(NPQ, block.NPQ);
|
||||
// Keep full channel tiles when the grid already fills the GPU.
|
||||
if (n_tiles >= shader_core_count) {
|
||||
return K % block.K == 0 ? CONV_SHAPE_256x128 : fallback;
|
||||
}
|
||||
static const bool disable_split_k = getenv("GGML_VK_DISABLE_CONV3D_SPLIT_K") != nullptr;
|
||||
// Do not use a tile with more than half its output channels padded.
|
||||
if (disable_split_k || K < block.K / 2 || !ggml_is_contiguous(dst) || get_misalign_bytes(ctx, dst) != 0) {
|
||||
return fallback;
|
||||
}
|
||||
// Use power-of-two splits to give each core at least one workgroup.
|
||||
uint64_t splits = 1;
|
||||
while (n_tiles * splits < shader_core_count) {
|
||||
splits *= 2;
|
||||
}
|
||||
const uint64_t crs = (uint64_t)ggml_get_op_params_i32(dst, 9) * kernel->ne[0] * kernel->ne[1] * kernel->ne[2];
|
||||
const uint64_t operand_bytes = block.K * ggml_type_size(kernel->type) + block.NPQ * sizeof(float);
|
||||
const uint64_t scratch_bytes = 2 * sizeof(float) * block.K * block.NPQ;
|
||||
// Do not add more scratch bytes than the estimated operand load bytes.
|
||||
if (crs * operand_bytes < splits * scratch_bytes) {
|
||||
return fallback;
|
||||
}
|
||||
const uint64_t ne = ggml_nelements(dst);
|
||||
const uint64_t bytes = ggml_nbytes(dst);
|
||||
if (ne == 0 || CEIL_DIV(ne, ctx->device->pipeline_matmul_split_k_reduce->wg_denoms[0]) > limits.maxComputeWorkGroupCount[0] ||
|
||||
splits > limits.maxStorageBufferRange / bytes ||
|
||||
splits > limits.maxComputeWorkGroupCount[0] / channel_tiles) {
|
||||
return fallback;
|
||||
}
|
||||
split_k = (uint32_t)splits;
|
||||
return CONV_SHAPE_256x128;
|
||||
}
|
||||
|
||||
static vk_pipeline ggml_vk_rms_norm_pipeline(const std::array<vk_pipeline, 3> & pipelines, int64_t ncols) {
|
||||
@@ -9997,7 +10046,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
const uint32_t IC = (uint32_t)ggml_get_op_params_i32(dst, 9);
|
||||
const uint32_t N = (uint32_t)ggml_get_op_params_i32(dst, 10);
|
||||
const uint32_t NPQ = N * dst->ne[2] * dst->ne[1] * dst->ne[0];
|
||||
const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, OC, NPQ);
|
||||
uint32_t split_k;
|
||||
const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, src0, dst, split_k);
|
||||
|
||||
const uint32_t KW = (uint32_t)src0->ne[0];
|
||||
const uint32_t KH = (uint32_t)src0->ne[1];
|
||||
@@ -10020,7 +10070,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
(CRS % BS_CRS == 0) &&
|
||||
(NPQ % BS_NPQ == 0)) ? 1u : 0u;
|
||||
|
||||
vk_conv3d_pipeline_state conv3d_pipeline_state(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned);
|
||||
vk_conv3d_pipeline_state conv3d_pipeline_state(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned, split_k);
|
||||
|
||||
std::map<vk_conv3d_pipeline_state, vk_pipeline> *pipelines = nullptr;
|
||||
if (src0->type == GGML_TYPE_F32) {
|
||||
@@ -10126,6 +10176,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
|
||||
init_pushconst_tensor_offsets(ctx, pc, src0, src1, src2, src3, dst);
|
||||
|
||||
std::array<uint32_t, 3> elements;
|
||||
uint32_t split_k = 1;
|
||||
|
||||
switch (op) {
|
||||
case GGML_OP_NORM:
|
||||
@@ -10301,10 +10352,12 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
|
||||
case GGML_OP_CONV_3D:
|
||||
if constexpr (std::is_same_v<PC, vk_op_conv3d_push_constants>) {
|
||||
const uint32_t NPQ = pc.N * pc.OD * pc.OH * pc.OW;
|
||||
const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, pc.OC, NPQ);
|
||||
const uint32_t NPQ_blocks = CEIL_DIV(NPQ, vk_conv_block_sizes[shape].NPQ);
|
||||
const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, src0, dst, split_k);
|
||||
const auto & block = vk_conv_block_sizes[shape];
|
||||
const uint32_t NPQ_blocks = CEIL_DIV(NPQ, block.NPQ);
|
||||
const uint32_t channel_groups = split_k > 1 ? CEIL_DIV(pc.OC, block.K) * block.K * split_k : pc.OC;
|
||||
|
||||
elements = { pc.OC, NPQ_blocks, 1 };
|
||||
elements = { channel_groups, NPQ_blocks, 1 };
|
||||
if (elements[1] > 512) {
|
||||
elements[2] = CEIL_DIV(elements[1], 512);
|
||||
elements[1] = 512;
|
||||
@@ -10435,6 +10488,29 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
|
||||
break;
|
||||
}
|
||||
|
||||
if constexpr (std::is_same_v<PC, vk_op_conv3d_push_constants>) {
|
||||
if (split_k > 1) {
|
||||
const uint32_t ne = (uint32_t)ggml_nelements(dst);
|
||||
const size_t scratch_size = ggml_nbytes(dst) * split_k;
|
||||
if (ctx->prealloc_size_split_k < scratch_size) {
|
||||
ctx->prealloc_size_split_k = scratch_size;
|
||||
ggml_vk_preallocate_buffers(ctx, subctx);
|
||||
}
|
||||
if (ctx->prealloc_split_k_need_sync) {
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
}
|
||||
vk_subbuffer scratch = { ctx->prealloc_split_k, 0, scratch_size };
|
||||
pc.dst_offset = 0;
|
||||
ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, 1);
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, scratch }, pc, elements);
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
const std::array<uint32_t, 2> reduce_pc = { ne, split_k };
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_matmul_split_k_reduce, { scratch, dst_buf }, reduce_pc, { ne, 1, 1 });
|
||||
ctx->prealloc_split_k_need_sync = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (op == GGML_OP_ADD || op == GGML_OP_RMS_NORM) {
|
||||
vk_subbuffer a_buf = src0_buf;
|
||||
if (ctx->do_add_rms_partials) {
|
||||
@@ -17228,7 +17304,7 @@ void vk_queue_handle_unsynchronized::submit(vk::ArrayProxy<const vk::SubmitInfo>
|
||||
|
||||
vk_device_struct::~vk_device_struct() {
|
||||
// the build failed before the device was created, nothing to destroy
|
||||
if (device == VK_NULL_HANDLE) {
|
||||
if (!device) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -112,6 +112,8 @@ const uint warps_M = BS_K / WM;
|
||||
const uint warps_N = BS_NPQ / WN;
|
||||
#endif
|
||||
|
||||
layout(constant_id = 22) const uint split_k = 1;
|
||||
|
||||
// without padding, ID_idx/IH_idx/IW_idx are in bounds by construction
|
||||
const bool dhw_in_bounds = (p0 == 0) && (p1 == 0) && (p2 == 0);
|
||||
|
||||
@@ -170,8 +172,11 @@ D,R,S=KD,KH,KW
|
||||
Z,P,Q=OD,OH,OW
|
||||
*/
|
||||
|
||||
uint32_t B_idx_K = gl_WorkGroupID.x;
|
||||
// Packing splits into the low bits would avoid runtime division, but adjacent channel tiles give better input cache reuse.
|
||||
uint32_t B_idx_K = split_k > 1 ? gl_WorkGroupID.x % splitWork(K, BS_K) : gl_WorkGroupID.x;
|
||||
uint32_t B_idx_split = split_k > 1 ? gl_WorkGroupID.x / splitWork(K, BS_K) : 0;
|
||||
uint32_t B_idx_NPQ = gl_WorkGroupID.y + gl_WorkGroupID.z * 512;
|
||||
uint32_t split_offset = B_idx_split * K * NPQ;
|
||||
|
||||
uint32_t T_y = tid / NT_NPQ;
|
||||
uint32_t T_x = tid % NT_NPQ;
|
||||
@@ -219,7 +224,7 @@ ACC_TYPE perElemOpStore(const in uint32_t r, const in uint32_t c, const in ACC_T
|
||||
split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx);
|
||||
uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3;
|
||||
if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
|
||||
dst_data[dst_idx + p.dst_offset] = D_TYPE(elem);
|
||||
dst_data[dst_idx + p.dst_offset + split_offset] = D_TYPE(elem);
|
||||
}
|
||||
return elem;
|
||||
}
|
||||
@@ -249,7 +254,11 @@ void main() {
|
||||
}
|
||||
#endif
|
||||
/* Advance block in CRS dim */
|
||||
[[dont_unroll]] for (uint32_t B_idx_CRS = 0; B_idx_CRS < NB_CRS; B_idx_CRS++) {
|
||||
const uint32_t crs_count = NB_CRS / split_k;
|
||||
const uint32_t crs_extra = NB_CRS % split_k;
|
||||
const uint32_t crs_begin = B_idx_split * crs_count + min(B_idx_split, crs_extra);
|
||||
const uint32_t crs_end = crs_begin + crs_count + uint32_t(B_idx_split < crs_extra);
|
||||
[[dont_unroll]] for (uint32_t B_idx_CRS = crs_begin; B_idx_CRS < crs_end; B_idx_CRS++) {
|
||||
uint32_t CRS_idx_a = B_idx_CRS * BS_CRS + Ac;
|
||||
uint32_t IC_idx_a;
|
||||
uint32_t KD_idx_a;
|
||||
@@ -398,7 +407,7 @@ void main() {
|
||||
split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx);
|
||||
uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3;
|
||||
if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
|
||||
dst_data[dst_idx + p.dst_offset] = D_TYPE(Csh[k_local * Csh_stride + npq_thread]);
|
||||
dst_data[dst_idx + p.dst_offset + split_offset] = D_TYPE(Csh[k_local * Csh_stride + npq_thread]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -420,7 +429,7 @@ void main() {
|
||||
split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx);
|
||||
uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3;
|
||||
if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
|
||||
dst_data[dst_idx + p.dst_offset] = D_TYPE(regC[T_ly][T_lx]);
|
||||
dst_data[dst_idx + p.dst_offset + split_offset] = D_TYPE(regC[T_ly][T_lx]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,11 +2,20 @@
|
||||
|
||||
#extension GL_EXT_control_flow_attributes : enable
|
||||
#extension GL_EXT_shader_16bit_storage : require
|
||||
#if USE_SUBGROUPS
|
||||
#extension GL_KHR_shader_subgroup_basic : enable
|
||||
#extension GL_KHR_shader_subgroup_arithmetic : enable
|
||||
#endif
|
||||
|
||||
#include "types.glsl"
|
||||
|
||||
layout(constant_id = 0) const int BLOCK_SIZE = 1024;
|
||||
layout(constant_id = 1) const int QSA = 0; // 1: fuse the qwen4 QSA indexer gather + f16 mask
|
||||
#if USE_SUBGROUPS
|
||||
layout(constant_id = 2) const int SUBGROUP_SIZE = 32;
|
||||
|
||||
const uint NUM_SUBGROUPS = BLOCK_SIZE / SUBGROUP_SIZE;
|
||||
#endif
|
||||
|
||||
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
@@ -26,12 +35,19 @@ layout (push_constant) uniform parameter {
|
||||
} p;
|
||||
|
||||
#define RADIX_BITS 8
|
||||
#define UNROLL 4
|
||||
#define RADIX_SIZE (1 << RADIX_BITS)
|
||||
|
||||
shared uint histo[RADIX_SIZE];
|
||||
shared uint sh_bucket;
|
||||
shared uint sh_above;
|
||||
shared uint sh_tied;
|
||||
shared uint out_count;
|
||||
#if USE_SUBGROUPS
|
||||
shared uint sg_tied[NUM_SUBGROUPS];
|
||||
#else
|
||||
shared uint tie_scan[2 * BLOCK_SIZE];
|
||||
#endif
|
||||
|
||||
// order-preserving float -> uint mapping
|
||||
uint f2ui(float x) {
|
||||
@@ -87,52 +103,155 @@ void topk(const uint row) {
|
||||
const bool first = (shift == 32 - RADIX_BITS);
|
||||
const uint hi_mask = (shift + RADIX_BITS >= 32) ? 0u : (0xFFFFFFFFu << uint(shift + RADIX_BITS));
|
||||
const uint prefix_hi = prefix & hi_mask;
|
||||
for (uint i = tid; i < ncols; i += BLOCK_SIZE) {
|
||||
const uint key = f2ui(load(row, i, first));
|
||||
if ((key & hi_mask) == prefix_hi) {
|
||||
atomicAdd(histo[(key >> uint(shift)) & (RADIX_SIZE - 1)], 1u);
|
||||
uint run_b = RADIX_SIZE;
|
||||
uint run_n = 0;
|
||||
for (uint i0 = tid; i0 < ncols; i0 += UNROLL * BLOCK_SIZE) {
|
||||
uint keys[UNROLL];
|
||||
[[unroll]] for (uint u = 0; u < UNROLL; ++u) {
|
||||
const uint i = i0 + u * BLOCK_SIZE;
|
||||
keys[u] = f2ui(load(row, min(i, ncols - 1), first));
|
||||
}
|
||||
[[unroll]] for (uint u = 0; u < UNROLL; ++u) {
|
||||
const uint key = keys[u];
|
||||
if (i0 + u * BLOCK_SIZE < ncols && (key & hi_mask) == prefix_hi) {
|
||||
const uint b = (key >> uint(shift)) & (RADIX_SIZE - 1);
|
||||
if (b != run_b) {
|
||||
if (run_n != 0) {
|
||||
atomicAdd(histo[run_b], run_n);
|
||||
}
|
||||
run_b = b;
|
||||
run_n = 0;
|
||||
}
|
||||
run_n++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (run_n != 0) {
|
||||
atomicAdd(histo[run_b], run_n);
|
||||
}
|
||||
barrier();
|
||||
|
||||
// top-down scan for the bucket holding the K-th value
|
||||
if (tid == 0) {
|
||||
uint acc = 0;
|
||||
uint b = 0;
|
||||
for (int bb = RADIX_SIZE - 1; bb >= 0; --bb) {
|
||||
const uint c = histo[bb];
|
||||
if (acc + c >= desired) { b = uint(bb); break; }
|
||||
acc += c;
|
||||
#if USE_SUBGROUPS
|
||||
uint c = 0, incl = 0;
|
||||
if (tid < RADIX_SIZE) {
|
||||
c = histo[RADIX_SIZE - 1 - tid];
|
||||
incl = subgroupInclusiveAdd(c);
|
||||
if (gl_SubgroupInvocationID == gl_SubgroupSize - 1) {
|
||||
sg_tied[gl_SubgroupID] = incl;
|
||||
}
|
||||
sh_bucket = b;
|
||||
sh_above = acc;
|
||||
}
|
||||
barrier();
|
||||
if (tid < RADIX_SIZE) {
|
||||
for (uint s = 0; s < gl_SubgroupID; ++s) {
|
||||
incl += sg_tied[s];
|
||||
}
|
||||
if (incl - c < desired && incl >= desired) {
|
||||
sh_bucket = RADIX_SIZE - 1 - tid;
|
||||
sh_above = incl - c;
|
||||
sh_tied = c;
|
||||
}
|
||||
}
|
||||
barrier();
|
||||
#else
|
||||
uint c = 0;
|
||||
if (tid < RADIX_SIZE) {
|
||||
c = histo[RADIX_SIZE - 1 - tid];
|
||||
tie_scan[tid] = c;
|
||||
}
|
||||
barrier();
|
||||
uint src = 0;
|
||||
[[unroll]] for (uint off = 1; off < RADIX_SIZE; off <<= 1) {
|
||||
if (tid < RADIX_SIZE) {
|
||||
const uint v = tie_scan[src + tid];
|
||||
tie_scan[(RADIX_SIZE - src) + tid] = tid >= off ? v + tie_scan[src + tid - off] : v;
|
||||
}
|
||||
src = RADIX_SIZE - src;
|
||||
barrier();
|
||||
}
|
||||
if (tid < RADIX_SIZE) {
|
||||
const uint incl = tie_scan[src + tid];
|
||||
if (incl - c < desired && incl >= desired) {
|
||||
sh_bucket = RADIX_SIZE - 1 - tid;
|
||||
sh_above = incl - c;
|
||||
sh_tied = c;
|
||||
}
|
||||
}
|
||||
barrier();
|
||||
#endif
|
||||
|
||||
prefix |= sh_bucket << uint(shift);
|
||||
desired -= sh_above;
|
||||
barrier();
|
||||
}
|
||||
|
||||
const uint threshold = prefix;
|
||||
const bool all_ties = sh_tied == desired;
|
||||
|
||||
if (tid == 0) {
|
||||
out_count = 0;
|
||||
}
|
||||
barrier();
|
||||
|
||||
// emit everything above the threshold, then fill the rest from ties
|
||||
const uint threshold = prefix;
|
||||
if (all_ties) {
|
||||
for (uint i = tid; i < ncols; i += BLOCK_SIZE) {
|
||||
if (f2ui(load(row, i, false)) >= threshold) {
|
||||
data_d[row_out + atomicAdd(out_count, 1u)] = int(i);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// more ties than slots: rank them in invocation order, which does not depend on the scheduling
|
||||
uint ct = 0;
|
||||
for (uint i = tid; i < ncols; i += BLOCK_SIZE) {
|
||||
if (f2ui(load(row, i, false)) > threshold) {
|
||||
const uint key = f2ui(load(row, i, false));
|
||||
if (key > threshold) {
|
||||
data_d[row_out + atomicAdd(out_count, 1u)] = int(i);
|
||||
}
|
||||
ct += key == threshold ? 1u : 0u;
|
||||
}
|
||||
#if USE_SUBGROUPS
|
||||
const uint sg = gl_SubgroupID;
|
||||
const uint st = subgroupAdd(ct);
|
||||
if (subgroupElect()) {
|
||||
sg_tied[sg] = st;
|
||||
}
|
||||
barrier();
|
||||
|
||||
if (sg == 0) {
|
||||
const uint lane = gl_SubgroupInvocationID;
|
||||
uint acc = 0;
|
||||
for (uint c0 = 0; c0 < NUM_SUBGROUPS; c0 += SUBGROUP_SIZE) {
|
||||
const uint c = c0 + lane;
|
||||
const uint t = c < NUM_SUBGROUPS ? sg_tied[c] : 0u;
|
||||
const uint e = subgroupExclusiveAdd(t);
|
||||
if (c < NUM_SUBGROUPS) {
|
||||
sg_tied[c] = acc + e;
|
||||
}
|
||||
acc += subgroupAdd(t);
|
||||
}
|
||||
}
|
||||
barrier();
|
||||
for (uint i = tid; i < ncols; i += BLOCK_SIZE) {
|
||||
|
||||
uint pos = p.k - desired + sg_tied[sg] + subgroupExclusiveAdd(ct);
|
||||
#else
|
||||
// inclusive Hillis-Steele scan, ping-ponging between the two halves
|
||||
uint src = 0;
|
||||
tie_scan[tid] = ct;
|
||||
barrier();
|
||||
for (uint off = 1; off < BLOCK_SIZE; off <<= 1) {
|
||||
const uint v = tie_scan[src + tid];
|
||||
tie_scan[(BLOCK_SIZE - src) + tid] = tid >= off ? v + tie_scan[src + tid - off] : v;
|
||||
src = BLOCK_SIZE - src;
|
||||
barrier();
|
||||
}
|
||||
uint pos = p.k - desired + tie_scan[src + tid] - ct;
|
||||
#endif
|
||||
const uint pos_end = min(p.k, pos + ct);
|
||||
for (uint i = tid; i < ncols && pos < pos_end; i += BLOCK_SIZE) {
|
||||
if (f2ui(load(row, i, false)) == threshold) {
|
||||
const uint pos = atomicAdd(out_count, 1u);
|
||||
if (pos < p.k) {
|
||||
data_d[row_out + pos] = int(i);
|
||||
}
|
||||
data_d[row_out + pos++] = int(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1115,6 +1115,7 @@ void process_shaders() {
|
||||
string_to_spv("topk_argsort_f32", "topk_argsort.comp", {{"A_TYPE", "float"}});
|
||||
string_to_spv("topk_nary_search_f32", "topk_nary_search.comp", {{"A_TYPE", "float"}});
|
||||
string_to_spv("topk_radix_select_f32", "topk_radix_select.comp", {{"A_TYPE", "float"}});
|
||||
string_to_spv("topk_radix_select_f32_subgroup", "topk_radix_select.comp", {{"A_TYPE", "float"}, {"USE_SUBGROUPS", "1"}});
|
||||
|
||||
string_to_spv("argmax_f32", "argmax.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "int"}}));
|
||||
string_to_spv("sum_rows_f32", "sum_rows.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
|
||||
|
||||
@@ -962,13 +962,16 @@ inline uint32_t ggml_webgpu_flash_attn_get_vec_kv_tile(size_t wg_mem_limit_byt
|
||||
uint32_t head_dim_qk,
|
||||
uint32_t head_dim_v,
|
||||
bool has_mask,
|
||||
bool kv_direct) {
|
||||
const uint32_t max_kv_tile =
|
||||
ggml_webgpu_flash_attn_max_kv_tile(wg_mem_limit_bytes, 1u, 1u, head_dim_qk, head_dim_v, has_mask, kv_direct);
|
||||
bool k_direct,
|
||||
bool v_direct) {
|
||||
// kv_shmem is dropped only when both K and V are read directly
|
||||
const uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(wg_mem_limit_bytes, 1u, 1u, head_dim_qk, head_dim_v,
|
||||
has_mask, k_direct && v_direct);
|
||||
GGML_ASSERT(max_kv_tile > 0);
|
||||
|
||||
uint32_t kv_tile = std::min(GGML_WEBGPU_FLASH_ATTN_VEC_MAX_KV_TILE, max_kv_tile);
|
||||
if (kv_direct) {
|
||||
// a direct read has no bounds check, so the tile must divide the KV padding
|
||||
if (k_direct || v_direct) {
|
||||
kv_tile = std::min(kv_tile, GGML_WEBGPU_KV_SEQ_PAD);
|
||||
while (GGML_WEBGPU_KV_SEQ_PAD % kv_tile != 0) {
|
||||
kv_tile -= 1u;
|
||||
@@ -3050,7 +3053,7 @@ class ggml_webgpu_shader_lib {
|
||||
uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
|
||||
context.wg_mem_limit_bytes, decisions.q_tile, decisions.use_sg_matrix ? context.sg_mat_n : 1u,
|
||||
key.common.head_dim_qk, key.common.head_dim_v, key.common.has_mask,
|
||||
key.common.k_direct || key.common.v_direct);
|
||||
key.common.k_direct && key.common.v_direct);
|
||||
|
||||
// WorkGroup storage size isn't enough for some params with subgroup matrices path (ref. https://github.com/ggml-org/llama.cpp/pull/26566)
|
||||
if (max_kv_tile == 0) {
|
||||
@@ -3063,7 +3066,7 @@ class ggml_webgpu_shader_lib {
|
||||
key.use_sg_matrix = false;
|
||||
max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
|
||||
context.wg_mem_limit_bytes, decisions.q_tile, 1u, key.common.head_dim_qk, key.common.head_dim_v,
|
||||
key.common.has_mask, key.common.k_direct || key.common.v_direct);
|
||||
key.common.has_mask, key.common.k_direct && key.common.v_direct);
|
||||
GGML_ASSERT(max_kv_tile > 0);
|
||||
}
|
||||
|
||||
@@ -3125,7 +3128,7 @@ class ggml_webgpu_shader_lib {
|
||||
ggml_webgpu_flash_attn_vec_decisions decisions = {};
|
||||
decisions.kv_tile = ggml_webgpu_flash_attn_get_vec_kv_tile(context.wg_mem_limit_bytes, key.common.head_dim_qk,
|
||||
key.common.head_dim_v, key.common.has_mask,
|
||||
key.common.k_direct || key.common.v_direct);
|
||||
key.common.k_direct, key.common.v_direct);
|
||||
decisions.wg_size = context.max_subgroup_size;
|
||||
|
||||
std::string variant = "flash_attn_vec";
|
||||
|
||||
@@ -90,7 +90,7 @@ static inline uint32_t ggml_webgpu_u32_from_f32(float value) {
|
||||
#define WEBGPU_NUM_PARAM_SLOT_SAFETY_MARGIN 10u
|
||||
#define WEBGPU_RUNTIME_WAIT_TIMEOUT_MS 30000u
|
||||
#define WEBGPU_RUNTIME_WAIT_TIMEOUT_NS (WEBGPU_RUNTIME_WAIT_TIMEOUT_MS * 1e6)
|
||||
#define WEBGPU_PARAMS_BUF_SIZE_BYTES 128 // enough for 32 parameters
|
||||
#define WEBGPU_PARAMS_BUF_SIZE_BYTES 256 // enough for 64 parameters
|
||||
#define WEBGPU_SET_ROWS_ERROR_BUF_SIZE_BYTES 4
|
||||
#define WEBGPU_STORAGE_BUF_BINDING_MULT 4 // a storage buffer binding size must be a multiple of 4
|
||||
|
||||
@@ -2104,8 +2104,10 @@ static ggml_webgpu_flash_attn_op ggml_webgpu_flash_attn_prepare(webgpu_context &
|
||||
(uint32_t) (V->nb[2] / ggml_type_size(V->type)), // stride (elements/blocks) of V in dimension 2
|
||||
(uint32_t) (V->nb[3] / ggml_type_size(V->type)), // stride (elements/blocks) of V in dimension 3
|
||||
op.has_mask ? (uint32_t) (mask->nb[3] / ggml_type_size(mask->type)) : 0, // stride of mask dim 3
|
||||
(uint32_t) (Q->ne[2] / K->ne[2]), // repeat factor for K/V in dim 2 (MHA/MQA/GQA)
|
||||
ggml_webgpu_u32_from_f32(scale), // scale (possibly adjusted for logit softcap)
|
||||
op.has_mask ? (uint32_t) (mask->nb[2] / ggml_type_size(mask->type)) : 0, // stride of mask dim 2
|
||||
op.has_mask ? (uint32_t) mask->ne[2] : 1u, // mask heads, broadcast over the Q heads
|
||||
(uint32_t) (Q->ne[2] / K->ne[2]), // repeat factor for K/V in dim 2 (MHA/MQA/GQA)
|
||||
ggml_webgpu_u32_from_f32(scale), // scale (possibly adjusted for logit softcap)
|
||||
ggml_webgpu_u32_from_f32(max_bias),
|
||||
ggml_webgpu_u32_from_f32(logit_softcap),
|
||||
ggml_webgpu_u32_from_f32(n_head_log2),
|
||||
@@ -2211,7 +2213,9 @@ static webgpu_encoded_op ggml_webgpu_flash_attn_vec(webgpu_context & ct
|
||||
blk_nblk1 = (uint32_t) Q->ne[1];
|
||||
blk_buf = ggml_webgpu_tensor_buf(dst);
|
||||
const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
|
||||
blk_batch_count = stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u;
|
||||
const uint32_t stride_mask2 = (uint32_t) (mask->nb[2] / ggml_type_size(mask->type));
|
||||
const uint32_t mask_ne2 = (uint32_t) mask->ne[2];
|
||||
blk_batch_count = (stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u) * mask_ne2;
|
||||
const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
|
||||
blk_size_bytes = ROUNDUP_POW2(blk_elems * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT);
|
||||
const ggml_webgpu_shader_lib_context blk_shader_ctx = op.shader_lib_ctx;
|
||||
@@ -2222,6 +2226,8 @@ static webgpu_encoded_op ggml_webgpu_flash_attn_vec(webgpu_context & ct
|
||||
(uint32_t) Q->ne[1], // seq_len_q
|
||||
(uint32_t) K->ne[1], // seq_len_kv
|
||||
stride_mask3, // stride_mask3
|
||||
stride_mask2, // stride_mask2
|
||||
mask_ne2, // mask_ne2
|
||||
blk_nblk0, // nblk0
|
||||
blk_nblk1, // nblk1
|
||||
};
|
||||
@@ -4381,12 +4387,11 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer
|
||||
// longest vec batch and the size stays monotonic in the batch length
|
||||
const int64_t n_q = std::min<int64_t>(Q->ne[1], GGML_WEBGPU_FLASH_ATTN_VEC_MAX_SEQ_LEN - 1);
|
||||
if (ggml_webgpu_flash_attn_use_vec_path(ctx->webgpu_global_ctx, Q, K, V, n_q)) {
|
||||
const bool kv_direct =
|
||||
ggml_webgpu_flash_attn_k_direct(Q, K, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH) ||
|
||||
ggml_webgpu_flash_attn_v_direct(Q, V, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH);
|
||||
const uint32_t kv_tile = ggml_webgpu_flash_attn_get_vec_kv_tile(
|
||||
capabilities.limits.maxComputeWorkgroupStorageSize, (uint32_t) Q->ne[0], (uint32_t) V->ne[0],
|
||||
mask != nullptr, kv_direct);
|
||||
mask != nullptr,
|
||||
ggml_webgpu_flash_attn_k_direct(Q, K, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH),
|
||||
ggml_webgpu_flash_attn_v_direct(Q, V, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH));
|
||||
|
||||
const uint32_t vec_nwg_cap = capabilities.min_subgroup_size;
|
||||
uint32_t nwg = ggml_webgpu_flash_attn_vec_nwg(vec_nwg_cap, kv_tile, (uint32_t) K->ne[1]);
|
||||
@@ -4403,11 +4408,12 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer
|
||||
res += WEBGPU_STORAGE_BUF_BINDING_MULT + align;
|
||||
}
|
||||
if (mask != nullptr) {
|
||||
const uint32_t blk_nblk0 = CEIL_DIV((uint32_t) K->ne[1], kv_tile);
|
||||
const uint32_t blk_nblk1 = (uint32_t) n_q;
|
||||
const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
|
||||
const uint32_t blk_batch_count = stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u;
|
||||
const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
|
||||
const uint32_t blk_nblk0 = CEIL_DIV((uint32_t) K->ne[1], kv_tile);
|
||||
const uint32_t blk_nblk1 = (uint32_t) n_q;
|
||||
const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
|
||||
const uint32_t blk_batch_count =
|
||||
(stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u) * (uint32_t) mask->ne[2];
|
||||
const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
|
||||
const size_t blk_size_bytes =
|
||||
ROUNDUP_POW2(blk_elems * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT);
|
||||
res += blk_size_bytes + align;
|
||||
@@ -5014,14 +5020,22 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
|
||||
const uint32_t q_tile =
|
||||
use_subgroup_matrix ? capabilities.sg_mat_m : GGML_WEBGPU_FLASH_ATTN_TILE_Q_TILE;
|
||||
const uint32_t kv_granularity = use_subgroup_matrix ? capabilities.sg_mat_n : 1u;
|
||||
const bool kv_direct = use_subgroup_matrix ?
|
||||
ggml_webgpu_flash_attn_k_direct(src0, src1, capabilities.sg_mat_k) ||
|
||||
ggml_webgpu_flash_attn_v_direct(src0, src2, capabilities.sg_mat_k) :
|
||||
false;
|
||||
const uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
|
||||
// same direct read rule as get_flash_attn_pipeline: F16 only, on the subgroup matrix path
|
||||
const bool kv_direct = use_subgroup_matrix && src1->type == GGML_TYPE_F16 &&
|
||||
src2->type == GGML_TYPE_F16 &&
|
||||
ggml_webgpu_flash_attn_k_direct(src0, src1, capabilities.sg_mat_k) &&
|
||||
ggml_webgpu_flash_attn_v_direct(src0, src2, capabilities.sg_mat_k);
|
||||
const uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
|
||||
capabilities.limits.maxComputeWorkgroupStorageSize, q_tile, kv_granularity, (uint32_t) src0->ne[0],
|
||||
(uint32_t) src2->ne[0], op->src[3] != nullptr, kv_direct);
|
||||
supports_op = max_kv_tile > 0;
|
||||
// get_flash_attn_pipeline falls back to the register tile path when the subgroup matrix tile does not fit
|
||||
const uint32_t fallback_kv_tile =
|
||||
use_subgroup_matrix ?
|
||||
ggml_webgpu_flash_attn_max_kv_tile(
|
||||
capabilities.limits.maxComputeWorkgroupStorageSize, GGML_WEBGPU_FLASH_ATTN_TILE_Q_TILE, 1u,
|
||||
(uint32_t) src0->ne[0], (uint32_t) src2->ne[0], op->src[3] != nullptr, false) :
|
||||
0u;
|
||||
supports_op = max_kv_tile > 0 || fallback_kv_tile > 0;
|
||||
break;
|
||||
}
|
||||
case GGML_OP_GROUP_NORM:
|
||||
|
||||
@@ -116,7 +116,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
|
||||
#ifdef MASK
|
||||
// mask offset
|
||||
let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + q_row_start * params.seq_len_kv;
|
||||
let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
|
||||
#endif
|
||||
|
||||
// note that the output is permuted, the layout is [head_dim_v, n_heads, seq_len_q, batch_size]
|
||||
|
||||
@@ -65,6 +65,8 @@ struct Params {
|
||||
stride_v2: u32,
|
||||
stride_v3: u32,
|
||||
stride_mask3: u32,
|
||||
stride_mask2: u32,
|
||||
mask_ne2: u32,
|
||||
|
||||
// repeat factors for K/V, e.g., MHA vs. MQA vs. GQA
|
||||
q_per_kv: u32,
|
||||
|
||||
@@ -65,7 +65,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
let row_active = subgroup_id < Q_TILE && global_q_row < params.seq_len_q;
|
||||
|
||||
#ifdef MASK
|
||||
let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + q_row_start * params.seq_len_kv;
|
||||
let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
|
||||
#endif
|
||||
|
||||
let dst_global_offset = dst_batch_offset + q_row_start * dst2_stride + head_idx * HEAD_DIM_V;
|
||||
|
||||
@@ -9,6 +9,8 @@ struct Params {
|
||||
seq_len_q: u32,
|
||||
seq_len_kv: u32,
|
||||
stride_mask3: u32,
|
||||
stride_mask2: u32,
|
||||
mask_ne2: u32,
|
||||
// Number of KV blocks and Q blocks per batch.
|
||||
// nblk0 = ceil(seq_len_kv / KV_TILE), nblk1 = seq_len_q.
|
||||
nblk0: u32,
|
||||
@@ -30,11 +32,12 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
@builtin(local_invocation_id) local_id: vec3<u32>) {
|
||||
// Dispatch mapping:
|
||||
// - x indexes KV blocks
|
||||
// - y flattens (batch_idx, q_blk) as y = batch_idx * nblk1 + q_blk
|
||||
// - y flattens (batch_idx, mask_head, q_blk) as y = (batch_idx * mask_ne2 + mask_head) * nblk1 + q_blk
|
||||
let kv_blk = wg_id.x;
|
||||
let y = wg_id.y;
|
||||
let q_blk = y % params.nblk1;
|
||||
let batch_idx = y / params.nblk1;
|
||||
let mask_head = (y / params.nblk1) % params.mask_ne2;
|
||||
let batch_idx = y / (params.nblk1 * params.mask_ne2);
|
||||
if (kv_blk >= params.nblk0) {
|
||||
return;
|
||||
}
|
||||
@@ -43,7 +46,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
let k_start = kv_blk * KV_TILE;
|
||||
|
||||
let mask_batch = select(0u, batch_idx, params.stride_mask3 > 0u);
|
||||
let mask_batch_base = params.offset_mask + mask_batch * params.stride_mask3;
|
||||
let mask_batch_base = params.offset_mask + mask_batch * params.stride_mask3 + mask_head * params.stride_mask2;
|
||||
|
||||
// We keep min/max to classify:
|
||||
// - fully masked (max <= MASK_MIN)
|
||||
@@ -95,7 +98,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
}
|
||||
}
|
||||
|
||||
let blk_idx = (batch_idx * params.nblk1 + q_blk) * params.nblk0 + kv_blk;
|
||||
let blk_idx = ((batch_idx * params.mask_ne2 + mask_head) * params.nblk1 + q_blk) * params.nblk0 + kv_blk;
|
||||
blk[blk_idx] = state;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,7 +107,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
|
||||
#ifdef MASK
|
||||
// mask offset
|
||||
let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + q_row_start * params.seq_len_kv;
|
||||
let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
|
||||
#endif
|
||||
|
||||
let head = f32(head_idx);
|
||||
@@ -129,7 +129,8 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
let q_blk = q_row_start;
|
||||
let kv_blk = kv_tile / KV_TILE;
|
||||
let blk_batch = select(0u, batch_idx, params.stride_mask3 > 0u);
|
||||
let blk_idx = params.blk_base + (blk_batch * params.blk_nblk1 + q_blk) * params.blk_nblk0 + kv_blk;
|
||||
let blk_head = head_idx % params.mask_ne2;
|
||||
let blk_idx = params.blk_base + ((blk_batch * params.mask_ne2 + blk_head) * params.blk_nblk1 + q_blk) * params.blk_nblk0 + kv_blk;
|
||||
let blk_state_local = blk[blk_idx];
|
||||
#else
|
||||
let blk_state_local = 1u;
|
||||
|
||||
@@ -545,6 +545,7 @@ class Keys:
|
||||
PROJECTOR_TYPE = "clip.gen.audio.projector_type" # for mixed modality models
|
||||
# name of the weight variant, for settings that are not in the checkpoint
|
||||
MODEL_VARIANT = "clip.gen.audio.model_variant"
|
||||
DEFAULT_VOICE = "clip.gen.audio.default_voice"
|
||||
EMBEDDING_LENGTH = "clip.gen.audio.embedding_length"
|
||||
FEED_FORWARD_LENGTH = "clip.gen.audio.feed_forward_length"
|
||||
BLOCK_COUNT = "clip.gen.audio.block_count"
|
||||
@@ -765,6 +766,7 @@ class MODEL_ARCH(IntEnum):
|
||||
NANBEIGE = auto()
|
||||
QWEN3TTS = auto()
|
||||
POCKETTTS = auto()
|
||||
ZONOS3 = auto()
|
||||
K2HORIZON = auto()
|
||||
KOLIBRI1 = auto()
|
||||
LIMITE = auto()
|
||||
@@ -1333,6 +1335,28 @@ class MODEL_TENSOR(IntEnum):
|
||||
A_GEN_CODE_EH_PROJ = auto() # drama3: per-layer [code embd; hidden] -> hidden projection
|
||||
A_GEN_CODE_BLK_EMBD = auto() # drama3: per-layer code embedding table
|
||||
A_GEN_CODE_BLK_OUT_NORM = auto() # drama3: per-layer output norm
|
||||
A_GEN_CODE_TOK_EMBD = auto() # fish_s2: code predictor input embedding, shared by all codebooks
|
||||
# fish_s2 codec encoder (voice cloning): reference clip -> codes
|
||||
A_CODEC_ENC_ENTRY = auto() # entry conv
|
||||
A_CODEC_ENC_SNAKE = auto() # encoder block, Snake before the strided conv
|
||||
A_CODEC_ENC_CONV = auto() # encoder block, strided conv
|
||||
A_CODEC_ENC_RES_ACT1 = auto()
|
||||
A_CODEC_ENC_RES_CONV1 = auto()
|
||||
A_CODEC_ENC_RES_ACT2 = auto()
|
||||
A_CODEC_ENC_RES_CONV2 = auto()
|
||||
A_CODEC_ENC_POST_SNAKE = auto()
|
||||
A_CODEC_ENC_POST_CONV = auto()
|
||||
A_CODEC_DOWN_CONV = auto() # quantizer downsample, strided conv + ConvNeXt
|
||||
A_CODEC_DOWN_DWCONV = auto()
|
||||
A_CODEC_DOWN_NORM = auto()
|
||||
A_CODEC_DOWN_PW1 = auto()
|
||||
A_CODEC_DOWN_PW2 = auto()
|
||||
A_CODEC_DOWN_GAMMA = auto()
|
||||
A_CODEC_PRE_NORM = auto() # output norm of the quantizer pre_module
|
||||
A_CODEC_VQ_FIRST_IN = auto() # semantic VQ input projection
|
||||
A_CODEC_VQ_FIRST_CB = auto() # semantic VQ codebook, l2-normalized
|
||||
A_CODEC_VQ_REST_IN = auto() # residual VQ input projections, merged 3D
|
||||
A_CODEC_VQ_REST_CB = auto() # residual VQ codebooks, l2-normalized, merged 3D
|
||||
# qwen3tts code2wav: RVQ codes -> raw PCM
|
||||
A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D)
|
||||
A_GEN_WAV_QUANT_FIRST_OUT = auto() # semantic RVQ, out_proj
|
||||
@@ -1384,6 +1408,14 @@ class MODEL_TENSOR(IntEnum):
|
||||
A_ENC_SEANET_RES_CONV1 = auto() # residual unit, dilated conv
|
||||
A_ENC_SEANET_RES_CONV2 = auto() # residual unit, pointwise conv
|
||||
A_ENC_SEANET_SCALE_CONV = auto() # strided downsample conv
|
||||
A_ENC_CNX_CONV = auto() # vibevoice: causal conv of a stage (stem, downsample, head), bid = encoder
|
||||
A_ENC_CNX_NORM = auto() # vibevoice: ConvNeXt block, bid = encoder, xid = block
|
||||
A_ENC_CNX_DW = auto()
|
||||
A_ENC_CNX_LS1 = auto()
|
||||
A_ENC_CNX_FFN_NORM = auto()
|
||||
A_ENC_CNX_FFN_UP = auto()
|
||||
A_ENC_CNX_FFN_DOWN = auto()
|
||||
A_ENC_CNX_LS2 = auto()
|
||||
A_ENC_ATTN_SCALE = auto() # layer scale (gamma) on the attn output
|
||||
A_ENC_FFN_SCALE_LS = auto() # layer scale (gamma) on the FFN output
|
||||
A_ENC_SPEAKER_PROJ = auto() # voice latent -> backbone embd
|
||||
@@ -1417,6 +1449,7 @@ class MODEL_TENSOR(IntEnum):
|
||||
A_GEN_WAV_OUTPUT = auto()
|
||||
A_MMPROJ = auto()
|
||||
A_MMPROJ_FC = auto()
|
||||
A_MMPROJ_NORM = auto() # vibevoice: norm after mm.a.mlp.{bid}
|
||||
A_MM_NORM_PRE = auto()
|
||||
A_MM_NORM_MID = auto()
|
||||
A_MM_EMBEDDING = auto() # gemma3n
|
||||
@@ -1703,6 +1736,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
|
||||
MODEL_ARCH.NANBEIGE: "nanbeige",
|
||||
MODEL_ARCH.QWEN3TTS: "qwen3tts",
|
||||
MODEL_ARCH.POCKETTTS: "pockettts",
|
||||
MODEL_ARCH.ZONOS3: "zonos3",
|
||||
MODEL_ARCH.K2HORIZON: "k2-horizon",
|
||||
MODEL_ARCH.KOLIBRI1: "kolibri1",
|
||||
MODEL_ARCH.LIMITE: "limite",
|
||||
@@ -2265,6 +2299,27 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
|
||||
MODEL_TENSOR.A_GEN_CODE_EH_PROJ: "a.gen.code.blk.{bid}.eh_proj",
|
||||
MODEL_TENSOR.A_GEN_CODE_BLK_EMBD: "a.gen.code.blk.{bid}.embd",
|
||||
MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM: "a.gen.code.blk.{bid}.out_norm",
|
||||
MODEL_TENSOR.A_GEN_CODE_TOK_EMBD: "a.gen.code.tok_embd",
|
||||
MODEL_TENSOR.A_CODEC_ENC_ENTRY: "a.codec.enc.entry",
|
||||
MODEL_TENSOR.A_CODEC_ENC_SNAKE: "a.codec.enc.blk.{bid}.snake",
|
||||
MODEL_TENSOR.A_CODEC_ENC_CONV: "a.codec.enc.blk.{bid}.conv",
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_ACT1: "a.codec.enc.blk.{bid}.res.{xid}.act1",
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_CONV1: "a.codec.enc.blk.{bid}.res.{xid}.conv1",
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_ACT2: "a.codec.enc.blk.{bid}.res.{xid}.act2",
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_CONV2: "a.codec.enc.blk.{bid}.res.{xid}.conv2",
|
||||
MODEL_TENSOR.A_CODEC_ENC_POST_SNAKE: "a.codec.enc.post_snake",
|
||||
MODEL_TENSOR.A_CODEC_ENC_POST_CONV: "a.codec.enc.post_conv",
|
||||
MODEL_TENSOR.A_CODEC_DOWN_CONV: "a.codec.down.blk.{bid}.conv",
|
||||
MODEL_TENSOR.A_CODEC_DOWN_DWCONV: "a.codec.down.blk.{bid}.dwconv",
|
||||
MODEL_TENSOR.A_CODEC_DOWN_NORM: "a.codec.down.blk.{bid}.norm",
|
||||
MODEL_TENSOR.A_CODEC_DOWN_PW1: "a.codec.down.blk.{bid}.pw1",
|
||||
MODEL_TENSOR.A_CODEC_DOWN_PW2: "a.codec.down.blk.{bid}.pw2",
|
||||
MODEL_TENSOR.A_CODEC_DOWN_GAMMA: "a.codec.down.blk.{bid}.gamma",
|
||||
MODEL_TENSOR.A_CODEC_PRE_NORM: "a.codec.pre_norm",
|
||||
MODEL_TENSOR.A_CODEC_VQ_FIRST_IN: "a.codec.vq.first.in_proj",
|
||||
MODEL_TENSOR.A_CODEC_VQ_FIRST_CB: "a.codec.vq.first.codebook",
|
||||
MODEL_TENSOR.A_CODEC_VQ_REST_IN: "a.codec.vq.rest.in_proj",
|
||||
MODEL_TENSOR.A_CODEC_VQ_REST_CB: "a.codec.vq.rest.codebook",
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN: "a.gen.wav.quant.first.in_proj",
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj",
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB: "a.gen.wav.quant.first.codebook",
|
||||
@@ -2314,6 +2369,14 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
|
||||
MODEL_TENSOR.A_ENC_SEANET_RES_CONV1: "a.seanet.blk.{bid}.res_conv1",
|
||||
MODEL_TENSOR.A_ENC_SEANET_RES_CONV2: "a.seanet.blk.{bid}.res_conv2",
|
||||
MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV: "a.seanet.blk.{bid}.scale_conv",
|
||||
MODEL_TENSOR.A_ENC_CNX_CONV: "a.cnx.{bid}.conv.{xid}",
|
||||
MODEL_TENSOR.A_ENC_CNX_NORM: "a.cnx.{bid}.blk.{xid}.norm",
|
||||
MODEL_TENSOR.A_ENC_CNX_DW: "a.cnx.{bid}.blk.{xid}.dw",
|
||||
MODEL_TENSOR.A_ENC_CNX_LS1: "a.cnx.{bid}.blk.{xid}.ls1",
|
||||
MODEL_TENSOR.A_ENC_CNX_FFN_NORM: "a.cnx.{bid}.blk.{xid}.ffn_norm",
|
||||
MODEL_TENSOR.A_ENC_CNX_FFN_UP: "a.cnx.{bid}.blk.{xid}.ffn_up",
|
||||
MODEL_TENSOR.A_ENC_CNX_FFN_DOWN: "a.cnx.{bid}.blk.{xid}.ffn_down",
|
||||
MODEL_TENSOR.A_ENC_CNX_LS2: "a.cnx.{bid}.blk.{xid}.ls2",
|
||||
MODEL_TENSOR.A_ENC_ATTN_SCALE: "a.blk.{bid}.ls1",
|
||||
MODEL_TENSOR.A_ENC_FFN_SCALE_LS: "a.blk.{bid}.ls2",
|
||||
MODEL_TENSOR.A_ENC_SPEAKER_PROJ: "a.speaker_proj",
|
||||
@@ -2346,6 +2409,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
|
||||
MODEL_TENSOR.A_GEN_WAV_OUTPUT: "a.gen.wav.output",
|
||||
MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}",
|
||||
MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc",
|
||||
MODEL_TENSOR.A_MMPROJ_NORM: "mm.a.mlp_norm.{bid}",
|
||||
MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre",
|
||||
MODEL_TENSOR.A_MM_NORM_MID: "mm.a.norm_mid",
|
||||
MODEL_TENSOR.A_MM_INP_PROJ: "mm.a.input_projection", # gemma3n
|
||||
@@ -2638,6 +2702,7 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.A_ENC_RVQ_CODEBOOK,
|
||||
MODEL_TENSOR.A_MMPROJ,
|
||||
MODEL_TENSOR.A_MMPROJ_FC,
|
||||
MODEL_TENSOR.A_MMPROJ_NORM,
|
||||
MODEL_TENSOR.A_MM_NORM_PRE,
|
||||
MODEL_TENSOR.A_MM_NORM_MID,
|
||||
MODEL_TENSOR.A_MM_CODE_EMBD,
|
||||
@@ -2689,6 +2754,27 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.A_GEN_CODE_EH_PROJ,
|
||||
MODEL_TENSOR.A_GEN_CODE_BLK_EMBD,
|
||||
MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM,
|
||||
MODEL_TENSOR.A_GEN_CODE_TOK_EMBD,
|
||||
MODEL_TENSOR.A_CODEC_ENC_ENTRY,
|
||||
MODEL_TENSOR.A_CODEC_ENC_SNAKE,
|
||||
MODEL_TENSOR.A_CODEC_ENC_CONV,
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_ACT1,
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_CONV1,
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_ACT2,
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_CONV2,
|
||||
MODEL_TENSOR.A_CODEC_ENC_POST_SNAKE,
|
||||
MODEL_TENSOR.A_CODEC_ENC_POST_CONV,
|
||||
MODEL_TENSOR.A_CODEC_DOWN_CONV,
|
||||
MODEL_TENSOR.A_CODEC_DOWN_DWCONV,
|
||||
MODEL_TENSOR.A_CODEC_DOWN_NORM,
|
||||
MODEL_TENSOR.A_CODEC_DOWN_PW1,
|
||||
MODEL_TENSOR.A_CODEC_DOWN_PW2,
|
||||
MODEL_TENSOR.A_CODEC_DOWN_GAMMA,
|
||||
MODEL_TENSOR.A_CODEC_PRE_NORM,
|
||||
MODEL_TENSOR.A_CODEC_VQ_FIRST_IN,
|
||||
MODEL_TENSOR.A_CODEC_VQ_FIRST_CB,
|
||||
MODEL_TENSOR.A_CODEC_VQ_REST_IN,
|
||||
MODEL_TENSOR.A_CODEC_VQ_REST_CB,
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN,
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT,
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB,
|
||||
@@ -2738,6 +2824,14 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.A_ENC_SEANET_RES_CONV1,
|
||||
MODEL_TENSOR.A_ENC_SEANET_RES_CONV2,
|
||||
MODEL_TENSOR.A_ENC_SEANET_SCALE_CONV,
|
||||
MODEL_TENSOR.A_ENC_CNX_CONV,
|
||||
MODEL_TENSOR.A_ENC_CNX_NORM,
|
||||
MODEL_TENSOR.A_ENC_CNX_DW,
|
||||
MODEL_TENSOR.A_ENC_CNX_LS1,
|
||||
MODEL_TENSOR.A_ENC_CNX_FFN_NORM,
|
||||
MODEL_TENSOR.A_ENC_CNX_FFN_UP,
|
||||
MODEL_TENSOR.A_ENC_CNX_FFN_DOWN,
|
||||
MODEL_TENSOR.A_ENC_CNX_LS2,
|
||||
MODEL_TENSOR.A_ENC_ATTN_SCALE,
|
||||
MODEL_TENSOR.A_ENC_FFN_SCALE_LS,
|
||||
MODEL_TENSOR.A_ENC_SPEAKER_PROJ,
|
||||
@@ -6722,6 +6816,39 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.FFN_DOWN,
|
||||
MODEL_TENSOR.FFN_UP,
|
||||
],
|
||||
MODEL_ARCH.ZONOS3: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
MODEL_TENSOR.OUTPUT_NORM,
|
||||
MODEL_TENSOR.OUTPUT_RES_SCORE,
|
||||
MODEL_TENSOR.ATTN_NORM,
|
||||
MODEL_TENSOR.ATTN_RES_SCORE,
|
||||
MODEL_TENSOR.ATTN_Q,
|
||||
MODEL_TENSOR.ATTN_Q_NORM,
|
||||
MODEL_TENSOR.ATTN_K,
|
||||
MODEL_TENSOR.ATTN_V,
|
||||
MODEL_TENSOR.ATTN_OUT,
|
||||
MODEL_TENSOR.ATTN_GATE,
|
||||
MODEL_TENSOR.ATTN_QKV,
|
||||
MODEL_TENSOR.SSM_A,
|
||||
MODEL_TENSOR.SSM_CONV1D,
|
||||
MODEL_TENSOR.SSM_DT,
|
||||
MODEL_TENSOR.SSM_NORM,
|
||||
MODEL_TENSOR.SSM_BETA,
|
||||
MODEL_TENSOR.SSM_ALPHA,
|
||||
MODEL_TENSOR.SSM_OUT,
|
||||
MODEL_TENSOR.FFN_NORM,
|
||||
MODEL_TENSOR.FFN_RES_SCORE,
|
||||
MODEL_TENSOR.FFN_GATE,
|
||||
MODEL_TENSOR.FFN_DOWN,
|
||||
MODEL_TENSOR.FFN_UP,
|
||||
MODEL_TENSOR.FFN_GATE_INP,
|
||||
MODEL_TENSOR.FFN_EXP_PROBS_B,
|
||||
MODEL_TENSOR.MOE_LATENT_DOWN,
|
||||
MODEL_TENSOR.MOE_LATENT_UP,
|
||||
MODEL_TENSOR.FFN_GATE_EXP,
|
||||
MODEL_TENSOR.FFN_DOWN_EXP,
|
||||
MODEL_TENSOR.FFN_UP_EXP,
|
||||
],
|
||||
MODEL_ARCH.K2HORIZON: [
|
||||
MODEL_TENSOR.TOKEN_EMBD,
|
||||
MODEL_TENSOR.OUTPUT_NORM,
|
||||
@@ -7119,6 +7246,8 @@ class VisionProjectorType:
|
||||
QWEN2A = "qwen2a" # audio
|
||||
QWEN3A = "qwen3a" # audio
|
||||
GLMA = "glma" # audio
|
||||
HIGGS_AUDIO = "higgs_audio" # audio
|
||||
VIBEVOICE_ASR = "vibevoice_asr" # audio
|
||||
QWEN25O = "qwen2.5o" # omni
|
||||
VOXTRAL = "voxtral"
|
||||
VOXTRAL_RT = "voxtral_rt"
|
||||
@@ -7156,6 +7285,11 @@ class VisionProjectorType:
|
||||
CHATTERBOX = "chatterbox" # audio generation: s3gen flow matching + HiFT vocoder
|
||||
SOPRANO = "soprano" # audio generation: Vocos decoder
|
||||
DRAMA3_GEN = "drama3_gen" # audio generation: codebook tower + Fish DAC decoder
|
||||
ZONOS3_GEN = "zonos3_gen" # audio generation: 10 codebook heads + ZAC decoder
|
||||
ZONOS3_SPKENC = "zonos3_spkenc" # audio: MECT-B2 speaker encoder
|
||||
YUE2_GEN = "yue2_gen" # audio generation: flow-matching NAR + Oobleck VAE decoder
|
||||
FISH_S2_GEN = "fish_s2_gen" # audio generation: fast AR code predictor + Fish DAC decoder
|
||||
FISH_S2_SPKENC = "fish_s2_spkenc" # audio: Fish DAC encoder + RVQ, reference clip -> prompt rows
|
||||
HUNYUANVL = "hunyuanvl"
|
||||
PARAKEET = "parakeet" # audio
|
||||
MINIMAXM3 = "minimax_m3"
|
||||
|
||||
@@ -1812,6 +1812,9 @@ class GGUFWriter:
|
||||
def add_gen_audio_model_variant(self, value: str) -> None:
|
||||
self.add_string(Keys.ClipGenAudio.MODEL_VARIANT, value)
|
||||
|
||||
def add_gen_audio_default_voice(self, value: str) -> None:
|
||||
self.add_string(Keys.ClipGenAudio.DEFAULT_VOICE, value)
|
||||
|
||||
def add_xielu_alpha_p(self, values: Sequence[float]):
|
||||
self.add_array(Keys.xIELU.ALPHA_P, values)
|
||||
|
||||
|
||||
@@ -2973,6 +2973,7 @@ class TensorNameMap:
|
||||
|
||||
MODEL_TENSOR.A_ENC_DOWNSAMPLE_CONV: (
|
||||
"encoder.down_sample_layer.0", # mimo-audio-tokenizer
|
||||
"audio_encoder_proj.temporal", # higgs-audio
|
||||
),
|
||||
|
||||
MODEL_TENSOR.A_ENC_DOWNSAMPLE_NORM: (
|
||||
@@ -3026,6 +3027,7 @@ class TensorNameMap:
|
||||
"mlp.model.{bid}", # lfm3a
|
||||
"audio_tower.proj{bid}", # qwen3omni
|
||||
"sound_projection.linear{bid}", # parakeet (linear1, linear2)
|
||||
"audio_encoder_proj.linear{bid}", # higgs-audio (linear1, linear2)
|
||||
),
|
||||
|
||||
MODEL_TENSOR.A_MMPROJ_FC: (
|
||||
|
||||
+15
-9
@@ -271,6 +271,7 @@ extern "C" {
|
||||
// - if not: only the last token is output
|
||||
// )
|
||||
//
|
||||
// [DEPRECATED] use llama_batch_ext instead
|
||||
typedef struct llama_batch {
|
||||
int32_t n_tokens;
|
||||
|
||||
@@ -945,9 +946,10 @@ extern "C" {
|
||||
//
|
||||
// NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
|
||||
//
|
||||
LLAMA_API struct llama_batch llama_batch_get_one(
|
||||
DEPRECATED(LLAMA_API struct llama_batch llama_batch_get_one(
|
||||
llama_token * tokens,
|
||||
int32_t n_tokens);
|
||||
int32_t n_tokens),
|
||||
"use llama_batch_ext_init and llama_batch_ext_add_token instead");
|
||||
|
||||
// Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
|
||||
// Each token can be assigned up to n_seq_max sequence ids
|
||||
@@ -956,13 +958,15 @@ extern "C" {
|
||||
// Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
|
||||
// The rest of the llama_batch members are allocated with size n_tokens
|
||||
// All members are left uninitialized
|
||||
LLAMA_API struct llama_batch llama_batch_init(
|
||||
DEPRECATED(LLAMA_API struct llama_batch llama_batch_init(
|
||||
int32_t n_tokens,
|
||||
int32_t embd,
|
||||
int32_t n_seq_max);
|
||||
int32_t n_seq_max),
|
||||
"use llama_batch_ext_init instead");
|
||||
|
||||
// Frees a batch of tokens allocated with llama_batch_init()
|
||||
LLAMA_API void llama_batch_free(struct llama_batch batch);
|
||||
DEPRECATED(LLAMA_API void llama_batch_free(struct llama_batch batch),
|
||||
"use llama_batch_ext_free instead");
|
||||
|
||||
// Process a batch of tokens.
|
||||
// In contrast to llama_decode() - this call does not use KV cache.
|
||||
@@ -970,9 +974,10 @@ extern "C" {
|
||||
// Can store the encoder output internally for later use by the decoder's cross-attention layers.
|
||||
// 0 - success
|
||||
// < 0 - error. the memory state is restored to the state before this call
|
||||
LLAMA_API int32_t llama_encode(
|
||||
DEPRECATED(LLAMA_API int32_t llama_encode(
|
||||
struct llama_context * ctx,
|
||||
struct llama_batch batch);
|
||||
struct llama_batch batch),
|
||||
"use llama_process with LLAMA_PROCESS_TYPE_ENCODE instead");
|
||||
|
||||
// Process a batch of tokens.
|
||||
// Requires the context to have a memory.
|
||||
@@ -986,9 +991,10 @@ extern "C" {
|
||||
// 2 - aborted (processed ubatches will remain in the context's memory)
|
||||
// -1 - invalid input batch
|
||||
// < -1 - fatal error (processed ubatches will remain in the context's memory)
|
||||
LLAMA_API int32_t llama_decode(
|
||||
DEPRECATED(LLAMA_API int32_t llama_decode(
|
||||
struct llama_context * ctx,
|
||||
struct llama_batch batch);
|
||||
struct llama_batch batch),
|
||||
"use llama_process with LLAMA_PROCESS_TYPE_DECODE instead");
|
||||
|
||||
//
|
||||
// Extended batch API
|
||||
|
||||
@@ -0,0 +1,627 @@
|
||||
{#- ZONOS3 "tts" template: the text to speak (first message), normalized; the helper adds BOS and EOS itself -#}
|
||||
{#- numbers, decimals, money, percents, ordinals, years and clock times are spelled out in en, fr, de, es, it, pt, ja and zh -#}
|
||||
{#- other languages pass through; patterns: tts_norm_scan() / tts_norm_sub(), see docs/tts-norm-patterns.md -#}
|
||||
{#- the ja / zh conventions follow the NeMo grammars of the reference (e.g. zh: 两千, 二百, 三百一十六, 二零零二年) -#}
|
||||
{#- optional identifies this template, so that llama.cpp can detect it and apply quick fixes to GGUFs already converted -#}
|
||||
{%- set template_name = "zonos3-mini" -%}
|
||||
|
||||
{#- English -#}
|
||||
{%- set ones = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "eleven", "twelve",
|
||||
"thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen"] -%}
|
||||
{%- set tens = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"] -%}
|
||||
{%- set scales = ["", " thousand", " million", " billion", " trillion"] -%}
|
||||
{%- set irregular = {"one": "first", "two": "second", "three": "third", "five": "fifth", "eight": "eighth", "nine": "ninth", "twelve": "twelfth"} -%}
|
||||
|
||||
{%- macro under100(n) -%}
|
||||
{{- ones[n] if n < 20 else tens[(n / 10)|int] ~ ("-" ~ ones[n % 10] if n % 10 else "") -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro under1000(n) -%}
|
||||
{{- (ones[(n / 100)|int] ~ " hundred" ~ (" " ~ under100(n % 100) if n % 100 else "")) if n >= 100 else under100(n) -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- all the *_card macros take a digit string of up to 15 digits -#}
|
||||
{%- macro en_card(s) -%}
|
||||
{%- set p = ["", "00", "0"][(s|length) % 3] ~ s -%}
|
||||
{%- set ns = namespace(out="", ng=((p|length) / 3)|int) -%}
|
||||
{%- for g in range(ns.ng) -%}
|
||||
{%- set v = p[g * 3:g * 3 + 3]|int -%}
|
||||
{%- if v > 0 -%}
|
||||
{%- set ns.out = ns.out ~ (" " if ns.out else "") ~ under1000(v) ~ scales[ns.ng - 1 - g] -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- ns.out if ns.out else "zero" -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- the last word of a cardinal becomes ordinal: twenty-one -> twenty-first -#}
|
||||
{%- macro en_ord(w) -%}
|
||||
{%- set last = (w.split(" ")|last).split("-")|last -%}
|
||||
{%- set head = w[0:(w|length) - (last|length)] -%}
|
||||
{{- head ~ (irregular[last] if last in irregular else (last[0:(last|length) - 1] ~ "ieth" if last.endswith("y") else last ~ "th")) -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- French: traditional spelling, quatre-vingts / deux cents take the plural s only at the end -#}
|
||||
{%- set fr_u = ["zéro", "un", "deux", "trois", "quatre", "cinq", "six", "sept", "huit", "neuf", "dix", "onze", "douze", "treize",
|
||||
"quatorze", "quinze", "seize"] -%}
|
||||
{%- set fr_t = ["", "", "vingt", "trente", "quarante", "cinquante", "soixante", "", "quatre-vingt", ""] -%}
|
||||
{%- set fr_big = [[], [], ["million", "millions"], ["milliard", "milliards"], ["billion", "billions"]] -%}
|
||||
|
||||
{%- macro fr99(n, final) -%}
|
||||
{%- if n < 17 -%}
|
||||
{{- fr_u[n] -}}
|
||||
{%- elif n < 20 -%}
|
||||
{{- "dix-" ~ fr_u[n - 10] -}}
|
||||
{%- else -%}
|
||||
{%- set t = (n / 10)|int -%}
|
||||
{%- set u = n % 10 -%}
|
||||
{%- if t == 7 or t == 9 -%}
|
||||
{{- ("soixante" if t == 7 else "quatre-vingt") ~ (" et onze" if t == 7 and u == 1 else "-" ~ fr99(n - (60 if t == 7 else 80), final)) -}}
|
||||
{%- else -%}
|
||||
{{- fr_t[t] ~ ("s" if t == 8 and u == 0 and final else "") ~ (" et un" if u == 1 and t != 8 else ("-" ~ fr_u[u] if u else "")) -}}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro fr999(n, final) -%}
|
||||
{%- set h = (n / 100)|int -%}
|
||||
{%- set r = n % 100 -%}
|
||||
{%- set hs = "" if h == 0 else ("cent" if h == 1 else fr_u[h] ~ " cent" ~ ("s" if r == 0 and final else "")) -%}
|
||||
{{- hs ~ (" " if hs and r else "") ~ (fr99(r, final) if r else "") -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro fr_card(s) -%}
|
||||
{%- set p = ["", "00", "0"][(s|length) % 3] ~ s -%}
|
||||
{%- set ns = namespace(out="", ng=((p|length) / 3)|int) -%}
|
||||
{%- for g in range(ns.ng) -%}
|
||||
{%- set v = p[g * 3:g * 3 + 3]|int -%}
|
||||
{%- set k = ns.ng - 1 - g -%}
|
||||
{%- if v > 0 -%}
|
||||
{%- if k == 0 -%}
|
||||
{%- set w = fr999(v, true) -%}
|
||||
{%- elif k == 1 -%}
|
||||
{%- set w = ("" if v == 1 else fr999(v, false) ~ " ") ~ "mille" -%}
|
||||
{%- else -%}
|
||||
{%- set w = fr999(v, true) ~ " " ~ fr_big[k][0 if v == 1 else 1] -%}
|
||||
{%- endif -%}
|
||||
{%- set ns.out = ns.out ~ (" " if ns.out else "") ~ w -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- ns.out if ns.out else "zéro" -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro fr_ord(w) -%}
|
||||
{%- set last = (w.split(" ")|last).split("-")|last -%}
|
||||
{%- set head = w[0:(w|length) - (last|length)] -%}
|
||||
{%- set base = last[0:(last|length) - 1] if last in ["vingts", "cents", "millions", "milliards", "billions"] else last -%}
|
||||
{%- set map = {"cinq": "cinquième", "neuf": "neuvième"} -%}
|
||||
{{- head ~ (map[base] if base in map else (base[0:(base|length) - 1] ~ "ième" if base.endswith("e") else base ~ "ième")) -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- German: below a million one word, eins / ein / eine -#}
|
||||
{%- set de_u = ["null", "eins", "zwei", "drei", "vier", "fünf", "sechs", "sieben", "acht", "neun", "zehn", "elf", "zwölf",
|
||||
"dreizehn", "vierzehn", "fünfzehn", "sechzehn", "siebzehn", "achtzehn", "neunzehn"] -%}
|
||||
{%- set de_t = ["", "", "zwanzig", "dreißig", "vierzig", "fünfzig", "sechzig", "siebzig", "achtzig", "neunzig"] -%}
|
||||
{%- set de_big = [[], [], ["Million", "Millionen"], ["Milliarde", "Milliarden"], ["Billion", "Billionen"]] -%}
|
||||
|
||||
{%- macro de99(n) -%}
|
||||
{{- de_u[n] if n < 20 else ((("ein" if n % 10 == 1 else de_u[n % 10]) ~ "und") if n % 10 else "") ~ de_t[(n / 10)|int] -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro de999(n) -%}
|
||||
{%- set h = (n / 100)|int -%}
|
||||
{%- set r = n % 100 -%}
|
||||
{{- (((("ein" if h == 1 else de_u[h]) ~ "hundert")) if h else "") ~ (de99(r) if r else "") -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- attributive form: eins -> ein -#}
|
||||
{%- macro de_attr(w) -%}
|
||||
{{- w[0:(w|length) - 1] if w.endswith("eins") else w -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro de_card(s) -%}
|
||||
{%- set p = ["", "00", "0"][(s|length) % 3] ~ s -%}
|
||||
{%- set ns = namespace(big="", small="", ng=((p|length) / 3)|int) -%}
|
||||
{%- for g in range(ns.ng) -%}
|
||||
{%- set v = p[g * 3:g * 3 + 3]|int -%}
|
||||
{%- set k = ns.ng - 1 - g -%}
|
||||
{%- if v > 0 -%}
|
||||
{%- if k == 0 -%}
|
||||
{%- set ns.small = ns.small ~ de999(v) -%}
|
||||
{%- elif k == 1 -%}
|
||||
{%- set ns.small = ns.small ~ de_attr(de999(v)) ~ "tausend" -%}
|
||||
{%- else -%}
|
||||
{%- set ns.big = ns.big ~ (" " if ns.big else "") ~ ("eine" if v == 1 else de_attr(de999(v))) ~ " " ~ de_big[k][0 if v == 1 else 1] -%}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- set out = ns.big ~ (" " if ns.big and ns.small else "") ~ ns.small -%}
|
||||
{{- out if out else "null" -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- join the non-empty parts with a space -#}
|
||||
{%- macro join4(a, b, c, d) -%}
|
||||
{%- set ns = namespace(out="") -%}
|
||||
{%- for x in [a, b, c, d] -%}
|
||||
{%- if x -%}
|
||||
{%- set ns.out = ns.out ~ (" " if ns.out else "") ~ x -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- ns.out -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- Spanish: long scale (mil millones, billón), un / veintiún before a noun or mil -#}
|
||||
{%- set es_u = ["cero", "uno", "dos", "tres", "cuatro", "cinco", "seis", "siete", "ocho", "nueve", "diez", "once", "doce",
|
||||
"trece", "catorce", "quince", "dieciséis", "diecisiete", "dieciocho", "diecinueve", "veinte", "veintiuno",
|
||||
"veintidós", "veintitrés", "veinticuatro", "veinticinco", "veintiséis", "veintisiete", "veintiocho", "veintinueve"] -%}
|
||||
{%- set es_t = ["", "", "", "treinta", "cuarenta", "cincuenta", "sesenta", "setenta", "ochenta", "noventa"] -%}
|
||||
{%- set es_h = ["", "ciento", "doscientos", "trescientos", "cuatrocientos", "quinientos", "seiscientos", "setecientos",
|
||||
"ochocientos", "novecientos"] -%}
|
||||
|
||||
{%- macro es99(n) -%}
|
||||
{{- es_u[n] if n < 30 else es_t[(n / 10)|int] ~ (" y " ~ es_u[n % 10] if n % 10 else "") -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro es999(n) -%}
|
||||
{%- if n == 100 -%}
|
||||
{{- "cien" -}}
|
||||
{%- else -%}
|
||||
{%- set h = (n / 100)|int -%}
|
||||
{%- set r = n % 100 -%}
|
||||
{{- es_h[h] ~ (" " if h and r else "") ~ (es99(r) if r else "") -}}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro es_apoc(w) -%}
|
||||
{{- (w[0:(w|length) - 9] ~ "veintiún") if w.endswith("veintiuno") else ((w[0:(w|length) - 3] ~ "un") if w.endswith("uno") else w) -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro es6(n) -%}
|
||||
{%- set a = (n / 1000)|int -%}
|
||||
{%- set b = n % 1000 -%}
|
||||
{%- set hi = "" if a == 0 else ("mil" if a == 1 else es_apoc(es999(a)) ~ " mil") -%}
|
||||
{{- hi ~ (" " if hi and b else "") ~ (es999(b) if b else "") -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro es_card(s) -%}
|
||||
{%- set p = ("000000000000000000" ~ s)[-18:] -%}
|
||||
{%- set top = p[0:6]|int -%}
|
||||
{%- set mid = p[6:12]|int -%}
|
||||
{%- set low = p[12:18]|int -%}
|
||||
{%- set a = "" if top == 0 else ("un billón" if top == 1 else es_apoc(es6(top)) ~ " billones") -%}
|
||||
{%- set b = "" if mid == 0 else ("un millón" if mid == 1 else es_apoc(es6(mid)) ~ " millones") -%}
|
||||
{%- set out = join4(a, b, es6(low) if low else "", "") -%}
|
||||
{{- out if out else "cero" -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- Italian: below a million one word, ventitré, centottanta -#}
|
||||
{%- set it_u = ["zero", "uno", "due", "tre", "quattro", "cinque", "sei", "sette", "otto", "nove", "dieci", "undici", "dodici",
|
||||
"tredici", "quattordici", "quindici", "sedici", "diciassette", "diciotto", "diciannove"] -%}
|
||||
{%- set it_t = ["", "", "venti", "trenta", "quaranta", "cinquanta", "sessanta", "settanta", "ottanta", "novanta"] -%}
|
||||
|
||||
{%- macro it99(n) -%}
|
||||
{%- if n < 20 -%}
|
||||
{{- it_u[n] -}}
|
||||
{%- else -%}
|
||||
{%- set t = it_t[(n / 10)|int] -%}
|
||||
{%- set u = n % 10 -%}
|
||||
{{- (t[0:(t|length) - 1] if u == 1 or u == 8 else t) ~ (it_u[u] if u else "") -}}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro it999(n) -%}
|
||||
{%- set h = (n / 100)|int -%}
|
||||
{%- set r = n % 100 -%}
|
||||
{%- set hs = "" if h == 0 else ("cento" if h == 1 else it_u[h] ~ "cento") -%}
|
||||
{{- (hs[0:(hs|length) - 1] if (r / 10)|int == 8 else hs) ~ (it99(r) if r else "") -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- without the final accent, see it_acc -#}
|
||||
{%- macro it6(n) -%}
|
||||
{%- set a = (n / 1000)|int -%}
|
||||
{%- set b = n % 1000 -%}
|
||||
{{- ("" if a == 0 else ("mille" if a == 1 else it999(a) ~ "mila")) ~ (it999(b) if b else "") -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro it_acc(w) -%}
|
||||
{{- (w[0:(w|length) - 3] ~ "tré") if w.endswith("tre") and w != "tre" else w -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro it_card(s) -%}
|
||||
{%- set p = ("000000000000000000" ~ s)[-18:] -%}
|
||||
{%- set bil = p[0:6]|int -%}
|
||||
{%- set mld = p[6:9]|int -%}
|
||||
{%- set mil = p[9:12]|int -%}
|
||||
{%- set low = p[12:18]|int -%}
|
||||
{%- set a = "" if bil == 0 else ("un bilione" if bil == 1 else it_acc(it6(bil)) ~ " bilioni") -%}
|
||||
{%- set b = "" if mld == 0 else ("un miliardo" if mld == 1 else it_acc(it999(mld)) ~ " miliardi") -%}
|
||||
{%- set c = "" if mil == 0 else ("un milione" if mil == 1 else it_acc(it999(mil)) ~ " milioni") -%}
|
||||
{%- set out = join4(a, b, c, it_acc(it6(low)) if low else "") -%}
|
||||
{{- out if out else "zero" -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- Portuguese (Brazil): short scale, "e" before the last group when it is below 100 or a round hundred -#}
|
||||
{%- set pt_u = ["zero", "um", "dois", "três", "quatro", "cinco", "seis", "sete", "oito", "nove", "dez", "onze", "doze",
|
||||
"treze", "catorze", "quinze", "dezesseis", "dezessete", "dezoito", "dezenove"] -%}
|
||||
{%- set pt_t = ["", "", "vinte", "trinta", "quarenta", "cinquenta", "sessenta", "setenta", "oitenta", "noventa"] -%}
|
||||
{%- set pt_h = ["", "cento", "duzentos", "trezentos", "quatrocentos", "quinhentos", "seiscentos", "setecentos", "oitocentos",
|
||||
"novecentos"] -%}
|
||||
{%- set pt_big = [[], [], ["milhão", "milhões"], ["bilhão", "bilhões"], ["trilhão", "trilhões"]] -%}
|
||||
|
||||
{%- macro pt99(n) -%}
|
||||
{{- pt_u[n] if n < 20 else pt_t[(n / 10)|int] ~ (" e " ~ pt_u[n % 10] if n % 10 else "") -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro pt999(n) -%}
|
||||
{%- if n == 100 -%}
|
||||
{{- "cem" -}}
|
||||
{%- else -%}
|
||||
{%- set h = (n / 100)|int -%}
|
||||
{%- set r = n % 100 -%}
|
||||
{{- pt_h[h] ~ (" e " if h and r else "") ~ (pt99(r) if r else "") -}}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro pt_card(s) -%}
|
||||
{%- set p = ["", "00", "0"][(s|length) % 3] ~ s -%}
|
||||
{%- set ns = namespace(out="", ng=((p|length) / 3)|int, last=-1) -%}
|
||||
{%- for g in range(ns.ng) -%}
|
||||
{%- if p[g * 3:g * 3 + 3]|int > 0 -%}
|
||||
{%- set ns.last = g -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- for g in range(ns.ng) -%}
|
||||
{%- set v = p[g * 3:g * 3 + 3]|int -%}
|
||||
{%- set k = ns.ng - 1 - g -%}
|
||||
{%- if v > 0 -%}
|
||||
{%- if k == 0 -%}
|
||||
{%- set w = pt999(v) -%}
|
||||
{%- elif k == 1 -%}
|
||||
{%- set w = "mil" if v == 1 else pt999(v) ~ " mil" -%}
|
||||
{%- else -%}
|
||||
{%- set w = ("um" if v == 1 else pt999(v)) ~ " " ~ pt_big[k][0 if v == 1 else 1] -%}
|
||||
{%- endif -%}
|
||||
{%- set sep = " e " if g == ns.last and (v < 100 or v % 100 == 0) else " " -%}
|
||||
{%- set ns.out = ns.out ~ (sep if ns.out else "") ~ w -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- ns.out if ns.out else "zero" -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- Japanese: 万 / 億 / 兆 groups, no 一 before 十 / 百 / 千 (but 一千万), zero digits are skipped -#}
|
||||
{%- set cjk_d = ["零", "一", "二", "三", "四", "五", "六", "七", "八", "九"] -%}
|
||||
|
||||
{%- macro ja4(n, big) -%}
|
||||
{%- set th = (n / 1000)|int -%}
|
||||
{%- set h = ((n / 100)|int) % 10 -%}
|
||||
{%- set t = ((n / 10)|int) % 10 -%}
|
||||
{%- set u = n % 10 -%}
|
||||
{{- ((("一" if big else "") if th == 1 else cjk_d[th]) ~ "千" if th else "")
|
||||
~ (("" if h == 1 else cjk_d[h]) ~ "百" if h else "")
|
||||
~ (("" if t == 1 else cjk_d[t]) ~ "十" if t else "")
|
||||
~ (cjk_d[u] if u else "") -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro ja_card(s) -%}
|
||||
{%- set p = ("0000000000000000" ~ s)[-16:] -%}
|
||||
{%- set ns = namespace(out="") -%}
|
||||
{%- for g in range(4) -%}
|
||||
{%- set v = p[g * 4:g * 4 + 4]|int -%}
|
||||
{%- if v > 0 -%}
|
||||
{%- set ns.out = ns.out ~ ja4(v, g < 3) ~ ["兆", "億", "万", ""][g] -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- ns.out if ns.out else "零" -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- Chinese: 万 / 亿 groups, 两 before 千 / 万 / 亿, 二 before 百, 十 at the head but 一十 inside, one 零 per gap -#}
|
||||
{%- macro zh4(n, head) -%}
|
||||
{%- set ds = [(n / 1000)|int, ((n / 100)|int) % 10, ((n / 10)|int) % 10, n % 10] -%}
|
||||
{%- set ns = namespace(out="", zero=false) -%}
|
||||
{%- for i in range(4) -%}
|
||||
{%- set d = ds[i] -%}
|
||||
{%- if d == 0 -%}
|
||||
{%- if ns.out -%}
|
||||
{%- set ns.zero = true -%}
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
{%- set w = "两" if d == 2 and i == 0 else cjk_d[d] -%}
|
||||
{%- if i == 2 and d == 1 and head and not ns.out -%}
|
||||
{%- set w = "" -%}
|
||||
{%- endif -%}
|
||||
{%- set ns.out = ns.out ~ ("零" if ns.zero else "") ~ w ~ ["千", "百", "十", ""][i] -%}
|
||||
{%- set ns.zero = false -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- ns.out -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro zh_card(s) -%}
|
||||
{%- set p = ("0000000000000000" ~ s)[-16:] -%}
|
||||
{%- set ns = namespace(out="", gap=false) -%}
|
||||
{%- for g in range(4) -%}
|
||||
{%- set v = p[g * 4:g * 4 + 4]|int -%}
|
||||
{%- if v == 0 -%}
|
||||
{%- if ns.out -%}
|
||||
{%- set ns.gap = true -%}
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
{%- set w = "两" if v == 2 and g < 3 else zh4(v, not ns.out) -%}
|
||||
{%- set ns.out = ns.out ~ ("零" if ns.out and (ns.gap or v < 1000) else "") ~ w ~ ["万亿", "亿", "万", ""][g] -%}
|
||||
{%- set ns.gap = false -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- ns.out if ns.out else "零" -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- per-language words -#}
|
||||
{%- set digit_words = {"en": ones[0:10], "fr": fr_u[0:10], "de": de_u[0:10], "es": es_u[0:10], "it": it_u[0:10], "pt": pt_u[0:10],
|
||||
"ja": cjk_d, "zh": cjk_d} -%}
|
||||
{%- set W = {
|
||||
"en": {"point": " point ", "minus": "minus ", "and": " and ", "percent": " percent", "sep": " "},
|
||||
"fr": {"point": " virgule ", "minus": "moins ", "and": " et ", "percent": " pour cent", "sep": " "},
|
||||
"de": {"point": " Komma ", "minus": "minus ", "and": " und ", "percent": " Prozent", "sep": " "},
|
||||
"es": {"point": " coma ", "minus": "menos ", "and": " con ", "percent": " por ciento", "sep": " "},
|
||||
"it": {"point": " virgola ", "minus": "meno ", "and": " e ", "percent": " per cento", "sep": " "},
|
||||
"pt": {"point": " vírgula ", "minus": "menos ", "and": " e ", "percent": " por cento", "sep": " "},
|
||||
"ja": {"point": "点", "minus": "マイナス", "and": "", "percent": "パーセント", "sep": ""},
|
||||
"zh": {"point": "点", "minus": "负", "and": "", "percent": "", "sep": ""},
|
||||
} -%}
|
||||
{#- currency -> [singular, plural, cent singular, cent plural]; ja / zh: the unit after the number -#}
|
||||
{%- set M = {
|
||||
"en": {"$": ["dollar", "dollars", "cent", "cents"], "€": ["euro", "euros", "cent", "cents"],
|
||||
"£": ["pound", "pounds", "penny", "pence"], "¥": ["yen", "yen", "sen", "sen"]},
|
||||
"fr": {"$": ["dollar", "dollars", "cent", "cents"], "€": ["euro", "euros", "centime", "centimes"]},
|
||||
"de": {"$": ["Dollar", "Dollar", "Cent", "Cent"], "€": ["Euro", "Euro", "Cent", "Cent"], "£": ["Pfund", "Pfund", "Penny", "Pence"]},
|
||||
"es": {"$": ["dólar", "dólares", "centavo", "centavos"], "€": ["euro", "euros", "céntimo", "céntimos"]},
|
||||
"it": {"$": ["dollaro", "dollari", "centesimo", "centesimi"], "€": ["euro", "euro", "centesimo", "centesimi"]},
|
||||
"pt": {"$": ["dólar", "dólares", "centavo", "centavos"], "€": ["euro", "euros", "cêntimo", "cêntimos"],
|
||||
"R$": ["real", "reais", "centavo", "centavos"]},
|
||||
"ja": {"$": "ドル", "€": "ユーロ", "£": "ポンド", "¥": "円"},
|
||||
"zh": {"$": "美元", "€": "欧元", "£": "英镑", "¥": "元"},
|
||||
} -%}
|
||||
{%- set es_ord = ["", "primero", "segundo", "tercero", "cuarto", "quinto", "sexto", "séptimo", "octavo", "noveno", "décimo"] -%}
|
||||
{%- set it_ord = ["", "primo", "secondo", "terzo", "quarto", "quinto", "sesto", "settimo", "ottavo", "nono", "decimo"] -%}
|
||||
{%- set pt_ord = ["", "primeiro", "segundo", "terceiro", "quarto", "quinto", "sexto", "sétimo", "oitavo", "nono", "décimo"] -%}
|
||||
|
||||
{%- macro card(L, s) -%}
|
||||
{%- if L == "fr" -%}{{- fr_card(s) -}}
|
||||
{%- elif L == "de" -%}{{- de_card(s) -}}
|
||||
{%- elif L == "es" -%}{{- es_card(s) -}}
|
||||
{%- elif L == "it" -%}{{- it_card(s) -}}
|
||||
{%- elif L == "pt" -%}{{- pt_card(s) -}}
|
||||
{%- elif L == "ja" -%}{{- ja_card(s) -}}
|
||||
{%- elif L == "zh" -%}{{- zh_card(s) -}}
|
||||
{%- else -%}{{- en_card(s) -}}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- attributive form before a noun: ein Euro, un euro, veintiún euros -#}
|
||||
{%- macro attr(L, w) -%}
|
||||
{%- if L == "de" -%}{{- de_attr(w) -}}
|
||||
{%- elif L == "es" -%}{{- es_apoc(w) -}}
|
||||
{%- elif L == "it" and w == "uno" -%}{{- "un" -}}
|
||||
{%- else -%}{{- w -}}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro spell(L, s) -%}
|
||||
{%- set ns = namespace(out="") -%}
|
||||
{%- for i in range(s|length) -%}
|
||||
{%- set ns.out = ns.out ~ (W[L]["sep"] if i > 0 else "") ~ digit_words[L][s[i:i + 1]|int] -%}
|
||||
{%- endfor -%}
|
||||
{{- ns.out -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- decimals: fr / es / it / pt read a short fraction as a number (virgule quatorze), the others digit by digit -#}
|
||||
{%- macro fraction(L, f) -%}
|
||||
{%- if L in ["fr", "es", "it", "pt"] and f|length <= 3 -%}
|
||||
{%- set z = tts_norm_sub(f, "^(0*).*", "%1") -%}
|
||||
{%- set rest = f[z|length:] -%}
|
||||
{{- spell(L, z) ~ (" " if z and rest else "") ~ (card(L, rest) if rest else "") -}}
|
||||
{%- else -%}
|
||||
{{- spell(L, f) -}}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- en: 1990 -> nineteen ninety, 1905 -> nineteen oh five, 2005 -> two thousand five; de: 1990 -> neunzehnhundertneunzig -#}
|
||||
{%- macro year(L, s) -%}
|
||||
{%- set hi = s[0:2]|int -%}
|
||||
{%- set lo = s[2:4]|int -%}
|
||||
{%- if L == "de" -%}
|
||||
{{- (de99(hi) ~ "hundert" ~ (de99(lo) if lo else "")) if hi < 20 else de_card(s) -}}
|
||||
{%- elif (hi == 20 and lo < 10) or (hi % 10 == 0 and lo == 0) -%}
|
||||
{{- en_card(s) -}}
|
||||
{%- elif lo == 0 -%}
|
||||
{{- under100(hi) ~ " hundred" -}}
|
||||
{%- else -%}
|
||||
{{- under100(hi) ~ (" oh " ~ ones[lo] if lo < 10 else " " ~ under100(lo)) -}}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- ip: digits with optional thousands separators, frac: decimal digits or empty -#}
|
||||
{%- macro number(L, ip, frac, year_ok) -%}
|
||||
{%- set d = tts_norm_sub(ip, "%D", "") -%}
|
||||
{%- if d|length > 15 or (d|length > 1 and d[0:1] == "0" and not frac) -%}
|
||||
{{- spell(L, d) ~ (W[L]["point"] ~ spell(L, frac) if frac else "") -}}
|
||||
{%- elif frac -%}
|
||||
{{- card(L, d) ~ W[L]["point"] ~ fraction(L, frac) -}}
|
||||
{%- elif year_ok and d == ip and d|length == 4 and d|int >= 1100 and d|int <= 2099 and L in ["en", "de"] -%}
|
||||
{{- year(L, d) -}}
|
||||
{%- else -%}
|
||||
{{- card(L, d) -}}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{#- fr / es / it / pt: million and above are nouns, a round amount takes "de" before the currency (deux millions d'euros) -#}
|
||||
{%- macro unit_sep(L, ip, unit) -%}
|
||||
{%- set d = tts_norm_sub(ip, "%D", "") -%}
|
||||
{%- if L in ["fr", "es", "it", "pt"] and d|length > 6 and d[(d|length) - 6:] == "000000" -%}
|
||||
{{- " d'" if L == "fr" and unit[0:1] in "aeiou" else " " ~ ("di" if L == "it" else "de") ~ " " -}}
|
||||
{%- else -%}
|
||||
{{- " " -}}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro money(L, cur, ip, cents, scale) -%}
|
||||
{%- if cur not in M[L] -%}
|
||||
{{- cur ~ W[L]["sep"] ~ number(L, ip, cents, false) -}}
|
||||
{%- elif L in ["ja", "zh"] -%}
|
||||
{{- number(L, ip, cents, false) ~ M[L][cur] -}}
|
||||
{%- else -%}
|
||||
{%- set u = M[L][cur] -%}
|
||||
{%- if scale -%}
|
||||
{{- number(L, ip, cents, false) ~ " " ~ scale ~ " " ~ u[1] -}}
|
||||
{%- elif cents|length == 2 -%}
|
||||
{%- set c = attr(L, card(L, cents)) ~ " " ~ u[2 if cents|int == 1 else 3] -%}
|
||||
{%- if tts_norm_sub(ip, "%D", "")|int == 0 -%}
|
||||
{{- c -}}
|
||||
{%- else -%}
|
||||
{{- attr(L, number(L, ip, "", false)) ~ unit_sep(L, ip, u[1]) ~ u[0 if ip == "1" else 1] ~ (W[L]["and"] ~ c if cents|int > 0 else "") -}}
|
||||
{%- endif -%}
|
||||
{%- elif cents -%}
|
||||
{{- number(L, ip, cents, false) ~ " " ~ u[1] -}}
|
||||
{%- else -%}
|
||||
{{- attr(L, number(L, ip, "", false)) ~ unit_sep(L, ip, u[1]) ~ u[0 if ip == "1" else 1] -}}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro clock(L, h, m) -%}
|
||||
{%- set hn = h|int -%}
|
||||
{%- set mm = m|int if m else 0 -%}
|
||||
{%- if L == "fr" -%}
|
||||
{%- set w = fr_card(h) -%}
|
||||
{{- w ~ ("e" if w.endswith("un") else "") ~ (" heure" if hn <= 1 else " heures") ~ (" " ~ fr_card(m) if mm else "") -}}
|
||||
{%- elif L == "de" -%}
|
||||
{{- de_attr(de_card(h)) ~ " Uhr" ~ (" " ~ de_card(m) if mm else "") -}}
|
||||
{%- elif L == "es" -%}
|
||||
{%- set w = es_card(h) -%}
|
||||
{{- (w[0:(w|length) - 1] ~ "a" if w.endswith("uno") else w) ~ (" y " ~ es_card(m) if mm else " en punto") -}}
|
||||
{%- elif L == "it" -%}
|
||||
{{- ("una" if hn == 1 else it_card(h)) ~ (" e " ~ it_card(m) if mm else "") -}}
|
||||
{%- elif L == "pt" -%}
|
||||
{%- set w = pt_card(h) -%}
|
||||
{%- set w = w ~ "a" if w.endswith("um") else (w[0:(w|length) - 4] ~ "duas" if w.endswith("dois") else w) -%}
|
||||
{{- w ~ (" e " ~ pt_card(m) if mm else (" hora" if hn == 1 else " horas")) -}}
|
||||
{%- elif L == "ja" -%}
|
||||
{{- ja_card(h) ~ "時" ~ (ja_card(m) ~ "分" if mm else "") -}}
|
||||
{%- elif L == "zh" -%}
|
||||
{{- ("两" if hn == 2 else zh_card(h)) ~ "点" ~ (zh_card(m) ~ "分" if mm else "") -}}
|
||||
{%- else -%}
|
||||
{{- en_card(h) ~ (" o'clock" if mm == 0 else (" oh " ~ ones[mm] if mm < 10 else " " ~ under100(mm))) -}}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- macro ordinal(L, d, suf) -%}
|
||||
{%- set n = d|int -%}
|
||||
{%- if L == "fr" -%}
|
||||
{%- set plural = "s" if suf.endswith("s") else "" -%}
|
||||
{%- if n == 1 -%}
|
||||
{{- ("première" if suf.startswith("r") or suf.startswith("è") else "premier") ~ plural -}}
|
||||
{%- elif n == 2 and suf.startswith("nd") -%}
|
||||
{{- ("seconde" if suf.startswith("nde") else "second") ~ plural -}}
|
||||
{%- else -%}
|
||||
{{- fr_ord(fr_card(d)) ~ plural -}}
|
||||
{%- endif -%}
|
||||
{%- elif L in ["es", "it", "pt"] -%}
|
||||
{%- set fem = suf == "ª" -%}
|
||||
{%- if n >= 1 and n <= 10 -%}
|
||||
{%- set w = {"es": es_ord, "it": it_ord, "pt": pt_ord}[L][n] -%}
|
||||
{{- w[0:(w|length) - 1] ~ "a" if fem else w -}}
|
||||
{%- elif L == "it" and n < 1000000 -%}
|
||||
{#- undicesimo, ventitreesimo, ventiseiesimo -#}
|
||||
{%- set w = it6(n) -%}
|
||||
{%- set w = w ~ "esimo" if w.endswith("tre") or w.endswith("sei") else w[0:(w|length) - 1] ~ "esimo" -%}
|
||||
{{- w[0:(w|length) - 1] ~ "a" if fem else w -}}
|
||||
{%- else -%}
|
||||
{{- card(L, d) -}}
|
||||
{%- endif -%}
|
||||
{%- else -%}
|
||||
{{- en_ord(en_card(d)) -}}
|
||||
{%- endif -%}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- set l = ((language | lower) if language is defined and language else "").replace("-", "_") -%}
|
||||
{%- set L = "en" if l == "" else ("zh" if l.startswith("zh") or l.startswith("cmn") else l.split("_")[0]) -%}
|
||||
{%- set text = messages[0]["content"] -%}
|
||||
{%- if L not in M -%}
|
||||
{{- text -}}
|
||||
{%- else -%}
|
||||
{%- set t = text -%}
|
||||
{%- if L not in ["ja", "zh"] -%}
|
||||
{#- a space between a number and the letters it touches (5km, mp3), except for ordinal suffixes and the French 15h30 -#}
|
||||
{%- set suffixes = {"en": "st|nd|rd|th|s", "fr": "ers?|res?|ères?|nde?s?|èmes?|es?|h", "es": "º|ª", "it": "º|ª", "pt": "º|ª"} -%}
|
||||
{%- if L in suffixes -%}
|
||||
{%- set t = tts_norm_sub(t, "(%d)(?!(?:" ~ suffixes[L] ~ ")(?!%a))(%a)", "%1 %2") -%}
|
||||
{%- else -%}
|
||||
{%- set t = tts_norm_sub(t, "(%d)(%a)", "%1 %2") -%}
|
||||
{%- endif -%}
|
||||
{%- set t = tts_norm_sub(t, "(?<!%d)(%a)(%d)", "%1 %2") -%}
|
||||
{%- endif -%}
|
||||
{#- integer part with thousands separators (groups of exactly three digits), decimal part not followed by another group -#}
|
||||
{%- set tsep = {"fr": "[%s.]", "de": "%.", "es": "%.", "it": "%.", "pt": "%."}.get(L, ",") -%}
|
||||
{%- set int = "(%d{1,3}(?:" ~ tsep ~ "%d{3})+(?!%d)|%d+)" -%}
|
||||
{%- set frac = ("(?:%.(%d+)(?![,.]%d))?" if L in ["en", "ja", "zh"] else "(?:[,.](%d+)(?![,.]%d))?") -%}
|
||||
{#- a minus sign at the start of the text, after a space or after an opening parenthesis; ja / zh: after anything but a digit -#}
|
||||
{%- set neg = "(?:(?<!%d)(-))?" if L in ["ja", "zh"] else "(?:(?<![^%s(])(-))?" -%}
|
||||
{%- set cur = "(R%$|[$€£¥])" -%}
|
||||
{#- the matcher never backtracks into a choice, so the longest hour forms come first -#}
|
||||
{%- set hour = "(?<!%d)(2[0-3]|[01]%d|%d)" -%}
|
||||
{%- set time = hour ~ "(?:h([0-5]%d)?|:([0-5]%d))(?![%d%a])" if L == "fr"
|
||||
else hour ~ ":([0-5]%d)(?!%d)" ~ ("(?: Uhr%b)?" if L == "de" else "") -%}
|
||||
{%- set ords = {
|
||||
"en": [["ordinal", "(?<![%d,.])(%d{1,15})(st|nd|rd|th)(?!%a)"]],
|
||||
"fr": [["ordinal", "(?<![%d,.])(%d{1,15})(ers?|res?|ères?|nde?s?|èmes?|es?)(?!%a)"]],
|
||||
"es": [["ordinal", "(?<![%d,.])(%d{1,15})%.?([ºª])"]],
|
||||
"it": [["ordinal", "(?<![%d,.])(%d{1,15})%.?([ºª])"]],
|
||||
"pt": [["ordinal", "(?<![%d,.])(%d{1,15})%.?([ºª])"]],
|
||||
}.get(L, []) -%}
|
||||
{#- en / de: a 4-digit number is read as a year only after one of these words (in 1990, im Jahr 1990), 1234 people stays a count -#}
|
||||
{%- set year_cues = {
|
||||
"en": "(?i)%b(in|since|by|from|until|till|to|through|between|of|around|circa|before|after|during|year|early|late|mid"
|
||||
~ "|january|february|march|april|may|june|july|august|september|october|november|december)",
|
||||
"de": "(?i)%b(im|seit|bis|von|ab|anno|um|vor|nach|zwischen|jahr|jahre|jahres|jahren|anfang|mitte|ende"
|
||||
~ "|januar|februar|märz|april|mai|juni|juli|august|september|oktober|november|dezember)",
|
||||
} -%}
|
||||
{%- set rules = [
|
||||
["money", cur ~ "%s?" ~ int ~ frac ~ "(?: (thousand|million|billion|trillion)%b)?"],
|
||||
["money2", int ~ frac ~ "%s?" ~ cur],
|
||||
["percent", neg ~ int ~ frac ~ "%s?[%%%]"],
|
||||
["date", "(?<![%d,./])(%d{1,2})[./](%d{1,2})[./](%d{4}|%d{2})(?!%d|[,./]%d)"],
|
||||
["time", time],
|
||||
] + ords
|
||||
+ ([["year", "(?<![%d,.])(%d{2,4})(?=年)"]] if L == "zh" else [])
|
||||
+ ([["year4", year_cues[L] ~ "(%s+)(%d{4})(?!%a|%d|[,.]%d|%s?[$€£¥%%])"]] if L in year_cues else [])
|
||||
+ ([["plural", "(?<![%d,.])(%d{1,15})s(?!%a)"]] if L == "en" else [])
|
||||
+ [["number", neg ~ int ~ frac]] -%}
|
||||
{%- for s in tts_norm_scan(t, rules) -%}
|
||||
{%- set g = s.groups -%}
|
||||
{%- if s.rule == "money" -%}
|
||||
{{- money(L, g[0], g[1], g[2], g[3]) -}}
|
||||
{%- elif s.rule == "money2" -%}
|
||||
{{- money(L, g[2], g[0], g[1], "") -}}
|
||||
{%- elif s.rule == "percent" -%}
|
||||
{%- set w = number(L, g[1], g[2], false) -%}
|
||||
{{- (W[L]["minus"] if g[0] else "") ~ ("百分之" ~ w if L == "zh" else w ~ W[L]["percent"]) -}}
|
||||
{%- elif s.rule == "date" -%}
|
||||
{#- read as three numbers, the order of day and month is the one of the text -#}
|
||||
{{- number(L, g[0], "", false) ~ W[L]["sep"] ~ number(L, g[1], "", false) ~ W[L]["sep"] ~ number(L, g[2], "", true) -}}
|
||||
{%- elif s.rule == "time" -%}
|
||||
{{- clock(L, g[0], g[1] or (g[2] if g|length > 2 else "")) -}}
|
||||
{%- elif s.rule == "ordinal" -%}
|
||||
{{- ordinal(L, g[0], g[1]) -}}
|
||||
{%- elif s.rule == "year" -%}
|
||||
{{- spell(L, g[0]) -}}
|
||||
{%- elif s.rule == "year4" -%}
|
||||
{{- g[0] ~ g[1] ~ number(L, g[2], "", true) -}}
|
||||
{%- elif s.rule == "plural" -%}
|
||||
{#- decades and plurals: 1990s -> nineteen nineties -#}
|
||||
{%- set w = number(L, g[0], "", true) -%}
|
||||
{{- (w[0:(w|length) - 1] ~ "ies") if w.endswith("y") else w ~ "s" -}}
|
||||
{%- elif s.rule == "number" -%}
|
||||
{{- (W[L]["minus"] if g[0] else "") ~ number(L, g[1], g[2], false) -}}
|
||||
{%- else -%}
|
||||
{{- s.text -}}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
@@ -0,0 +1,130 @@
|
||||
{%- set effort_map = {"none": 0.0, "minimal": 0.1, "low": 0.2, "medium": 0.7, "high": 0.9, "xhigh": 0.99, "max": 0.99} -%}
|
||||
{%- set role_token = {"user": "<|message_user|>", "assistant": "<|message_model|>", "system": "<|message_system|>", "tool": "<|message_tool|>"} -%}
|
||||
|
||||
{%- macro emit_thinking_effort() -%}
|
||||
{%- set eff = reasoning_effort if reasoning_effort is defined and reasoning_effort is not none else 0.9 -%}
|
||||
{%- if eff is string -%}
|
||||
{%- set key = eff | trim -%}
|
||||
{%- if key not in effort_map -%}
|
||||
{{- raise_exception("Unknown reasoning_effort: " ~ eff) -}}
|
||||
{%- endif -%}
|
||||
{%- set num = effort_map[key] -%}
|
||||
{%- else -%}
|
||||
{%- set num = eff | float -%}
|
||||
{%- endif -%}
|
||||
{%- if num < 0.0 or num > 0.99 -%}
|
||||
{{- raise_exception("reasoning_effort must be in [0.0, 0.99]") -}}
|
||||
{%- endif -%}
|
||||
{{- "<|message_system|><|content_text|>Thinking effort level: " -}}
|
||||
{%- if num == 0.0 -%}0{%- else -%}{{ num }}{%- endif -%}
|
||||
{{- "<|end_message|>" -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- if tools -%}
|
||||
{%- set tool_state = namespace(specs=[]) -%}
|
||||
{%- for tool in tools -%}
|
||||
{%- set fn = tool.function if tool.function is defined else tool -%}
|
||||
{%- set spec = {
|
||||
"description": (fn.description if fn.description is defined and fn.description else ""),
|
||||
"name": fn.name,
|
||||
"parameters": (fn.parameters if fn.parameters is defined and fn.parameters else {}),
|
||||
"type": (tool.type if tool.type is defined and tool.type else "function"),
|
||||
} -%}
|
||||
{%- set tool_state.specs = tool_state.specs + [spec] -%}
|
||||
{%- endfor -%}
|
||||
{{- "<|message_system|>tool_declare<|content_xml|>" -}}
|
||||
{{- tool_state.specs | tojson(sort_keys=true, separators=(",", ":")) -}}
|
||||
{{- "<|end_message|>" -}}
|
||||
{%- endif -%}
|
||||
|
||||
{%- set state = namespace(effort_emitted=false) -%}
|
||||
{%- for message in messages -%}
|
||||
{%- if message.role not in role_token -%}
|
||||
{{- raise_exception("Unknown message role: " ~ message.role) -}}
|
||||
{%- endif -%}
|
||||
{%- if not state.effort_emitted and message.role != "system" -%}
|
||||
{{- emit_thinking_effort() -}}
|
||||
{%- set state.effort_emitted = true -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- set header = namespace(text=role_token[message.role]) -%}
|
||||
{%- if message.role == "tool" -%}
|
||||
{%- if message.name is defined and message.name -%}
|
||||
{%- set header.text = header.text ~ message.name -%}
|
||||
{%- elif message.tool_call_id is defined and message.tool_call_id -%}
|
||||
{%- for prev in messages -%}
|
||||
{%- if prev.role == "assistant" and prev.tool_calls -%}
|
||||
{%- for tc in prev.tool_calls -%}
|
||||
{%- if tc.id is defined and tc.id == message.tool_call_id and tc.function.name is defined -%}
|
||||
{%- set header.text = header.text ~ tc.function.name -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
{%- elif message.role == "assistant" and message.name is defined and message.name -%}
|
||||
{%- set header.text = header.text ~ message.name -%}
|
||||
{%- endif -%}
|
||||
{%- set rtok = header.text -%}
|
||||
|
||||
{%- if message.role == "assistant" and message.reasoning_content is defined and message.reasoning_content -%}
|
||||
{{- rtok ~ "<|content_thinking|>" ~ message.reasoning_content ~ "<|end_message|>" -}}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message.content is string and (message.content or not (message.role == "assistant" and (message.tool_calls or message.reasoning_content))) -%}
|
||||
{{- rtok ~ "<|content_text|>" ~ message.content ~ "<|end_message|>" -}}
|
||||
{%- elif message.content -%}
|
||||
{%- for part in message.content -%}
|
||||
{%- if part is string -%}
|
||||
{{- rtok ~ "<|content_text|>" ~ part ~ "<|end_message|>" -}}
|
||||
{%- elif part.type is not defined or part.type in ("text", "input_text") -%}
|
||||
{%- set text_part = (part.text if part.text is defined and part.text is string else "") -%}
|
||||
{{- rtok ~ "<|content_text|>" ~ text_part ~ "<|end_message|>" -}}
|
||||
{%- elif part.type in ("image", "input_image", "image_url") -%}
|
||||
{%- if message.role not in ("user", "tool") -%}
|
||||
{{- raise_exception("Image content is only supported for user and tool messages") -}}
|
||||
{%- endif -%}
|
||||
{{- rtok ~ "<|content_image|><|unused_200054|><|end_message|>" -}}
|
||||
{%- elif part.type in ("audio", "input_audio", "audio_url") -%}
|
||||
{{- raise_exception("Inkling 2 does not accept audio input") -}}
|
||||
{%- else -%}
|
||||
{{- raise_exception("Unsupported content part type: " ~ part.type) -}}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message.role == "assistant" and message.tool_calls -%}
|
||||
{%- for tc in message.tool_calls -%}
|
||||
{%- set fn = tc.function -%}
|
||||
{%- if fn.name is not defined or fn.name is not string -%}
|
||||
{{- raise_exception("tool call function name must be a string") -}}
|
||||
{%- endif -%}
|
||||
{%- if message.name is defined and message.name and message.name != fn.name -%}
|
||||
{{- raise_exception("assistant name " ~ message.name ~ " does not match its tool call " ~ fn.name) -}}
|
||||
{%- endif -%}
|
||||
{%- set args = fn.arguments if fn.arguments is defined and fn.arguments else {} -%}
|
||||
{%- if args is string -%}
|
||||
{{- raise_exception("tool call arguments must be a parsed object, not a JSON string; canonicalize upstream") -}}
|
||||
{%- endif -%}
|
||||
{%- if args is not mapping -%}
|
||||
{{- raise_exception("tool call arguments must be an object") -}}
|
||||
{%- endif -%}
|
||||
{{- "<|message_model|>" ~ fn.name ~ "<|content_invoke_tool_json|>" -}}
|
||||
{{- '{"name":' ~ (fn.name | tojson(sort_keys=true, separators=(",", ":"))) ~ ',"args":' -}}
|
||||
{{- "{" -}}
|
||||
{%- for key, value in args | dictsort(case_sensitive=true) -%}
|
||||
{{- (key | tojson) ~ ":" ~ (value | tojson(separators=(",", ":"))) -}}
|
||||
{%- if not loop.last -%},{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- "}}<|end_message|>" -}}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message.role == "assistant" -%}
|
||||
{{- "<|content_model_end_sampling|>" -}}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
{%- if not state.effort_emitted -%}
|
||||
{{- emit_thinking_effort() -}}
|
||||
{%- endif -%}
|
||||
@@ -0,0 +1,129 @@
|
||||
{%- set effort_map = {"none": 0.0, "minimal": 0.1, "low": 0.2, "medium": 0.7, "high": 0.9, "max": 0.99} -%}
|
||||
{%- set role_token = {"user": "<|message_user|>", "assistant": "<|message_model|>", "system": "<|message_system|>", "tool": "<|message_tool|>"} -%}
|
||||
|
||||
{%- macro emit_thinking_effort() -%}
|
||||
{%- set eff = reasoning_effort if reasoning_effort is defined and reasoning_effort is not none else 0.9 -%}
|
||||
{%- if eff is string -%}
|
||||
{%- set key = eff | trim -%}
|
||||
{%- if key not in effort_map -%}
|
||||
{{- raise_exception("Unknown reasoning_effort: " ~ eff) -}}
|
||||
{%- endif -%}
|
||||
{%- set num = effort_map[key] -%}
|
||||
{%- else -%}
|
||||
{%- set num = eff | float -%}
|
||||
{%- endif -%}
|
||||
{%- if num < 0.0 or num > 0.99 -%}
|
||||
{{- raise_exception("reasoning_effort must be in [0.0, 0.99]") -}}
|
||||
{%- endif -%}
|
||||
{{- "<|message_system|><|content_text|>Thinking effort level: " -}}
|
||||
{%- if num == 0.0 -%}0{%- else -%}{{ num }}{%- endif -%}
|
||||
{{- "<|end_message|>" -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- if tools -%}
|
||||
{%- set tool_state = namespace(specs=[]) -%}
|
||||
{%- for tool in tools -%}
|
||||
{%- set fn = tool.function if tool.function is defined else tool -%}
|
||||
{%- set spec = {
|
||||
"description": (fn.description if fn.description is defined and fn.description else ""),
|
||||
"name": fn.name,
|
||||
"parameters": (fn.parameters if fn.parameters is defined and fn.parameters else {}),
|
||||
"type": (tool.type if tool.type is defined and tool.type else "function"),
|
||||
} -%}
|
||||
{%- set tool_state.specs = tool_state.specs + [spec] -%}
|
||||
{%- endfor -%}
|
||||
{{- "<|message_system|>tool_declare<|content_xml|>" -}}
|
||||
{{- tool_state.specs | tojson(sort_keys=true, separators=(",", ":")) -}}
|
||||
{{- "<|end_message|>" -}}
|
||||
{%- endif -%}
|
||||
|
||||
{%- set state = namespace(effort_emitted=false) -%}
|
||||
{%- for message in messages -%}
|
||||
{%- if message.role not in role_token -%}
|
||||
{{- raise_exception("Unknown message role: " ~ message.role) -}}
|
||||
{%- endif -%}
|
||||
{%- if not state.effort_emitted and message.role != "system" -%}
|
||||
{{- emit_thinking_effort() -}}
|
||||
{%- set state.effort_emitted = true -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- set rtok = role_token[message.role] -%}
|
||||
|
||||
{%- if message.role == "tool" -%}
|
||||
{%- set tool_name_state = namespace(name="") -%}
|
||||
{%- if message.name is defined and message.name -%}
|
||||
{%- set tool_name_state.name = message.name -%}
|
||||
{%- elif message.tool_call_id is defined and message.tool_call_id -%}
|
||||
{%- for prev in messages -%}
|
||||
{%- if prev.role == "assistant" and prev.tool_calls -%}
|
||||
{%- for tc in prev.tool_calls -%}
|
||||
{%- if tc.id is defined and tc.id == message.tool_call_id and tc.function.name is defined -%}
|
||||
{%- set tool_name_state.name = tc.function.name -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
{{- rtok -}}
|
||||
{%- if tool_name_state.name -%}{{- tool_name_state.name -}}{%- endif -%}
|
||||
{{- "<|content_text|>" -}}
|
||||
{%- if message.content is string -%}{{- message.content -}}{%- endif -%}
|
||||
{{- "<|end_message|>" -}}
|
||||
|
||||
{%- else -%}
|
||||
{%- if message.role == "assistant" and message.reasoning_content is defined and message.reasoning_content -%}
|
||||
{{- "<|message_model|><|content_thinking|>" ~ message.reasoning_content ~ "<|end_message|>" -}}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message.content is string -%}
|
||||
{{- rtok ~ "<|content_text|>" ~ message.content ~ "<|end_message|>" -}}
|
||||
{%- elif message.content -%}
|
||||
{%- for part in message.content -%}
|
||||
{%- if part is string -%}
|
||||
{{- rtok ~ "<|content_text|>" ~ part ~ "<|end_message|>" -}}
|
||||
{%- elif part.type is not defined or part.type in ("text", "input_text") -%}
|
||||
{%- set text_part = (part.text if part.text is defined and part.text is string else "") -%}
|
||||
{{- rtok ~ "<|content_text|>" ~ text_part ~ "<|end_message|>" -}}
|
||||
{%- elif part.type in ("image", "input_image", "image_url") -%}
|
||||
{{- rtok ~ "<|content_image|><|unused_200054|><|end_message|>" -}}
|
||||
{%- elif part.type in ("audio", "input_audio", "audio_url") -%}
|
||||
{{- rtok ~ "<|content_audio_input|><|unused_200053|><|audio_end|><|end_message|>" -}}
|
||||
{%- else -%}
|
||||
{{- raise_exception("Unsupported content part type: " ~ part.type) -}}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message.role == "assistant" and message.tool_calls -%}
|
||||
{%- for tc in message.tool_calls -%}
|
||||
{%- set fn = tc.function -%}
|
||||
{%- if fn.name is not defined or fn.name is not string -%}
|
||||
{{- raise_exception("tool call function name must be a string") -}}
|
||||
{%- endif -%}
|
||||
{%- set args = fn.arguments if fn.arguments is defined and fn.arguments else {} -%}
|
||||
{%- if args is string -%}
|
||||
{{- raise_exception("tool call arguments must be a parsed object, not a JSON string; canonicalize upstream") -}}
|
||||
{%- endif -%}
|
||||
{%- if args is not mapping -%}
|
||||
{{- raise_exception("tool call arguments must be an object") -}}
|
||||
{%- endif -%}
|
||||
{{- "<|message_model|>" ~ fn.name ~ "<|content_invoke_tool_json|>" -}}
|
||||
{{- '{"name":' ~ (fn.name | tojson(sort_keys=true, separators=(",", ":"))) ~ ',"args":' -}}
|
||||
{{- (args | tojson(sort_keys=true, separators=(",", ":"))) -}}
|
||||
{{- "}<|end_message|>" -}}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message.role == "assistant" -%}
|
||||
{{- "<|content_model_end_sampling|>" -}}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
{%- if not state.effort_emitted -%}
|
||||
{{- emit_thinking_effort() -}}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if add_generation_prompt -%}
|
||||
{{- "<|message_model|>" -}}
|
||||
{%- endif -%}
|
||||
@@ -33,7 +33,7 @@ Write a summary of the change log in a few sections:
|
||||
|
||||
### UI changes (if applicable)
|
||||
|
||||
[summarize the changes to `/tools/ui/`]
|
||||
[summarize the changes to `/ui/apps/chat/`]
|
||||
|
||||
### ggml changes (if applicable)
|
||||
|
||||
|
||||
@@ -144,7 +144,7 @@ endfunction()
|
||||
# Generate ui.cpp/ui.h embedding every file of ${dist_dir} (empty table when
|
||||
# it has no index.html), gzip-compressed when LLAMA_UI_GZIP is enabled.
|
||||
function(emit_files dist_dir)
|
||||
set(UI_TEMPLATE_DIR "${LLAMA_SOURCE_DIR}/tools/ui")
|
||||
set(UI_TEMPLATE_DIR "${LLAMA_SOURCE_DIR}/ui/apps/chat")
|
||||
|
||||
# Collect the asset list once and reuse it for the fingerprint,
|
||||
# validation, compression and embedding.
|
||||
@@ -518,7 +518,7 @@ function(hf_download version out_var out_resolved)
|
||||
endfunction()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Priority 1: pre-built assets supplied in tools/ui/dist
|
||||
# 1. Priority 1: pre-built assets supplied in ui/apps/chat/dist
|
||||
# ---------------------------------------------------------------------------
|
||||
if(EXISTS "${SRC_DIST_DIR}/index.html")
|
||||
message(STATUS "UI: using pre-built assets from ${SRC_DIST_DIR}")
|
||||
@@ -585,7 +585,7 @@ if(NOT provisioned)
|
||||
"In a disconnected environment, download the pre-built UI "
|
||||
"from a llama.cpp release at "
|
||||
"https://github.com/ggml-org/llama.cpp/releases and "
|
||||
"extract to tools/ui/dist.")
|
||||
"extract to ui/apps/chat/dist.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
@@ -22,11 +22,24 @@ Identify what actually changed and which area checklists below apply. Run `git d
|
||||
- `ggml/` (any backend, op, or `ggml.h`) -> **ggml / backend**
|
||||
- `include/llama.h` and other public headers -> **Public API**
|
||||
- `tools/server/` -> **Server**
|
||||
- `tools/ui/` (or the legacy `tools/server/webui/`) -> **WebUI**
|
||||
- `ui/apps/chat/` (or the legacy `tools/ui/`, `tools/server/webui/`) -> **Llama Chat**
|
||||
- `tests/`, `tools/server/tests/`, or any other added/changed test -> **Tests**
|
||||
- anything else, plus all of the above -> **General** (always runs)
|
||||
|
||||
Always run the **Scope and quick-reject gate**, the **Security review**, and the **General** checklist. Run each area checklist whose paths were touched. Additionally, if the diff introduces a new component, subsystem, or piece of infrastructure (a new file/class/module, a new abstraction, or hand-rolled machinery), run the **Approach and design** review. Tell the user which checklists you're running and why.
|
||||
Some areas have a developer README that documents internals and conventions the review must enforce. If the diff touches one of these areas, read the matching file into context before reviewing (required, not optional):
|
||||
|
||||
- `tools/server/` -> [tools/server/README-dev.md](../../tools/server/README-dev.md)
|
||||
- `tools/mtmd/` -> [tools/mtmd/README-dev.md](../../tools/mtmd/README-dev.md)
|
||||
|
||||
Always run the **Prior review context** step, the **Scope and quick-reject gate**, the **Security review**, and the **General** checklist. Run each area checklist whose paths were touched. Additionally, if the diff introduces a new component, subsystem, or piece of infrastructure (a new file/class/module, a new abstraction, or hand-rolled machinery), run the **Approach and design** review. Tell the user which checklists you're running and why.
|
||||
|
||||
## Prior review context (always)
|
||||
|
||||
Maintainers tend to repeat the same comments on the same parts of the code. Before reviewing, find out what they said last time so this diff doesn't repeat a mistake that was already flagged.
|
||||
|
||||
- For each touched file (or the hot spots in a large diff), find recent PRs that changed the same code: `git log --oneline -n 20 -- <file>` (merged PR numbers appear as `(#NNNNN)` in the subject), and `git blame -L <start>,<end> master -- <file>` for the exact lines being changed. For in-flight work, `gh pr list --state all --search "<file or symbol>"`.
|
||||
- Read the reviewer feedback on the most relevant few: `gh pr view <n> --comments` for the conversation and `gh api repos/ggml-org/llama.cpp/pulls/<n>/comments` for inline review comments.
|
||||
- Note recurring reviewer requests and frequently-occurring mistakes for this area (e.g. a helper that should be reused, a naming or layering rule, a backend that keeps getting broken, a test that is always asked for), then check the current diff against each one. Report a match as a finding and cite the earlier PR so the user can read the original context.
|
||||
|
||||
## Scope and quick-reject gate (always)
|
||||
|
||||
@@ -119,9 +132,9 @@ Public API changes carry a higher bar than internal ones (`CONTRIBUTING.md`). Re
|
||||
- Security: don't trust client-supplied headers (e.g. `X-Forwarded-For`) or add footguns; things like IP allowlisting belong at a reverse proxy unless there's a trusted-proxy design.
|
||||
- Wire new behavior into the existing request/response and checkpoint paths correctly; watch for resource leaks across requests.
|
||||
|
||||
## WebUI (`tools/ui/`)
|
||||
## Llama Chat (`ui/apps/chat/`)
|
||||
|
||||
These apply only to changes under `tools/ui/` (formerly `tools/server/webui/`) - the SvelteKit + Svelte 5 frontend. They are frontend conventions, not repo-wide ones, so do not apply them to C++ or other tool code.
|
||||
These apply only to changes under `ui/apps/chat/` (formerly `tools/ui/`, `tools/server/webui/`) - the SvelteKit + Svelte 5 frontend. They are frontend conventions, not repo-wide ones, so do not apply them to C++ or other tool code.
|
||||
|
||||
The dominant review theme in this area is placement: most accepted nits are about putting a value, function, or piece of logic in the layer where it belongs.
|
||||
|
||||
@@ -192,7 +205,7 @@ Adjacent data-shape rule: empty string is not a valid "unset" initial state - us
|
||||
- Sanitize anything rendered from model or remote data: the rehype/`{@html}` pipeline, SVG content, and URLs (use the `$lib/utils` URL sanitizer rather than ad-hoc checks or a third-party lib for trivial cases).
|
||||
- Proxy headers and agent/tool filesystem access are security boundaries - scope enforcement (e.g. browse-root sandboxing) must be validated end-to-end, not just in the UI; loop in server reviewers for the C++ side.
|
||||
|
||||
### PR hygiene specific to `tools/ui`
|
||||
### PR hygiene specific to `ui/apps/chat`
|
||||
|
||||
- Build artifacts are not committed - the UI is built by CI (`.github/workflows/ui-build.yml`) and downloaded/bundled at release time. `npm run build` must still succeed locally; a red UI build job is blocking.
|
||||
- `npm run lint && npm run check` (and `npm run format`) must pass; a failing Storybook test is blocking.
|
||||
|
||||
@@ -175,6 +175,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
|
||||
{ LLM_ARCH_NANBEIGE, "nanbeige" },
|
||||
{ LLM_ARCH_QWEN3TTS, "qwen3tts" },
|
||||
{ LLM_ARCH_POCKETTTS, "pockettts" },
|
||||
{ LLM_ARCH_ZONOS3, "zonos3" },
|
||||
{ LLM_ARCH_K2_HORIZON, "k2-horizon" },
|
||||
{ LLM_ARCH_BEAM, "beam" },
|
||||
{ LLM_ARCH_KOLIBRI1, "kolibri1" },
|
||||
@@ -1416,6 +1417,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
|
||||
case LLM_ARCH_DEEPSEEK4:
|
||||
case LLM_ARCH_DEEPSEEK41:
|
||||
case LLM_ARCH_MINIMAX_01:
|
||||
case LLM_ARCH_ZONOS3:
|
||||
case LLM_ARCH_INKLING2:
|
||||
return true;
|
||||
default:
|
||||
|
||||
@@ -176,6 +176,7 @@ enum llm_arch {
|
||||
LLM_ARCH_NANBEIGE,
|
||||
LLM_ARCH_QWEN3TTS,
|
||||
LLM_ARCH_POCKETTTS,
|
||||
LLM_ARCH_ZONOS3,
|
||||
LLM_ARCH_MINIMAX_01,
|
||||
LLM_ARCH_HRM_TEXT,
|
||||
LLM_ARCH_K2_HORIZON,
|
||||
|
||||
+14
-16
@@ -2908,6 +2908,7 @@ uint32_t llama_context::graph_max_nodes() const {
|
||||
model.arch == LLM_ARCH_BAILINGMOE3 ||
|
||||
model.arch == LLM_ARCH_QWEN35 ||
|
||||
model.arch == LLM_ARCH_QWEN35MOE ||
|
||||
model.arch == LLM_ARCH_ZONOS3 ||
|
||||
model.arch == LLM_ARCH_CLEF ||
|
||||
model.arch == LLM_ARCH_CLOVER_MOE ||
|
||||
model.arch == LLM_ARCH_QWEN4EXP ||
|
||||
@@ -4253,7 +4254,7 @@ void llama_context::opt_epoch_iter(
|
||||
ggml_opt_result_t result,
|
||||
const std::vector<llama_token> & tokens,
|
||||
const std::vector<llama_token> & labels_sparse,
|
||||
llama_batch & batch,
|
||||
llama_batch_ext & batch,
|
||||
ggml_opt_epoch_callback callback,
|
||||
bool train,
|
||||
int64_t idata_in_loop,
|
||||
@@ -4267,22 +4268,21 @@ void llama_context::opt_epoch_iter(
|
||||
memory->clear(true);
|
||||
|
||||
for (uint32_t pos_ctx = 0; pos_ctx < n_ctx; pos_ctx += n_batch) {
|
||||
batch.n_tokens = n_batch;
|
||||
batch.clear();
|
||||
for (uint32_t pos_batch = 0; pos_batch < n_batch; ++pos_batch) {
|
||||
batch.token [pos_batch] = tokens[pos_ctx + pos_batch];
|
||||
batch.pos [pos_batch] = pos_ctx + pos_batch;
|
||||
batch.n_seq_id[pos_batch] = 1;
|
||||
batch.seq_id [pos_batch][0] = 0;
|
||||
batch.logits [pos_batch] = true;
|
||||
}
|
||||
|
||||
// TODO: use llama_batch_ext here
|
||||
{
|
||||
llama_batch_compat compat(this, batch);
|
||||
if (!balloc->init(*compat.batch_ext, model.vocab, true)) {
|
||||
const int32_t idx = batch.add_token(0);
|
||||
const llama_pos pos = pos_ctx + pos_batch;
|
||||
if (idx < 0 || !batch.set_token_id(idx, tokens[pos_ctx + pos_batch])) {
|
||||
LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__);
|
||||
return;
|
||||
}
|
||||
batch.set_token_pos(idx, &pos);
|
||||
batch.set_output(idx, true);
|
||||
}
|
||||
|
||||
if (!balloc->init(batch, model.vocab, true)) {
|
||||
LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__);
|
||||
return;
|
||||
}
|
||||
|
||||
const uint32_t n_tokens_all = balloc->get_n_tokens();
|
||||
@@ -4381,7 +4381,7 @@ void llama_context::opt_epoch(
|
||||
|
||||
const uint32_t ubatch_per_ctx = n_ctx / n_ubatch;
|
||||
|
||||
struct llama_batch batch = llama_batch_init(n_batch, 0, 1);
|
||||
llama_batch_ext batch(this);
|
||||
std::vector<llama_token> tokens(n_ctx);
|
||||
std::vector<llama_token> labels_sparse(n_ctx);
|
||||
|
||||
@@ -4408,8 +4408,6 @@ void llama_context::opt_epoch(
|
||||
opt_epoch_iter(dataset, result_eval, tokens, labels_sparse, batch,
|
||||
callback_eval, train, idata_in_loop, ndata_in_loop, t_loop_start);
|
||||
}
|
||||
|
||||
llama_batch_free(batch);
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
+1
-1
@@ -233,7 +233,7 @@ struct llama_context {
|
||||
ggml_opt_result_t result,
|
||||
const std::vector<llama_token> & tokens,
|
||||
const std::vector<llama_token> & labels_sparse,
|
||||
llama_batch & batch,
|
||||
llama_batch_ext & batch,
|
||||
ggml_opt_epoch_callback callback,
|
||||
bool train,
|
||||
int64_t idata_in_loop,
|
||||
|
||||
@@ -403,6 +403,8 @@ size_t llama_grammar_trigger_pattern::find(const std::string & input) const {
|
||||
if (std::regex_match(input, match, regex)) {
|
||||
return find_start_pos(match);
|
||||
}
|
||||
// some regex implementations match ^/$ at line breaks
|
||||
return std::string::npos;
|
||||
}
|
||||
|
||||
// search anywhere
|
||||
|
||||
+1
-13
@@ -458,19 +458,7 @@ bool llama_kv_cache::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
|
||||
auto & cells = v_cells[seq_to_stream[seq_id]];
|
||||
auto & head = v_heads[seq_to_stream[seq_id]];
|
||||
|
||||
uint32_t new_head = cells.size();
|
||||
|
||||
for (uint32_t i = 0; i < cells.size(); ++i) {
|
||||
if (!cells.pos_in(i, p0, p1)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cells.seq_has(i, seq_id) && cells.seq_rm(i, seq_id)) {
|
||||
if (new_head == cells.size()) {
|
||||
new_head = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
const uint32_t new_head = cells.seq_rm(seq_id, p0, p1);
|
||||
|
||||
// If we freed up a slot, set head to it so searching can start there.
|
||||
if (new_head != cells.size() && new_head < head) {
|
||||
|
||||
@@ -308,6 +308,27 @@ public:
|
||||
return false;
|
||||
}
|
||||
|
||||
// remove seq_id from its cells at positions in [p0, p1)
|
||||
// return the lowest index of a cell that becomes empty, or size() if none does
|
||||
uint32_t seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
|
||||
assert(seq_id >= 0);
|
||||
assert(seq_id < LLAMA_MAX_SEQ);
|
||||
|
||||
uint32_t res = size();
|
||||
|
||||
const auto & sp = seq_pos[seq_id];
|
||||
for (auto it = sp.lower_bound({ p0, 0 }); it != sp.end() && it->first < p1; ) {
|
||||
const uint32_t i = it->second;
|
||||
++it; // advance first: seq_rm(i, seq_id) erases this entry from sp
|
||||
|
||||
if (seq_rm(i, seq_id)) {
|
||||
res = std::min(res, i);
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
// return true if the cell becomes empty (i.e. it did not contain seq_id before the call)
|
||||
bool seq_keep(uint32_t i, llama_seq_id seq_id) {
|
||||
assert(i < pos.size());
|
||||
|
||||
+9
-3
@@ -126,6 +126,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
|
||||
return new llama_model_qwen3tts(params);
|
||||
case LLM_ARCH_POCKETTTS:
|
||||
return new llama_model_pockettts(params);
|
||||
case LLM_ARCH_ZONOS3:
|
||||
return new llama_model_zonos3(params);
|
||||
case LLM_ARCH_PHI2:
|
||||
return new llama_model_phi2(params);
|
||||
case LLM_ARCH_PHI3:
|
||||
@@ -687,8 +689,9 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
|
||||
return {{conv_state_size, 3}};
|
||||
}
|
||||
// TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN]
|
||||
// zonos3 has the qwen35 GDN layout, its attention layers have no fused qkv
|
||||
if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF || ud->model->arch == LLM_ARCH_CLOVER_MOE ||
|
||||
ud->model->arch == LLM_ARCH_QWEN4EXP) {
|
||||
ud->model->arch == LLM_ARCH_QWEN4EXP || ud->model->arch == LLM_ARCH_ZONOS3) {
|
||||
|
||||
// fused full attention layers with Q gate tensors that need n_embd doubled:
|
||||
if (!hparams.is_recr(il) && (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias))) {
|
||||
@@ -731,7 +734,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str
|
||||
GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);
|
||||
return {{key_dim, 2 + head_ratio}};
|
||||
}
|
||||
if (std::regex_match(tensor_name, pattern_attn_gate_weight) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {
|
||||
if ((hparams.is_recr(il) && std::regex_match(tensor_name, pattern_attn_gate_weight)) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {
|
||||
return {{key_dim, head_ratio}};
|
||||
}
|
||||
if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||
|
||||
@@ -2700,6 +2703,7 @@ void llama_model::print_info() const {
|
||||
arch == LLM_ARCH_QWEN35 ||
|
||||
arch == LLM_ARCH_QWEN35MOE ||
|
||||
arch == LLM_ARCH_CLOVER_MOE ||
|
||||
arch == LLM_ARCH_ZONOS3 ||
|
||||
arch == LLM_ARCH_NEMOTRON_H ||
|
||||
arch == LLM_ARCH_NEMOTRON_H_MOE) {
|
||||
LLAMA_LOG_INFO("%s: ssm_d_conv = %u\n", __func__, hparams.ssm_d_conv);
|
||||
@@ -3331,7 +3335,8 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
|
||||
filter_recr = [&](uint32_t il) {
|
||||
return hparams.is_recr(il) && hparams.n_ff(il) == 0;
|
||||
};
|
||||
} else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_CLOVER_MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_MINIMAX_01) {
|
||||
} else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_CLOVER_MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_MINIMAX_01 ||
|
||||
arch == LLM_ARCH_ZONOS3) {
|
||||
filter_attn = [&](uint32_t il) {
|
||||
return il < hparams.n_layer() && !hparams.is_recr(il);
|
||||
};
|
||||
@@ -3745,6 +3750,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
|
||||
case LLM_ARCH_DOTS3NOTE:
|
||||
case LLM_ARCH_NANBEIGE:
|
||||
case LLM_ARCH_POCKETTTS:
|
||||
case LLM_ARCH_ZONOS3:
|
||||
case LLM_ARCH_LIMITE:
|
||||
return LLAMA_ROPE_TYPE_NORM;
|
||||
case LLM_ARCH_BAILINGMOE3:
|
||||
|
||||
@@ -584,6 +584,12 @@ struct llm_tokenizer_bpe : llm_tokenizer {
|
||||
regex_exprs = { "\\p{N}", "\\s+|[\\p{L}\\p{N}_]+|[^\\p{L}\\p{N}_\\s]+" };
|
||||
byte_encode = false;
|
||||
break;
|
||||
case LLAMA_VOCAB_PRE_TYPE_ZONOS3:
|
||||
// SuperBPE: no split on whitespace, tokens can span several words
|
||||
regex_exprs = {
|
||||
"\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]{2,}[\\r\\n/]*| +(?!\\S)",
|
||||
};
|
||||
break;
|
||||
case LLAMA_VOCAB_PRE_TYPE_WHITESPACE:
|
||||
// whitespace pre-tokenizer (jinaai/jina-embeddings-v2-base-zh)
|
||||
regex_exprs = {
|
||||
@@ -2774,6 +2780,10 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
|
||||
} else if (
|
||||
tokenizer_pre == "soprano") {
|
||||
pre_type = LLAMA_VOCAB_PRE_TYPE_SOPRANO;
|
||||
} else if (
|
||||
tokenizer_pre == "zonos3") {
|
||||
pre_type = LLAMA_VOCAB_PRE_TYPE_ZONOS3;
|
||||
clean_spaces = false;
|
||||
} else {
|
||||
throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
|
||||
}
|
||||
|
||||
@@ -75,6 +75,7 @@ enum llama_vocab_pre_type {
|
||||
LLAMA_VOCAB_PRE_TYPE_NEMOTRON_3_5_EMBD = 64,
|
||||
LLAMA_VOCAB_PRE_TYPE_SOPRANO = 65,
|
||||
LLAMA_VOCAB_PRE_TYPE_TEKKEN_NUM3 = 66,
|
||||
LLAMA_VOCAB_PRE_TYPE_ZONOS3 = 67,
|
||||
};
|
||||
|
||||
// HF Metaspace pre-tokenizer prepend_scheme
|
||||
|
||||
@@ -769,6 +769,28 @@ struct llama_model_pockettts : public llama_model_base {
|
||||
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
|
||||
};
|
||||
|
||||
struct llama_model_zonos3 : public llama_model_base {
|
||||
llama_model_zonos3(const struct llama_model_params & params) : llama_model_base(params) {}
|
||||
void load_arch_hparams(llama_model_loader & ml) override;
|
||||
void load_arch_tensors(llama_model_loader & ml) override;
|
||||
|
||||
struct graph : public llm_build_delta_net_base {
|
||||
graph(const llm_graph_params & params);
|
||||
private:
|
||||
// Full AttnRes bank: embedding + every sublayer output so far, [n_embd, n_src, n_tokens]
|
||||
ggml_tensor * bank = nullptr;
|
||||
|
||||
void attn_res_push(ggml_tensor * cur);
|
||||
ggml_tensor * attn_res_route(ggml_tensor * query, int il);
|
||||
|
||||
ggml_tensor * build_layer_attn(llm_graph_input_attn_kv * inp, ggml_tensor * cur, ggml_tensor * inp_pos, int il);
|
||||
ggml_tensor * build_layer_gdn(llm_graph_input_rs * inp, ggml_tensor * cur, int il);
|
||||
ggml_tensor * build_layer_ffn(ggml_tensor * cur, int il);
|
||||
};
|
||||
|
||||
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
|
||||
};
|
||||
|
||||
struct llama_model_codeshell : public llama_model_base {
|
||||
llama_model_codeshell(const struct llama_model_params & params) : llama_model_base(params) {}
|
||||
void load_arch_hparams(llama_model_loader & ml) override;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user