mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-16 15:57:37 -06:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e8fbcad855 | ||
|
|
95e837aaf1 | ||
|
|
d998c676ff | ||
|
|
92192d31e4 | ||
|
|
ad5cbfa911 | ||
|
|
d261e49e7e | ||
|
|
4ddf2b9470 | ||
|
|
4303cd94ae | ||
|
|
c7518dbec5 | ||
|
|
a68c6136bb | ||
|
|
3a7660371d | ||
|
|
e8698ac163 | ||
|
|
7804be0634 | ||
|
|
ef6bc6f367 | ||
|
|
a06044b934 | ||
|
|
62a6d3a302 | ||
|
|
ef3848f9d7 | ||
|
|
47fe133318 | ||
|
|
301db63370 | ||
|
|
a001af915e | ||
|
|
164ab018f9 | ||
|
|
5cb9516798 | ||
|
|
8bfd64b5ac | ||
|
|
e1925b3695 | ||
|
|
95db855bce | ||
|
|
1d2e880bad | ||
|
|
adf554d239 | ||
|
|
6b01a4767f | ||
|
|
f477c1b90a | ||
|
|
21fa9c043f | ||
|
|
a946eb541a |
@@ -19,12 +19,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${CANN_BASE_IMAGE} AS build
|
||||
@@ -40,7 +40,7 @@ WORKDIR /app
|
||||
# -- Copy project files --
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
# -- Set CANN environment variables (required for compilation) --
|
||||
# Using ENV instead of `source` allows environment variables to persist across the entire image layer
|
||||
|
||||
@@ -9,12 +9,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -28,7 +28,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "$TARGETARCH" = "amd64" ] || [ "$TARGETARCH" = "arm64" ]; then \
|
||||
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON; \
|
||||
|
||||
@@ -17,12 +17,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_CUDA_DEV_CONTAINER} AS build
|
||||
@@ -40,7 +40,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "${CUDA_DOCKER_ARCH}" != "default" ]; then \
|
||||
export CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CUDA_DOCKER_ARCH}"; \
|
||||
|
||||
@@ -11,12 +11,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS build
|
||||
@@ -41,7 +41,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "${GGML_SYCL_F16}" = "ON" ]; then \
|
||||
echo "GGML_SYCL_F16 is set" \
|
||||
|
||||
@@ -15,12 +15,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM ${BASE_MUSA_DEV_CONTAINER} AS build
|
||||
@@ -45,7 +45,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN if [ "${MUSA_DOCKER_ARCH}" != "default" ]; then \
|
||||
export CMAKE_ARGS="-DMUSA_ARCHITECTURES=${MUSA_DOCKER_ARCH}"; \
|
||||
|
||||
@@ -133,11 +133,11 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
src = lib.cleanSource ../../.;
|
||||
};
|
||||
|
||||
# Builds the webui locally, taking care not to require updating any sha256 hash.
|
||||
# Builds Llama Chat locally, taking care not to require updating any sha256 hash.
|
||||
webui = stdenvNoCC.mkDerivation {
|
||||
pname = "webui";
|
||||
version = llamaVersion;
|
||||
src = lib.cleanSource ../../tools/ui;
|
||||
src = lib.cleanSource ../../ui/apps/chat;
|
||||
|
||||
nativeBuildInputs = [
|
||||
nodejs
|
||||
@@ -146,7 +146,7 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
|
||||
# no sha256 required when using buildNodeModules
|
||||
npmDeps = importNpmLock.buildNodeModules {
|
||||
npmRoot = ../../tools/ui;
|
||||
npmRoot = ../../ui/apps/chat;
|
||||
inherit nodejs;
|
||||
};
|
||||
|
||||
@@ -156,8 +156,8 @@ effectiveStdenv.mkDerivation (finalAttrs: {
|
||||
};
|
||||
|
||||
postPatch = lib.optionalString useWebUi ''
|
||||
cp -r ${finalAttrs.webui} tools/ui/dist
|
||||
chmod -R u+w tools/ui/dist
|
||||
cp -r ${finalAttrs.webui} ui/apps/chat/dist
|
||||
chmod -R u+w ui/apps/chat/dist
|
||||
'';
|
||||
|
||||
# With PR#6015 https://github.com/ggml-org/llama.cpp/pull/6015,
|
||||
|
||||
@@ -28,12 +28,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
## Build Image
|
||||
@@ -83,7 +83,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
# Build Stage
|
||||
RUN bash -c "source ${OpenVINO_DIR}/setupvars.sh && \
|
||||
|
||||
@@ -17,12 +17,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
### Build image
|
||||
@@ -52,7 +52,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \
|
||||
cmake -S . -B build \
|
||||
|
||||
@@ -9,12 +9,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -31,7 +31,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN cmake -B build -DGGML_NATIVE=OFF -DGGML_VULKAN=ON -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON && \
|
||||
cmake --build build --config Release -j$(nproc)
|
||||
|
||||
@@ -9,12 +9,12 @@ FROM docker.io/node:$NODE_VERSION AS web
|
||||
|
||||
ARG APP_VERSION
|
||||
|
||||
WORKDIR /app/tools/ui
|
||||
WORKDIR /app/ui/apps/chat
|
||||
|
||||
COPY tools/ui/package.json tools/ui/package-lock.json ./
|
||||
COPY ui/apps/chat/package.json ui/apps/chat/package-lock.json ./
|
||||
RUN npm ci
|
||||
|
||||
COPY tools/ui/ ./
|
||||
COPY ui/apps/chat/ ./
|
||||
RUN LLAMA_BUILD_NUMBER="$APP_VERSION" npm run build
|
||||
|
||||
FROM docker.io/ubuntu:$UBUNTU_VERSION AS build
|
||||
@@ -28,7 +28,7 @@ WORKDIR /app
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY --from=web /app/tools/ui/dist tools/ui/dist
|
||||
COPY --from=web /app/ui/apps/chat/dist ui/apps/chat/dist
|
||||
|
||||
RUN cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_TESTS=OFF -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_ZENDNN=ON && \
|
||||
cmake --build build -j $(nproc)
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
|
||||
build*/
|
||||
|
||||
tools/ui/node_modules/
|
||||
ui/apps/chat/node_modules/
|
||||
|
||||
models/*
|
||||
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ insert_final_newline = unset
|
||||
[vendor/**.patch]
|
||||
trim_trailing_whitespace = unset
|
||||
|
||||
[tools/ui/**]
|
||||
[ui/**]
|
||||
indent_style = unset
|
||||
indent_size = unset
|
||||
end_of_line = unset
|
||||
|
||||
@@ -14,7 +14,9 @@ runs:
|
||||
run: |
|
||||
BUILD_NUMBER="$(git rev-list --count HEAD)"
|
||||
SHORT_HASH="$(git rev-parse --short=7 HEAD)"
|
||||
if [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
if [[ "$GITHUB_REF_TYPE" == "tag" && "${{ env.BRANCH_NAME }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "name=${{ env.BRANCH_NAME }}" >> $GITHUB_OUTPUT
|
||||
elif [[ "${{ env.BRANCH_NAME }}" == "master" || "${{ env.BRANCH_NAME }}" == "b${BUILD_NUMBER}" ]]; then
|
||||
echo "name=b${BUILD_NUMBER}" >> $GITHUB_OUTPUT
|
||||
else
|
||||
SAFE_NAME=$(echo "${{ env.BRANCH_NAME }}" | tr '/' '-')
|
||||
|
||||
+1
-2
@@ -48,7 +48,6 @@ examples:
|
||||
- all-globs-to-all-files:
|
||||
- '!tools/server/**'
|
||||
- '!tools/mtmd/**'
|
||||
- '!tools/ui/**'
|
||||
testing:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
@@ -72,7 +71,7 @@ android:
|
||||
server/ui:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
- tools/ui/**
|
||||
- ui/**
|
||||
server:
|
||||
- changed-files:
|
||||
- any-glob-to-any-file:
|
||||
|
||||
@@ -16,6 +16,10 @@ on:
|
||||
description: "Skip the s390x build target (useful for fast test runs that do not need the IBM Z runner)"
|
||||
type: boolean
|
||||
default: false
|
||||
skip_ref_check:
|
||||
description: "Skip the requirement to run from a tag (allows branch refs)"
|
||||
type: boolean
|
||||
default: false
|
||||
schedule:
|
||||
# Rebuild daily rather than on every push because it is expensive
|
||||
- cron: '12 4 * * *'
|
||||
@@ -38,6 +42,14 @@ jobs:
|
||||
source_tag: ${{ steps.srctag.outputs.name }}
|
||||
|
||||
steps:
|
||||
- name: Check manual run ref
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && !inputs.skip_ref_check }}
|
||||
run: |
|
||||
if [[ "$GITHUB_REF_TYPE" != "tag" ]]; then
|
||||
echo "::error::Manual runs of this workflow must be started from a tag"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Clone
|
||||
id: checkout
|
||||
uses: actions/checkout@v6
|
||||
@@ -170,7 +182,7 @@ jobs:
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
# - name: Set up QEMU
|
||||
# if: ${{ contains(matrix.config.platforms, 'linux/amd64') }}
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
BODY: ${{ github.event.pull_request.body }}
|
||||
HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'video demo') }}
|
||||
run: |
|
||||
# sections written by demo:pr (see tools/ui/demo/cli.ts): the managed comment
|
||||
# sections written by demo:pr (see ui/apps/chat/demo/cli.ts): the managed comment
|
||||
# markers, or a preview linking to a demo video uploaded for this PR
|
||||
has_demo=false
|
||||
if { grep -qF '<!-- demo:start -->' <<< "$BODY" && grep -qF '<!-- demo:end -->' <<< "$BODY"; } ||
|
||||
|
||||
@@ -130,7 +130,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -196,7 +196,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -274,7 +274,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -381,7 +381,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -479,7 +479,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v5
|
||||
@@ -571,7 +571,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -624,7 +624,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
@@ -672,7 +672,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
@@ -894,7 +894,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Install Ninja
|
||||
run: |
|
||||
@@ -1460,7 +1460,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
@@ -1532,7 +1532,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Free up disk space
|
||||
uses: ggml-org/free-disk-space@v1.3.1
|
||||
@@ -1647,7 +1647,7 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist
|
||||
path: ui/apps/chat/dist
|
||||
|
||||
- name: Dependencies
|
||||
id: depends
|
||||
|
||||
@@ -7,6 +7,40 @@ on:
|
||||
description: 'Commit SHA1 to build'
|
||||
required: false
|
||||
type: string
|
||||
spec_methods:
|
||||
description: 'Speculative methods to test'
|
||||
required: true
|
||||
default: 'compact'
|
||||
type: choice
|
||||
options:
|
||||
- compact
|
||||
- nightly
|
||||
- all
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/server-speculative.yml',
|
||||
'common/speculative.*',
|
||||
'src/models/dflash*.cpp',
|
||||
'src/models/dspark.cpp',
|
||||
'src/models/eagle3.cpp',
|
||||
'src/models/lilicorr.cpp',
|
||||
'src/models/qwen35.cpp',
|
||||
'tools/server/tests/unit/test_spec_methods.py'
|
||||
]
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/server-speculative.yml',
|
||||
'common/speculative.*',
|
||||
'src/models/dflash*.cpp',
|
||||
'src/models/dspark.cpp',
|
||||
'src/models/eagle3.cpp',
|
||||
'src/models/lilicorr.cpp',
|
||||
'src/models/qwen35.cpp',
|
||||
'tools/server/tests/unit/test_spec_methods.py'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
@@ -56,8 +90,9 @@ jobs:
|
||||
|
||||
- name: Test speculative decoding
|
||||
env:
|
||||
SPEC_METHODS: compact
|
||||
SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || format('{0}/spec-models', runner.temp) }}
|
||||
SPEC_METHODS: ${{ github.event.inputs.spec_methods || 'compact' }}
|
||||
# pull requests from forks do not get repository variables
|
||||
SPEC_MODEL_DIR: ${{ vars.SPEC_METHOD_MODEL_DIR || '/mnt/cache/spec-models' }}
|
||||
N_GPU_LAYERS: 99
|
||||
run: |
|
||||
cd tools/server/tests
|
||||
|
||||
@@ -23,20 +23,20 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build application
|
||||
run: npm run build
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Upload built UI
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
retention-days: 1
|
||||
|
||||
@@ -37,26 +37,26 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build application
|
||||
env:
|
||||
LLAMA_BUILD_NUMBER: ${{ steps.version.outputs.ui_version }}
|
||||
run: npm run build
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run PWA unit tests (versioned build output)
|
||||
run: npx vitest --project=unit --run tests/unit/pwa.spec.ts
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Upload built UI
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
retention-days: 1
|
||||
|
||||
@@ -33,15 +33,15 @@ jobs:
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
run-id: ${{ inputs.run_id }}
|
||||
github-token: ${{ github.token }}
|
||||
|
||||
- name: Create distribution archive
|
||||
run: |
|
||||
tar -czf dist.tar.gz -C tools/ui/dist .
|
||||
tar -czf dist.tar.gz -C ui/apps/chat/dist .
|
||||
sha256sum dist.tar.gz > dist.tar.gz.sha256
|
||||
mv dist.tar.gz dist.tar.gz.sha256 tools/ui/dist/
|
||||
mv dist.tar.gz dist.tar.gz.sha256 ui/apps/chat/dist/
|
||||
|
||||
- name: Install Hugging Face Hub CLI
|
||||
run: pip install -U huggingface_hub
|
||||
@@ -52,12 +52,12 @@ jobs:
|
||||
- name: Sync built files to Hugging Face bucket (version tag)
|
||||
run: |
|
||||
# Upload the built files to the Hugging Face bucket under the release version
|
||||
hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet
|
||||
hf buckets sync ui/apps/chat/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/${{ inputs.version_tag }} --delete --quiet
|
||||
|
||||
- name: Sync built files to Hugging Face bucket (latest)
|
||||
run: |
|
||||
# Also upload to the 'latest' directory for fallback downloads
|
||||
hf buckets sync tools/ui/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet
|
||||
hf buckets sync ui/apps/chat/dist hf://buckets/ggml-org/${{ env.HF_BUCKET_NAME }}/latest --delete --quiet
|
||||
|
||||
- name: Verify upload
|
||||
run: |
|
||||
|
||||
@@ -17,7 +17,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -25,7 +25,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui-self-hosted.yml',
|
||||
'.github/workflows/ui-build-self-hosted.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
@@ -67,33 +67,33 @@ jobs:
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Run type checking
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run linting
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run lint
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Client tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:client
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Unit tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:unit
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
@@ -109,25 +109,25 @@ jobs:
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Build Storybook
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run build-storybook
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run UI tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:ui -- --testTimeout=60000
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run E2E tests
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run test:e2e
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
+17
-17
@@ -13,7 +13,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
pull_request:
|
||||
@@ -21,7 +21,7 @@ on:
|
||||
paths: [
|
||||
'.github/workflows/ui.yml',
|
||||
'.github/workflows/ui-build.yml',
|
||||
'tools/ui/**.*',
|
||||
'ui/apps/chat/**.*',
|
||||
'tools/server/tests/**.*'
|
||||
]
|
||||
|
||||
@@ -66,46 +66,46 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Download built UI artifacts
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
if: ${{ steps.node.conclusion == 'success' }}
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run type checking
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run linting
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run lint
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npx playwright install --with-deps
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Client tests
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:client
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run Unit tests (uses pre-built dist/ from ui-build)
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:unit
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
e2e-tests:
|
||||
name: E2E Tests
|
||||
@@ -124,38 +124,38 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
# cache: "npm"
|
||||
# cache-dependency-path: "tools/ui/package-lock.json"
|
||||
# cache-dependency-path: "ui/apps/chat/package-lock.json"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
id: setup
|
||||
if: ${{ steps.node.conclusion == 'success' }}
|
||||
run: npm ci
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Download built UI artifacts (reuses ui-build)
|
||||
uses: actions/download-artifact@v6
|
||||
with:
|
||||
name: llama-ui.zip
|
||||
path: tools/ui/dist/
|
||||
path: ui/apps/chat/dist/
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npx playwright install --with-deps
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build Storybook
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run build-storybook
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run UI tests
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:ui -- --testTimeout=60000
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Run E2E tests (uses pre-built dist/ from ui-build)
|
||||
if: ${{ always() && steps.playwright.conclusion == 'success' }}
|
||||
run: npm run test:e2e
|
||||
working-directory: tools/ui
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# date: Wed Nov 4 12:03:31 EET 2026
|
||||
# date: Mon Nov 16 10:05:56 EET 2026
|
||||
# this file is auto-generated by scripts/gen-authors.sh
|
||||
|
||||
Нияз Гарифзянов <112617865+garrnizon@users.noreply.github.com>
|
||||
@@ -43,7 +43,9 @@ Aarni Koskela <akx@iki.fi>
|
||||
Aaron Miller <apage43@ninjawhale.com>
|
||||
Aaron Teo <57927438+taronaeo@users.noreply.github.com>
|
||||
Aaron Teo <aaron.teo1@ibm.com>
|
||||
Aaron Thomas <84493696+arxhr007@users.noreply.github.com>
|
||||
Aaryaman Vasishta <aaryaman.vasishta@amd.com>
|
||||
Aayan Karmali <44980366+StockerMC@users.noreply.github.com>
|
||||
Abheek Gulati <abheekg@hotmail.com>
|
||||
abhijain1204fujitsu <139222713+abhijain1204fujitsu@users.noreply.github.com>
|
||||
Abhijit Ramesh <abhijitramesh2k@gmail.com>
|
||||
@@ -112,10 +114,12 @@ Aleksander Grygier <admin@serveurperso.com>
|
||||
Aleksander Grygier <aleksander.grygier@gmail.com>
|
||||
Aleksander Grygier <thichthat@gmail.com>
|
||||
Aleksei Nikiforov <103434461+AlekseiNikiforovIBM@users.noreply.github.com>
|
||||
Alen Požun <kalyx.solutions@proton.me>
|
||||
Alessandro de Oliveira Faria (A.K.A.CABELO) <cabelo@opensuse.org>
|
||||
Alessandro98-git <61804547+Alessandro98-git@users.noreply.github.com>
|
||||
Alex <18387287+wadealexc@users.noreply.github.com>
|
||||
Alex <59368173+AlexGabbia@users.noreply.github.com>
|
||||
Alex <alexzkhr@gmail.com>
|
||||
Alex <awhill19@icloud.com>
|
||||
Alex Azarov <alex@azarov.by>
|
||||
Alex Azarov <alexander.azarov@mapbox.com>
|
||||
@@ -132,6 +136,7 @@ Alex Trotta <44127594+Ahajha@users.noreply.github.com>
|
||||
Alex Tuddenham <61622354+AlexsCode@users.noreply.github.com>
|
||||
Alex von Gluck IV <kallisti5@unixzen.com>
|
||||
Alex Wu <dindinw@users.noreply.github.com>
|
||||
Alex-JP-93 <Jianpeng.Li@amd.com>
|
||||
alex-spacemit <jinghui.huang@spacemit.com>
|
||||
Alexander Batischev <eual.jp@gmail.com>
|
||||
Alexander Heisler <126129661+heislera763@users.noreply.github.com>
|
||||
@@ -143,6 +148,7 @@ Alexis Williams <typedrat@users.noreply.github.com>
|
||||
alexpinel <93524949+alexpinel@users.noreply.github.com>
|
||||
Alfred <zxu3@clemson.edu>
|
||||
Ali Chraghi <63465728+alichraghi@users.noreply.github.com>
|
||||
Ali F. Al-Ramadan <55710790+aalramadan@users.noreply.github.com>
|
||||
Ali Nehzat <ali.nehzat@thanks.dev>
|
||||
Ali Tariq <ali.tariq@10xengineers.ai>
|
||||
Ali Tariq <alitariq4589@gmail.com>
|
||||
@@ -158,6 +164,7 @@ Aman Karki <amankarki151@gmail.com>
|
||||
Aman Karki <itsamankarki@gmail.com>
|
||||
amd-dwang <dong.wang@amd.com>
|
||||
amd-lalithnc <lalithnc@amd.com>
|
||||
AMIR <127450356+amirthn6533@users.noreply.github.com>
|
||||
Amir <amir_zia@outlook.com>
|
||||
amirai21 <89905406+amirai21@users.noreply.github.com>
|
||||
AmirAli Mirian <37371367+amiralimi@users.noreply.github.com>
|
||||
@@ -213,6 +220,7 @@ Antoine Viallon <antoine@lesviallon.fr>
|
||||
Anton I <inflyer@hotmail.com>
|
||||
Anton Mitkov <anton_b_mitkov@abv.bg>
|
||||
Anton Mitkov <anton.mitkov@codeplay.com>
|
||||
Anton Sokolchenko <wsevendays@gmail.com>
|
||||
Antonis Makropoulos <benuix@gmail.com>
|
||||
anubhavfujitsu <Anubhav.Srivastava@fujitsu.com>
|
||||
Anudit Nagar <nagaranudit@gmail.com>
|
||||
@@ -233,10 +241,12 @@ arlo-phoenix <140345165+arlo-phoenix@users.noreply.github.com>
|
||||
Armen Kaleshian <kriation@users.noreply.github.com>
|
||||
Arsen Arutunan <58118221+limloop@users.noreply.github.com>
|
||||
Arseniy Obolenskiy <good.doog.2012@gmail.com>
|
||||
ArshM17-NV <amaknojia@nvidia.com>
|
||||
Artem <guinmoon@gmail.com>
|
||||
Artem Zinnatullin <ceo@abstractny.gay>
|
||||
Artyom Lebedev <vagran.ast@gmail.com>
|
||||
aryantandon01 <80969509+aryantandon01@users.noreply.github.com>
|
||||
Asahi-Prv <246092086+Asahi-Prv@users.noreply.github.com>
|
||||
Asahi-Prv <asahihori7@gmail.com>
|
||||
asbelin <71226900+asbelin@users.noreply.github.com>
|
||||
Asbjørn Olling <asbjornolling@gmail.com>
|
||||
@@ -258,6 +268,7 @@ AustinMroz <austinmroz@utexas.edu>
|
||||
AUTOMATIC1111 <16777216c@gmail.com>
|
||||
automaticcat <daogiatuank54@gmail.com>
|
||||
awatuna <23447591+awatuna@users.noreply.github.com>
|
||||
Awss <gitj.18@alt.awss1i.com>
|
||||
b4b4o <zwbao@foxmail.com>
|
||||
Bach Le <bach@bullno1.com>
|
||||
BADR <contact@pythops.com>
|
||||
@@ -351,6 +362,7 @@ CarterLi999 <664681047@qq.com>
|
||||
Casey Primozic <casey@cprimozic.net>
|
||||
Casey Primozic <me@ameo.link>
|
||||
CausalLM <148736309+CausalLM@users.noreply.github.com>
|
||||
CC <65151926+realMoana@users.noreply.github.com>
|
||||
ccbinn <ccbinn@163.com>
|
||||
cduk <19917266+cduk@users.noreply.github.com>
|
||||
cebtenzzre <cebtenzzre@gmail.com>
|
||||
@@ -379,6 +391,7 @@ Cheng Shao <terrorjack@type.dance>
|
||||
Chenguang Li <757486878@qq.com>
|
||||
Chenguang Li <87689256+noemotiovon@users.noreply.github.com>
|
||||
chiheb ben cheikh <100644963+ChihebBENCHEIKH1@users.noreply.github.com>
|
||||
chinmaykamjith001 <chinmaykamjith@gmail.com>
|
||||
Chipmunk <101038159+CHIPMUNK-T0T@users.noreply.github.com>
|
||||
chiranko <96988916+chiranko@users.noreply.github.com>
|
||||
Chris Danis <cdanis@gmail.com>
|
||||
@@ -434,6 +447,7 @@ crasm <crasm@git.vczf.us>
|
||||
crat0z <11581854+crat0z@users.noreply.github.com>
|
||||
CRD716 <crd716@gmail.com>
|
||||
CrispStrobe <154636388+CrispStrobe@users.noreply.github.com>
|
||||
Cristian Adam <cristian.adam@gmail.com>
|
||||
Cristiano Pinto <140563307+crowmoed@users.noreply.github.com>
|
||||
crsawyer <7572190+crsawyer@users.noreply.github.com>
|
||||
Csaba Kecskemeti <csaba.kecskemeti@gmail.com>
|
||||
@@ -527,6 +541,7 @@ Djip007 <3705339+Djip007@users.noreply.github.com>
|
||||
Djip007 <djip.perois@free.fr>
|
||||
dm4 <dm4@secondstate.io>
|
||||
dm4 <sunrisedm4@gmail.com>
|
||||
Dmitrii Panchev <dmnobunaga@users.noreply.github.com>
|
||||
Dmitry Atamanov <data-man@users.noreply.github.com>
|
||||
Dmytro Minochkin <dmytro.minochkin@gmail.com>
|
||||
Dmytro Romanov <casteldazur@gmail.com>
|
||||
@@ -567,6 +582,7 @@ Ed Addario <29247825+EAddario@users.noreply.github.com>
|
||||
Ed Lee <edilee@mozilla.com>
|
||||
Ed Lepedus <ed.lepedus@googlemail.com>
|
||||
Eddie-Wang <wangjinheng1120@163.com>
|
||||
eddiechen <Echen1246@gmail.com>
|
||||
Eden <aa9736195201@gmail.com>
|
||||
eduardopessin <100053075+eduardopessin@users.noreply.github.com>
|
||||
Edward Taylor <edeetee@gmail.com>
|
||||
@@ -598,6 +614,7 @@ Erik Winter <93544977+ewintr@users.noreply.github.com>
|
||||
Ervin Áron Tasnádi <etasnadi@protonmail.com>
|
||||
Esko Toivonen <eskot98@gmail.com>
|
||||
Ethan Guo <ethanguo.dev@gmail.com>
|
||||
Ethan Lin <linsen458@gmail.com>
|
||||
Ethan Turner <eturner64@gmail.com>
|
||||
Ettore Di Giacinto <mudler@users.noreply.github.com>
|
||||
EugeoSynthesisThirtyTwo <gabriel.dhimoila@gmail.com>
|
||||
@@ -625,6 +642,7 @@ fanyang <fanyang89@outlook.com>
|
||||
Farbod Bijary <110523279+farbodbj@users.noreply.github.com>
|
||||
Fathi Boudra <fathi.boudra@linaro.org>
|
||||
Fattire <528174+fat-tire@users.noreply.github.com>
|
||||
Feiteng <lifeiteng0422@gmail.com>
|
||||
felix <felix314159@users.noreply.github.com>
|
||||
Felix <stenbackfelix@gmail.com>
|
||||
Felix Ye <felixyjs@gmail.com>
|
||||
@@ -632,6 +650,7 @@ fengerhu1 <2748250768@qq.com>
|
||||
feni6 <fenissova@proton.me>
|
||||
fidoriel <49869342+fidoriel@users.noreply.github.com>
|
||||
fiesh <fiesh@zefix.tv>
|
||||
figkcoci-debug <figkcoci@gmail.com>
|
||||
FINAL-Bench <kkms1116@koreacu.ac.kr>
|
||||
Finn Voorhees <finnvoorhees@gmail.com>
|
||||
Firat <firatkiral@gmail.com>
|
||||
@@ -646,6 +665,7 @@ Foad Abo Dahood <32059146+masterFoad@users.noreply.github.com>
|
||||
Folko-Ven <71110216+Folko-Ven@users.noreply.github.com>
|
||||
forforever73 <63285796+forforever73@users.noreply.github.com>
|
||||
Foul-Tarnished <107711110+Foul-Tarnished@users.noreply.github.com>
|
||||
fox5611 <66219952+fox5611@users.noreply.github.com>
|
||||
Francisco Herrera <ppaanncchhoo507@gmail.com>
|
||||
Francisco Melo <43780565+francis2tm@users.noreply.github.com>
|
||||
Francois Dugast <francois.dugast@intel.com>
|
||||
@@ -709,6 +729,7 @@ GiviMAD <GiviMAD@users.noreply.github.com>
|
||||
gliptic <gliptic@users.noreply.github.com>
|
||||
gn64 <yukikaze.jp@gmail.com>
|
||||
goerch <jhr.walter@t-online.de>
|
||||
Gotya-OFFCIAL <zhengfreeman@gmail.com>
|
||||
Govlzkoy <gotope@users.noreply.github.com>
|
||||
grahameth <96447521+grahameth@users.noreply.github.com>
|
||||
Gregor Jasny <gjasny@googlemail.com>
|
||||
@@ -749,6 +770,7 @@ Haowei Wu <breadcyanide@icloud.com>
|
||||
Haoxiang Fei <tonyfettes@tonyfettes.com>
|
||||
Harald Fernengel <harald.fernengel@here.com>
|
||||
Harapan Rachman <harapanrachman@gmail.com>
|
||||
Harkanwal Bedi <bedi1887@gmail.com>
|
||||
Harkirat Gill <harkirat.gill@amd.com>
|
||||
HarrisonSec <gzxharrison@gmail.com>
|
||||
Hatsune Miku <129688334+at8u@users.noreply.github.com>
|
||||
@@ -768,6 +790,7 @@ Henrik Forstén <henrik.forsten@gmail.com>
|
||||
Henry Linjamäki <henry.linjamaki@gmail.com>
|
||||
Henry Linjamäki <henry.mikael.linjamaki@intel.com>
|
||||
Henry147147 <44851451+Henry147147@users.noreply.github.com>
|
||||
Hera <58063257+hera2019@users.noreply.github.com>
|
||||
Herman Semenoff <GermanAizek@yandex.ru>
|
||||
Herman Semenov <GermanAizek@yandex.ru>
|
||||
Hesen Peng <hesen.peng@gmail.com>
|
||||
@@ -846,6 +869,7 @@ Imad Saddik <79410781+ImadSaddik@users.noreply.github.com>
|
||||
iMil <imil@NetBSD.org>
|
||||
Incarnas <119618389+bit-incarnas@users.noreply.github.com>
|
||||
InflexCZE <martin.pavlicek11@gmail.com>
|
||||
Iñigo Imaz <49730431+inimaz@users.noreply.github.com>
|
||||
Intel AI Get-to Market Customer Success and Solutions <ai.gtm.css@gmail.com>
|
||||
intelmatt <61025942+intelmatt@users.noreply.github.com>
|
||||
iohub <rickyang.pro@gmail.com>
|
||||
@@ -857,7 +881,9 @@ IsaacDynamo <61521674+IsaacDynamo@users.noreply.github.com>
|
||||
Ishaan Gandhi <Ishaangandhi@gmail.com>
|
||||
iSma <ismail.senhaji@gmail.com>
|
||||
Ismail <115064057+AlrIsmail@users.noreply.github.com>
|
||||
IsNot <209144869@qq.com>
|
||||
issixx <46835150+issixx@users.noreply.github.com>
|
||||
Iswarya Alex <47045679+iswaryaalex@users.noreply.github.com>
|
||||
itsnotoger <19309683+itsnotoger@users.noreply.github.com>
|
||||
itterative <190138728+itterative@users.noreply.github.com>
|
||||
Ivan <nekotekina@gmail.com>
|
||||
@@ -888,6 +914,7 @@ James Francis <6763899+JamesFranc@users.noreply.github.com>
|
||||
James O'Leary <65884233+jpohhhh@users.noreply.github.com>
|
||||
James Reynolds <magnusviri@users.noreply.github.com>
|
||||
jameswu2014 <545426914@qq.com>
|
||||
JamieBordoni <jamie.bordoni@protonmail.com>
|
||||
Jan Boon <jan.boon@kaetemi.be>
|
||||
Jan Boon <kaetemi@gmail.com>
|
||||
Jan Ekström <jeebjp@gmail.com>
|
||||
@@ -956,8 +983,10 @@ Jinyang He <hejinyang@loongson.cn>
|
||||
jinzihao <jinzihao1996@gmail.com>
|
||||
Jiří Podivín <66251151+jpodivin@users.noreply.github.com>
|
||||
Jiří Sejkora <Sejseloid@gmail.com>
|
||||
Jiří Svítil <32205472+mistrjirka@users.noreply.github.com>
|
||||
Jiwoong Song <55657581+jwsong98@users.noreply.github.com>
|
||||
JJJYmmm <92386084+JJJYmmm@users.noreply.github.com>
|
||||
jjperry69 <jjperry69@gmail.com>
|
||||
jklincn <985765408@qq.com>
|
||||
jklincn <jklincn@outlook.com>
|
||||
JM Robles <roblesjm@gmail.com>
|
||||
@@ -970,6 +999,7 @@ Joe Rowell <joerowell4@gmail.com>
|
||||
Joe Todd <joe.todd@codeplay.com>
|
||||
joecryptotoo <80373433+joecryptotoo@users.noreply.github.com>
|
||||
Joel <agnel.joel@gmail.com>
|
||||
Joel Fernandes <joelagnelf@nvidia.com>
|
||||
Johan <JohanAR@users.noreply.github.com>
|
||||
Johannes Gäßler <johannesg@5d6.de>
|
||||
Johannes Rudolph <johannes.rudolph@gmail.com>
|
||||
@@ -988,6 +1018,7 @@ Jonas J <111707981+John-194@users.noreply.github.com>
|
||||
Jonas Jankaitis <111707981+John-194@users.noreply.github.com>
|
||||
Jonas Wunderlich <32615971+jonas-w@users.noreply.github.com>
|
||||
Jonathan <47618606+jbuchananr@users.noreply.github.com>
|
||||
Jonathan Buck <jonathan.j.buck@gmail.com>
|
||||
Jonathan Clohessy <jonathan.clohessy@arm.com>
|
||||
Jonathan Graehl <99024+graehl@users.noreply.github.com>
|
||||
Jorge A <161275481+jorgealias@users.noreply.github.com>
|
||||
@@ -1097,6 +1128,7 @@ klosax <131523366+klosax@users.noreply.github.com>
|
||||
KnightYao <KnightYao@users.noreply.github.com>
|
||||
KokerZhou <111279477+KokerZhou@users.noreply.github.com>
|
||||
Kolen Cheung <ickc@users.noreply.github.com>
|
||||
Kong Jiale <2049447146@qq.com>
|
||||
kononnable <kononnable@gmail.com>
|
||||
Konrad Moren <kmoren@nvidia.com>
|
||||
konradmb <konradmb@o2.pl>
|
||||
@@ -1167,6 +1199,7 @@ li-lizhe <82501526+li-lizhe@users.noreply.github.com>
|
||||
liangliangchang <lichang@amd.com>
|
||||
liminfei-amd <91481003+liminfei-amd@users.noreply.github.com>
|
||||
limitedAtonement <limitedAtonement@users.noreply.github.com>
|
||||
Lines <linellazatin@gmail.com>
|
||||
lingyezhixing <144504450+lingyezhixing@users.noreply.github.com>
|
||||
linsen458-spec <linsen458@gmail.com>
|
||||
Linux User <linuxusergd@outlook.com>
|
||||
@@ -1227,6 +1260,7 @@ makomk <makosoft@googlemail.com>
|
||||
manayang <jackmanayang@gmail.com>
|
||||
manikbhandari <mbbhandarimanik2@gmail.com>
|
||||
Manuel <44313466+makuche@users.noreply.github.com>
|
||||
ManuelPDM <153872964+ManuelPDM@users.noreply.github.com>
|
||||
maor-ps <154728172+maor-ps@users.noreply.github.com>
|
||||
Maphist0 <28743569+Maphist0@users.noreply.github.com>
|
||||
Marc Köhlbrugge <subscriptions@marckohlbrugge.com>
|
||||
@@ -1336,6 +1370,7 @@ Michael Hueschen <m@mhueschen.dev>
|
||||
Michael Kesper <mkesper@schokokeks.org>
|
||||
Michael Klimenko <mklimenko29@gmail.com>
|
||||
Michael Lamothe <michael.lamothe@gmail.com>
|
||||
Michael Paulauski <mike10010100@gmail.com>
|
||||
Michael Podvitskiy <podvitskiymichael@gmail.com>
|
||||
Michael Potter <NanoTekGuy@Gmail.com>
|
||||
Michael Taylor <162068037+mctylr-gh@users.noreply.github.com>
|
||||
@@ -1455,6 +1490,7 @@ Nikolay Popov <131475237+npopov-vst@users.noreply.github.com>
|
||||
Nils Gladitz <nilsgladitz@gmail.com>
|
||||
Nindaleth <Nindaleth@users.noreply.github.com>
|
||||
ningshanwutuobang <ningshanwutuobang@gmail.com>
|
||||
Nisbenz <benazzaanis783@gmail.com>
|
||||
Noah <99681487+NoahOksuz@users.noreply.github.com>
|
||||
nold <Nold360@users.noreply.github.com>
|
||||
nopperl <54780682+nopperl@users.noreply.github.com>
|
||||
@@ -1462,6 +1498,7 @@ nullname <chraac@gmail.com>
|
||||
Nuno <rare-magma@posteo.eu>
|
||||
nuri <yoonuri1@gmail.com>
|
||||
nusu-github <29514220+nusu-github@users.noreply.github.com>
|
||||
nv-vankit <162085734+nv-vankit@users.noreply.github.com>
|
||||
nwyin <tommynguyen0512@gmail.com>
|
||||
o7si <32285332+o7si@users.noreply.github.com>
|
||||
Oğuzhan Akkaya <oakkaya@ymail.com>
|
||||
@@ -1485,6 +1522,7 @@ oobabooga <oobabooga4@gmail.com>
|
||||
opparco <parco.opaai@gmail.com>
|
||||
Ori Pekelman <ori@pekelman.com>
|
||||
Oscar Barenys <rtfss1@gmail.com>
|
||||
Oscar M. Herrera <oscar@kss-inc.com>
|
||||
OSecret <135510162+OLSecret@users.noreply.github.com>
|
||||
ostix360 <55257054+ostix360@users.noreply.github.com>
|
||||
Ouadie EL FAROUKI <ouadie.elfarouki@codeplay.com>
|
||||
@@ -1611,6 +1649,7 @@ Raj Hammeer Singh Hada <hammeerraj@gmail.com>
|
||||
Rajendra Matcha <matcraje@qti.qualcomm.com>
|
||||
Ralph Soika <ralph.soika@imixs.com>
|
||||
Raman Shinde <raman.shinde15@gmail.com>
|
||||
Ramzi <146260554+rmz-24@users.noreply.github.com>
|
||||
Rand Xie <randxiexyy29@gmail.com>
|
||||
Randall Fitzgerald <randall@dasaku.net>
|
||||
Random Fly <renfei8@live.cn>
|
||||
@@ -1651,6 +1690,7 @@ Rich Dougherty <rich@rd.nz>
|
||||
Richard <r-burton@hotmail.co.uk>
|
||||
Richard Davison <richard.davison1@gmail.com>
|
||||
Richard Kiss <him@richardkiss.com>
|
||||
Richard Palethorpe <io@richiejp.com>
|
||||
Richard Roberson <richardr1126@gmail.com>
|
||||
RichardScottOZ <rnmscott@netspace.net.au>
|
||||
Rick G <26732651+TheFlipbook@users.noreply.github.com>
|
||||
@@ -1695,6 +1735,7 @@ Roni <sulpher@gmx.net>
|
||||
Ronny Brendel <ronnyb@nvidia.com>
|
||||
Ronny Brendel <ronnybrendel@gmail.com>
|
||||
Ronsor <ronsor@ronsor.pw>
|
||||
Roshan Kumar <52444089+roshankumarg529@users.noreply.github.com>
|
||||
Rotem Dan <rotemdan@gmail.com>
|
||||
Rowan Hart <rowanbhart@gmail.com>
|
||||
rspOverflow <217881046+rspOverflow@users.noreply.github.com>
|
||||
@@ -1727,9 +1768,11 @@ Saba Fallah <10401143+sfallah@users.noreply.github.com>
|
||||
Saba Fallah <sabafallah@gmail.com>
|
||||
Sachin Desai <smdesai@gmail.com>
|
||||
Sachin Sharma <sachin@zettabolt.com>
|
||||
Saejin Kim-Heinert <saejinmh@gmail.com>
|
||||
Safi Ullah <safiullah.3915@gmail.com>
|
||||
safranowith <bsh155762@gmail.com>
|
||||
Said Taghadouini <84044788+staghado@users.noreply.github.com>
|
||||
SaidSultani <62264302+SaidSultani@users.noreply.github.com>
|
||||
Sait Furkan Teke <35101659+stfurkan@users.noreply.github.com>
|
||||
SakuraUmi <yukinon244@gmail.com>
|
||||
Salvador E. Tropea <stropea@inti.gob.ar>
|
||||
@@ -1754,6 +1797,7 @@ Sang-Kil Park <sang.park@42dot.ai>
|
||||
Sanjay Ahari <sanjayahari1704@gmail.com>
|
||||
Sarah Wu <sarahwu185@gmail.com>
|
||||
Sarah Wu <sarwu@nvidia.com>
|
||||
sargentmajord <jasonlather13@gmail.com>
|
||||
Sascha Rogmann <59577610+srogmann@users.noreply.github.com>
|
||||
sasha0552 <admin@sasha0552.org>
|
||||
Satinder Grewal <grewal.satinder@gmail.com>
|
||||
@@ -1786,6 +1830,7 @@ Sergiu <8598216+mzsergiu@users.noreply.github.com>
|
||||
serhii-nakon <57632032+serhii-nakon@users.noreply.github.com>
|
||||
Sertaç Özercan <852750+sozercan@users.noreply.github.com>
|
||||
seryogakovalyov <seryogakovalyov@gmail.com>
|
||||
Seungjun Bae <linkedlatch@gmail.com>
|
||||
Seungmin Kim <8457324+ehfd@users.noreply.github.com>
|
||||
SeungWon Jeong <65549245+redlion0929@users.noreply.github.com>
|
||||
Seyoung Jeong <seyoungjeong@gmail.com>
|
||||
@@ -1871,6 +1916,7 @@ SmartestWashingMachine <ottobizness@gmail.com>
|
||||
smugman-dot <wbsmoke101@gmail.com>
|
||||
SnA1lGo <44647694+skrandy@users.noreply.github.com>
|
||||
snadampal <87143774+snadampal@users.noreply.github.com>
|
||||
Sofian Elmotiem <sofianelmotiem@gmail.com>
|
||||
SoftwareRenderer <138734813+SoftwareRenderer@users.noreply.github.com>
|
||||
Someone <sergei.kozlukov@aalto.fi>
|
||||
Someone Serge <sergei.kozlukov@aalto.fi>
|
||||
@@ -1886,6 +1932,7 @@ sprayandwipe <spray.and.wipe@gmail.com>
|
||||
Sreeram <sreeram.sivasubramony@amd.com>
|
||||
SRHMorris <69468379+SRHMorris@users.noreply.github.com>
|
||||
Srihari-mcw <96763064+Srihari-mcw@users.noreply.github.com>
|
||||
Srijan Keshri <212402043+arcusbuilds@users.noreply.github.com>
|
||||
Srinivas Billa <nivibilla@gmail.com>
|
||||
srkizer <github.soreepeong.prqovzj1d@srkizer.net>
|
||||
ssweens <1149151+ssweens@users.noreply.github.com>
|
||||
@@ -1919,6 +1966,7 @@ Sutou Kouhei <kou@cozmixng.org>
|
||||
Svetlozar Georgiev <55534064+sgeor255@users.noreply.github.com>
|
||||
Swigler <124839156+Swigler@users.noreply.github.com>
|
||||
swittk <switt1995@gmail.com>
|
||||
sxq <18856096915@163.com>
|
||||
SXX <song_xiaoxi@126.com>
|
||||
SXX <sxx1136965276@gmail.com>
|
||||
Tai Duc Nguyen <taiducnguyen.drexel@gmail.com>
|
||||
@@ -2044,6 +2092,7 @@ Ulrich Drepper <drepper@gmail.com>
|
||||
unbounded <haakon@likedan.net>
|
||||
Unbug Lee <tidelgl@gmail.com>
|
||||
unraido <127105806+unraido@users.noreply.github.com>
|
||||
usrlocalben <benjamin@rqdq.com>
|
||||
uvos <carl@uvos.xyz>
|
||||
uvos <devnull@uvos.xyz>
|
||||
uvos <philipp@uvos.xyz>
|
||||
@@ -2084,6 +2133,7 @@ vk <89937361+itsvedantkumar@users.noreply.github.com>
|
||||
Vlad <spitfireage@gmail.com>
|
||||
Vladimir <bogdad@gmail.com>
|
||||
Vladimir Malyutin <first-leon@yandex.ru>
|
||||
Vladimir Sotnikov <bubert9@gmail.com>
|
||||
Vladimir Vuksanovic <109677816+vvuksanovic@users.noreply.github.com>
|
||||
Vladimir Zorin <vladimir@deviant.guru>
|
||||
Vladislav <vladplotnikov34@gmail.com>
|
||||
@@ -2139,6 +2189,7 @@ Wroclaw <wroclaw223@outlook.com>
|
||||
wsbagnsv1 <sclumpfpapa36@gmail.com>
|
||||
Wu Jian Ping <wujjpp@hotmail.com>
|
||||
Wu Jian Ping <wujp@greatld.com>
|
||||
WuXin <apuravmanhas123@gmail.com>
|
||||
wwoodsTM <104587230+wwoodsTM@users.noreply.github.com>
|
||||
Wyatt Caldwell <218154709+Detensable@users.noreply.github.com>
|
||||
wzy <32936898+Freed-Wu@users.noreply.github.com>
|
||||
@@ -2162,6 +2213,7 @@ Xinpeng Dou <15529241576@163.com>
|
||||
Xinpeng Dou <81913537+Dou-Git@users.noreply.github.com>
|
||||
xloem <0xloem@gmail.com>
|
||||
xris99 <79798089+xris99@users.noreply.github.com>
|
||||
Xuan Son Nguyen <son@huggingface.co>
|
||||
Xuan Son Nguyen <thichthat@gmail.com>
|
||||
Xuan-Son Nguyen <son@huggingface.co>
|
||||
Xuan-Son Nguyen <thichthat@gmail.com>
|
||||
@@ -2247,6 +2299,7 @@ zhangrunda <zhangrunda1234@outlook.com>
|
||||
zhangtao2-1 <478679312@qq.com>
|
||||
ZHAOKAI WANG <sanxianwei@163.com>
|
||||
Zhaolun Yin <129580161+ZhaolunYin@users.noreply.github.com>
|
||||
zhaoyshine <30611065+zhaoyshine@users.noreply.github.com>
|
||||
Zheng.Deng <32841220+dengzheng-cloud@users.noreply.github.com>
|
||||
zhentaoyu <zhentao.yu@intel.com>
|
||||
Zhenwei Jin <109658203+kylo5aby@users.noreply.github.com>
|
||||
|
||||
+1
-1
@@ -134,7 +134,7 @@ option(LLAMA_BUILD_TOOLS "llama: build tools"
|
||||
option(LLAMA_BUILD_EXAMPLES "llama: build examples" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_SERVER "llama: build server example" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_APP "llama: build the unified binary" ${LLAMA_STANDALONE})
|
||||
option(LLAMA_BUILD_UI "llama: build the embedded Web UI for server" OFF)
|
||||
option(LLAMA_BUILD_UI "llama: build Llama Chat, the embedded chat app, for server" OFF)
|
||||
option(LLAMA_USE_PREBUILT_UI "llama: use prebuilt UI from HF Bucket when available" ON)
|
||||
option(LLAMA_CONNECT "llama: include the llama-connect tool (downloads a prebuilt binary)" OFF)
|
||||
|
||||
|
||||
+1
-1
@@ -102,7 +102,7 @@
|
||||
/tools/rpc/ @ggml-org/ggml-rpc
|
||||
/tools/server/* @ggml-org/llama-server # no subdir
|
||||
/tools/server/tests/ @ggml-org/llama-server
|
||||
/tools/ui/ @ggml-org/llama-ui
|
||||
/ui/ @ggml-org/llama-ui
|
||||
/tools/tokenize/ @ggerganov
|
||||
/tools/tts/ @ggerganov
|
||||
/vendor/ @ggerganov
|
||||
|
||||
@@ -16,6 +16,8 @@ The project differentiates between 3 levels of contributors:
|
||||
>
|
||||
> Detailed information regarding permissible and restricted uses of AI can be found in the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
### Requirements
|
||||
|
||||
If AI is used to generate any portion of the code, contributors must adhere to the following requirements:
|
||||
|
||||
1. Explicitly disclose the manner in which AI was employed.
|
||||
@@ -24,6 +26,16 @@ If AI is used to generate any portion of the code, contributors must adhere to t
|
||||
4. Be prepared to explain every line of code you submit when asked about it by a maintainer.
|
||||
5. It is strictly prohibited to use AI to write your posts for you (bug reports, feature requests, pull request descriptions, Github discussions, responding to humans, ...).
|
||||
|
||||
### Checklist
|
||||
|
||||
> [!IMPORTANT]
|
||||
>
|
||||
> **If you use AI, do all of the following:**
|
||||
>
|
||||
> 1. **Make sure [AGENTS.md](AGENTS.md) is loaded into your agent's context.** Skipping this step is one of the most common reasons PRs get rejected.
|
||||
> 2. **Search for related discussions, PRs and issues** to get the broader context before writing any code.
|
||||
> 3. **Run a local review pass before pushing anything**, using the [`code-review`](skills/code-review/SKILL.md) skill, once the change is implemented. Fix what it finds.
|
||||
|
||||
For more info, please refer to the [AGENTS.md](AGENTS.md) file.
|
||||
|
||||
# Pull requests (for contributors & collaborators)
|
||||
|
||||
@@ -13,10 +13,16 @@
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
|
||||
|
||||
[ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
[v1.0.0](https://huggingface.co/blog/ggml-org/llama-cpp-v1) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md) / [maintainer PRs](https://github.com/ggml-org/llama.cpp/pulls?q=is%3Apr%20is%3Aopen%20draft%3AFalse%20(author%3Asanmai%20OR%20author%3Argerganov%20OR%20author%3AKitaitiMakoto%20OR%20author%3Adanbev%20OR%20author%3Aaldehir%20OR%20author%3Amax-krasnyansky%20OR%20author%3ACISC%20OR%20author%3Aggerganov%20OR%20author%3Aam17an%20OR%20author%3Ajhen0409%20OR%20author%3Abartowski1182%20OR%20author%3Anikwen%20OR%20author%3Ahipudding%20OR%20author%3Aravi9%20OR%20author%3AServeurpersoCom%20OR%20author%3Athelittlefireman%20OR%20author%3Apwilkin%20OR%20author%3Areeselevine%20OR%20author%3Angxson%20OR%20author%3Ajeffbolznv%20OR%20author%3Amarty1885%20OR%20author%3A0cc4m%20OR%20author%3ATitaniumtown%20OR%20author%3Aangt%20OR%20author%3AIMbackK%20OR%20author%3Aarthw%20OR%20author%3AJohannesGaessler%20OR%20author%3AORippler%20OR%20author%3Aruixiang63%20OR%20author%3Axctan%20OR%20author%3AmasterFoad%20OR%20author%3Aallozaur%20OR%20author%3AAndreas-Krebbel%20OR%20author%3Ayomaytk%20OR%20author%3Aaendk%20OR%20author%3Awine99%20OR%20author%3Agaugarg-nv%20OR%20author%3Ataronaeo%20OR%20author%3Aforforever73%20OR%20author%3Achaxu01%20OR%20author%3Alhez%20OR%20author%3AAlekseiNikiforovIBM%20OR%20author%3Anetrunnereve%20OR%20author%3Afairydreaming%20OR%20author%3Aynankani)%20sort%3Aupdated-desc) / [dev stats](https://github.com/ggml-org/llama.cpp-dev) / [lib llama API](https://github.com/ggml-org/llama.cpp/issues/9289) / [llama-server REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
||||
|
||||
</div>
|
||||
|
||||
## Description
|
||||
|
||||
The main goal of `llama.cpp` is to enable LLM (and MLLM) inference with minimal setup and state-of-the-art performance on a wide range of hardware - locally and in the cloud.
|
||||
|
||||
The `llama.cpp` project is built on top of the [ggml](https://github.com/ggml-org/ggml) library.
|
||||
|
||||
## Quick start
|
||||
|
||||
A few options to get `llama.cpp` installed on your machine:
|
||||
@@ -51,50 +57,12 @@ llama serve -hf ggml-org/Qwen3.5-0.8B-GGUF
|
||||
<i>VLM session with <b>llama cli</b></i>
|
||||
</td>
|
||||
<td align="center">
|
||||
<img width="1392" height="958" alt="Built-in web UI against `llama serve` running Qwen 3.6" src="https://github.com/user-attachments/assets/b402f972-2e32-4def-8771-8d849f08cf2e" />
|
||||
<i>Built-in web UI against <b>llama serve</b></i>
|
||||
<img width="1392" height="958" alt="Llama Chat, the built-in chat app, against `llama serve` running Qwen 3.6" src="https://github.com/user-attachments/assets/b402f972-2e32-4def-8771-8d849f08cf2e" />
|
||||
<i>Llama Chat, the built-in chat app, against <b>llama serve</b></i>
|
||||
</td>
|
||||
</tr>
|
||||
<table>
|
||||
|
||||
## Description
|
||||
|
||||
The main goal of `llama.cpp` is to enable LLM (and VLM) inference with minimal setup and state-of-the-art performance on
|
||||
a wide range of hardware - locally and in the cloud.
|
||||
|
||||
- Plain C/C++ implementation without any dependencies
|
||||
- Apple silicon is a first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks
|
||||
- AVX, AVX2, AVX512 and AMX support for x86 architectures
|
||||
- RVV, ZVFH, ZFH, ZICBOP and ZIHINTPAUSE support for RISC-V architectures
|
||||
- 1.5-bit, 2-bit, 3-bit, 4-bit, 5-bit, 6-bit, and 8-bit integer quantization for faster inference and reduced memory use
|
||||
- Custom CUDA kernels for running LLMs on NVIDIA GPUs (support for AMD GPUs via HIP and Moore Threads GPUs via MUSA)
|
||||
- Vulkan and SYCL backend support
|
||||
- CPU+GPU hybrid inference to partially accelerate models larger than the total VRAM capacity
|
||||
|
||||
The `llama.cpp` project is build on top of the [ggml](https://github.com/ggml-org/ggml) library.
|
||||
|
||||
## Supported backends
|
||||
|
||||
| Backend | Target devices |
|
||||
| --- | --- |
|
||||
| [BLAS](docs/build.md#blas-build) | All |
|
||||
| [BLIS](docs/backend/BLIS.md) | All |
|
||||
| [CANN](docs/build.md#cann) | Huawei Ascend NPU |
|
||||
| [CUDA](docs/build.md#cuda) | Nvidia GPU |
|
||||
| [HIP](docs/build.md#hip) | AMD GPU |
|
||||
| [Hexagon](docs/backend/snapdragon/README.md) | Snapdragon |
|
||||
| [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
|
||||
| [MUSA](docs/build.md#musa) | Moore Threads GPU |
|
||||
| [Metal](docs/build.md#metal-build) | Apple Silicon |
|
||||
| [OpenCL](docs/backend/OPENCL.md) | Adreno GPU |
|
||||
| [OpenVINO [In Progress]](docs/backend/OPENVINO.md) | Intel CPUs, GPUs, and NPUs |
|
||||
| [RPC](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) | All |
|
||||
| [SYCL](docs/backend/SYCL.md) | Intel GPU |
|
||||
| [VirtGPU](docs/backend/VirtGPU.md) | VirtGPU APIR |
|
||||
| [Vulkan](docs/build.md#vulkan) | GPU |
|
||||
| [WebGPU](docs/build.md#webgpu) | All |
|
||||
| [ZenDNN](docs/build.md#zendnn) | AMD CPU |
|
||||
|
||||
## Documentation
|
||||
|
||||
#### Tools
|
||||
|
||||
+1
-1
@@ -46,7 +46,7 @@ Only vulnerabilities that fall within these parts of the project are considered
|
||||
- `ggml/**/*`
|
||||
- `gguf-py/**/*`
|
||||
- `tools/server/*`, **excluding** the following topics:
|
||||
- Web UI
|
||||
- Llama Chat, the built-in chat app (`ui/apps/chat`)
|
||||
- Features marked as experimental
|
||||
- Features not recommended for use in untrusted environments (e.g., router, MCP)
|
||||
- Bugs that can lead to Denial-of-Service attack
|
||||
|
||||
+12
-4
@@ -2829,7 +2829,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
const std::regex regex{ R"([,/]+)" };
|
||||
std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
|
||||
std::vector<std::string> split_arg{ it, {} };
|
||||
if (split_arg.size() >= llama_max_devices()) {
|
||||
if (split_arg.size() > llama_max_devices()) {
|
||||
throw std::invalid_argument(
|
||||
string_format("got %zu input configs, but system only has %zu devices", split_arg.size(), llama_max_devices())
|
||||
);
|
||||
@@ -2895,7 +2895,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
const std::regex regex{ R"([,/]+)" };
|
||||
std::sregex_token_iterator it{ arg_next.begin(), arg_next.end(), regex, -1 };
|
||||
std::vector<std::string> split_arg{ it, {} };
|
||||
if (split_arg.size() >= llama_max_devices()) {
|
||||
if (split_arg.size() > llama_max_devices()) {
|
||||
throw std::invalid_argument(
|
||||
string_format("got %zu input configs, but system only has %zu devices", split_arg.size(), llama_max_devices())
|
||||
);
|
||||
@@ -3416,7 +3416,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui-mcp-proxy", "--webui-mcp-proxy"},
|
||||
{"--no-ui-mcp-proxy", "--no-webui-mcp-proxy"},
|
||||
"experimental: whether to enable MCP CORS proxy, for the Web UI to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)\n"
|
||||
"experimental: whether to enable MCP CORS proxy, for Llama Chat to reach public MCP servers without CORS headers; local and private addresses are refused - do not enable in untrusted environments (default: disabled)\n"
|
||||
"note: for security reasons, this will limit --cors-origins to localhost by default",
|
||||
[](common_params & params, bool value) {
|
||||
params.ui_mcp_proxy = value;
|
||||
@@ -3485,7 +3485,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
add_opt(common_arg(
|
||||
{"--ui", "--webui"},
|
||||
{"--no-ui", "--no-webui"},
|
||||
string_format("whether to enable the Web UI (default: %s)", params.ui ? "enabled" : "disabled"),
|
||||
string_format("whether to enable Llama Chat, the built-in chat app (default: %s)", params.ui ? "enabled" : "disabled"),
|
||||
[](common_params & params, bool value) {
|
||||
params.ui = value;
|
||||
}
|
||||
@@ -4522,6 +4522,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
|
||||
params.tts_speaker_file = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
add_opt(common_arg(
|
||||
{"--tts-speaker-text"}, "TEXT",
|
||||
"transcript of --tts-speaker-file, for models that need it to clone the voice\n"
|
||||
"see docs/multimodal/tts.md for per-model usage notes",
|
||||
[](common_params & params, const std::string & value) {
|
||||
params.tts_speaker_text = value;
|
||||
}
|
||||
).set_examples({LLAMA_EXAMPLE_TTS}));
|
||||
|
||||
//
|
||||
// diffusion params
|
||||
|
||||
+7
-3
@@ -1444,6 +1444,12 @@ static common_chat_params_init_fn common_chat_template_detect_params_init(const
|
||||
return common_chat_params_init_qwen3_coder;
|
||||
}
|
||||
|
||||
// Inkling 1 and 2
|
||||
if (is_inkling_template(src)) {
|
||||
LOG_DBG("Using specialized template: Inkling\n");
|
||||
return common_chat_params_init_inkling;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
@@ -1626,9 +1632,6 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_
|
||||
auto_params.thinking_end_tags = {std::move(end_tag)};
|
||||
}
|
||||
}
|
||||
if (is_inkling2_template(src)) {
|
||||
common_chat_inkling2_split_media(auto_params.prompt);
|
||||
}
|
||||
const auto & arena = auto_params.parser;
|
||||
LOG_DBG("%s: generated parser:\n%s\n\nparser generation prompt: %s\n", __func__, arena.dump(arena.root()).c_str(), auto_params.generation_prompt.c_str());
|
||||
return auto_params;
|
||||
@@ -1979,5 +1982,6 @@ common_json common_chat_templates_get_caps(const common_chat_templates * chat_te
|
||||
: chat_templates->template_default->caps;
|
||||
common_json result = caps.to_map();
|
||||
result["reasoning_effort_levels"] = caps.reasoning_effort_levels;
|
||||
result["reasoning_effort_rejected"] = caps.reasoning_effort_rejected;
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -796,6 +796,7 @@ struct common_params {
|
||||
std::string tts_lang = "";
|
||||
std::string tts_voice = "";
|
||||
std::string tts_speaker_file = "";
|
||||
std::string tts_speaker_text = "";
|
||||
|
||||
bool is_gen_docs = false; // whether we are running inside llama-gen-docs
|
||||
};
|
||||
|
||||
+11
-1
@@ -627,9 +627,14 @@ caps caps_get(jinja::program & prog) {
|
||||
// levels sharing a render are aliases and the highest one is kept
|
||||
const auto unknown = render("REASONING_EFFORT_UNKNOWN");
|
||||
std::vector<std::string> renders;
|
||||
std::vector<std::string> rejected;
|
||||
for (const auto & level : candidates) {
|
||||
const auto rendered = render(level);
|
||||
if (!rendered || rendered == unknown) {
|
||||
if (!rendered) {
|
||||
rejected.push_back(level);
|
||||
continue;
|
||||
}
|
||||
if (rendered == unknown) {
|
||||
continue;
|
||||
}
|
||||
const auto it = std::find(renders.begin(), renders.end(), *rendered);
|
||||
@@ -640,6 +645,11 @@ caps caps_get(jinja::program & prog) {
|
||||
result.reasoning_effort_levels.push_back(level);
|
||||
}
|
||||
}
|
||||
|
||||
// a template that renders no level fails the probe for another reason, so its failures say nothing about the levels
|
||||
if (!result.reasoning_effort_levels.empty()) {
|
||||
result.reasoning_effort_rejected = rejected;
|
||||
}
|
||||
}
|
||||
|
||||
JJ_DEBUG("%s\n", result.to_string().c_str());
|
||||
|
||||
@@ -23,6 +23,9 @@ struct caps {
|
||||
// reasoning effort levels the template renders distinctly, from lowest to highest
|
||||
std::vector<std::string> reasoning_effort_levels;
|
||||
|
||||
// reasoning effort levels the template raises on
|
||||
std::vector<std::string> reasoning_effort_rejected;
|
||||
|
||||
// one of the 2 content capabilities must be true
|
||||
bool supports_string_content = true;
|
||||
bool supports_typed_content = false;
|
||||
|
||||
@@ -811,13 +811,14 @@ value call_statement::execute_impl(context & ctx) const {
|
||||
}
|
||||
auto * callee_func = cast_val<value_func>(callee_val);
|
||||
|
||||
context caller_ctx(ctx); // new scope for caller execution
|
||||
// caller() can be saved and called after this scope ended, it owns a snapshot of the variables
|
||||
auto caller_ctx = std::make_shared<context>(ctx.snapshot());
|
||||
|
||||
const func_handler func = [this, caller_ctx = std::move(caller_ctx)](const func_args & args) -> value {
|
||||
const func_handler func = [this, caller_ctx](const func_args & args) -> value {
|
||||
if (args.ctx.macro_depth >= JINJA_RECURSIVE_LIMIT) {
|
||||
throw std::runtime_error("macro calls are nested more than " + std::to_string(JINJA_RECURSIVE_LIMIT) + " levels deep");
|
||||
}
|
||||
context block_ctx(caller_ctx); // new scope for block execution
|
||||
context block_ctx(*caller_ctx); // new scope for block execution
|
||||
block_ctx.macro_depth = args.ctx.macro_depth + 1;
|
||||
|
||||
bind_parameters("caller", this->caller_args, args, block_ctx);
|
||||
|
||||
+30
-3
@@ -81,12 +81,38 @@ struct context {
|
||||
current_time(parent.current_time),
|
||||
is_get_stats(parent.is_get_stats),
|
||||
macro_depth(parent.macro_depth),
|
||||
// inherit variables (for example, when entering a new scope)
|
||||
env(mk_val<value_object>(*parent.env)) {}
|
||||
// child scope: own variables only, lookups fall back to the parent (which must outlive this scope)
|
||||
parent(&parent),
|
||||
env(mk_val<value_object>()) {
|
||||
env->has_builtins = false;
|
||||
}
|
||||
|
||||
context(context &&) = default;
|
||||
|
||||
// copy with all visible variables flattened in, for a scope that can outlive its parents (caller() of a call block)
|
||||
context snapshot() const {
|
||||
std::vector<const context *> chain;
|
||||
for (const context * c = this; c; c = c->parent) {
|
||||
chain.push_back(c);
|
||||
}
|
||||
context res(*this);
|
||||
res.parent = nullptr;
|
||||
for (auto it = chain.rbegin(); it != chain.rend(); ++it) {
|
||||
for (const auto & kv : (*it)->env->as_ordered_object()) {
|
||||
res.env->insert(kv.first, kv.second);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
// returns nullptr if not found
|
||||
value find_val(const value & key) {
|
||||
return env->find(key);
|
||||
for (const context * c = this; c; c = c->parent) {
|
||||
if (value v = c->env->find(key)) {
|
||||
return v;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
value get_val(const std::string & name) {
|
||||
@@ -107,6 +133,7 @@ struct context {
|
||||
}
|
||||
|
||||
private:
|
||||
const context * parent = nullptr;
|
||||
value_object env;
|
||||
};
|
||||
|
||||
|
||||
+20
-14
@@ -281,14 +281,12 @@ static value tojson(const func_args & args) {
|
||||
if (is_val<value_int>(val_indent)) {
|
||||
indent = static_cast<int>(val_indent->as_int());
|
||||
}
|
||||
if (val_sort->as_bool()) { // undefined == false
|
||||
throw not_implemented_exception("tojson sort_keys=true not implemented");
|
||||
}
|
||||
const bool sort_keys = val_sort->as_bool(); // undefined == false
|
||||
const bool ensure_ascii = val_ascii->as_bool(); // undefined == false
|
||||
auto separators = (is_val<value_array>(val_separators) ? val_separators : mk_val<value_array>())->as_array();
|
||||
std::string item_sep = separators.size() > 0 ? separators[0]->as_string().str() : (indent < 0 ? ", " : ",");
|
||||
std::string key_sep = separators.size() > 1 ? separators[1]->as_string().str() : ": ";
|
||||
std::string json_str = value_to_json(args.get_pos(0), indent, item_sep, key_sep);
|
||||
std::string json_str = value_to_json(args.get_pos(0), indent, item_sep, key_sep, sort_keys);
|
||||
if (ensure_ascii) {
|
||||
json_str = json_ensure_ascii_preserving_format(json_str);
|
||||
}
|
||||
@@ -1761,7 +1759,7 @@ static std::string float_json(double val) {
|
||||
|
||||
// recursively convert value to JSON string
|
||||
// TODO: avoid circular references
|
||||
static void value_to_json_internal(std::ostringstream & oss, const value & val, int curr_lvl, int indent, const std::string_view item_sep, const std::string_view key_sep) {
|
||||
static void value_to_json_internal(std::ostringstream & oss, const value & val, int curr_lvl, int indent, const std::string_view item_sep, const std::string_view key_sep, bool sort_keys) {
|
||||
auto indent_str = [indent, curr_lvl]() -> std::string {
|
||||
return (indent > 0) ? std::string(curr_lvl * indent, ' ') : "";
|
||||
};
|
||||
@@ -1808,7 +1806,7 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
|
||||
oss << newline();
|
||||
for (size_t i = 0; i < arr.size(); ++i) {
|
||||
oss << indent_str() << (indent > 0 ? std::string(indent, ' ') : "");
|
||||
value_to_json_internal(oss, arr[i], curr_lvl + 1, indent, item_sep, key_sep);
|
||||
value_to_json_internal(oss, arr[i], curr_lvl + 1, indent, item_sep, key_sep, sort_keys);
|
||||
if (i < arr.size() - 1) {
|
||||
oss << item_sep;
|
||||
}
|
||||
@@ -1819,17 +1817,25 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
|
||||
oss << "]";
|
||||
} else if (is_val<value_object>(val)) {
|
||||
const auto & obj = val->as_ordered_object(); // IMPORTANT: need to keep exact order
|
||||
std::vector<std::pair<std::string, value>> items;
|
||||
items.reserve(obj.size());
|
||||
for (const auto & pair : obj) {
|
||||
items.emplace_back(is_val<value_float>(pair.first) ? float_json(pair.first->as_float()) : pair.first->as_string().str(), pair.second);
|
||||
}
|
||||
if (sort_keys) {
|
||||
// same order as python json.dumps(sort_keys=True): utf-8 byte order is code point order
|
||||
std::stable_sort(items.begin(), items.end(), [](const auto & a, const auto & b) { return a.first < b.first; });
|
||||
}
|
||||
oss << "{";
|
||||
if (!obj.empty()) {
|
||||
if (!items.empty()) {
|
||||
oss << newline();
|
||||
size_t i = 0;
|
||||
for (const auto & pair : obj) {
|
||||
for (const auto & [key, item] : items) {
|
||||
oss << indent_str() << (indent > 0 ? std::string(indent, ' ') : "");
|
||||
const std::string key = is_val<value_float>(pair.first) ? float_json(pair.first->as_float()) : pair.first->as_string().str();
|
||||
value_to_json_internal(oss, mk_val<value_string>(key), curr_lvl + 1, indent, item_sep, key_sep);
|
||||
value_to_json_internal(oss, mk_val<value_string>(key), curr_lvl + 1, indent, item_sep, key_sep, sort_keys);
|
||||
oss << key_sep;
|
||||
value_to_json_internal(oss, pair.second, curr_lvl + 1, indent, item_sep, key_sep);
|
||||
if (i < obj.size() - 1) {
|
||||
value_to_json_internal(oss, item, curr_lvl + 1, indent, item_sep, key_sep, sort_keys);
|
||||
if (i < items.size() - 1) {
|
||||
oss << item_sep;
|
||||
}
|
||||
oss << newline();
|
||||
@@ -1843,9 +1849,9 @@ static void value_to_json_internal(std::ostringstream & oss, const value & val,
|
||||
}
|
||||
}
|
||||
|
||||
std::string value_to_json(const value & val, int indent, const std::string_view item_sep, const std::string_view key_sep) {
|
||||
std::string value_to_json(const value & val, int indent, const std::string_view item_sep, const std::string_view key_sep, bool sort_keys) {
|
||||
std::ostringstream oss;
|
||||
value_to_json_internal(oss, val, 0, indent, item_sep, key_sep);
|
||||
value_to_json_internal(oss, val, 0, indent, item_sep, key_sep, sort_keys);
|
||||
JJ_DEBUG("value_to_json: result=%s", oss.str().c_str());
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
@@ -199,7 +199,7 @@ value find_builtin_func(context & ctx, const std::string & name, value & input);
|
||||
value try_builtin_func(context & ctx, const std::string & name, value & input, bool undef_on_missing = false);
|
||||
value apply_filter_by_name(context & ctx, std::string filter_id, value input, const func_args & args);
|
||||
|
||||
std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": ");
|
||||
std::string value_to_json(const value & val, int indent = -1, const std::string_view item_sep = ", ", const std::string_view key_sep = ": ", bool sort_keys = false);
|
||||
|
||||
// Note: only used for debugging purposes
|
||||
std::string value_to_string_repr(const value & val);
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
#include "parsers.h"
|
||||
|
||||
bool is_inkling_template(const std::string & src) {
|
||||
return src.find("<|message_user|>") != std::string::npos &&
|
||||
src.find("<|content_image|>") != std::string::npos;
|
||||
}
|
||||
|
||||
// the template makes one message block per content part, but a media marker comes inside the text of a message
|
||||
// <header><|content_text|>{media}text<|end_message|> --> <header>{media}<|end_message|><header><|content_text|>text<|end_message|>
|
||||
static void inkling_split_media(common_chat_prompt & prompt) {
|
||||
const std::string TEXT = "<|content_text|>";
|
||||
const std::string END = "<|end_message|>";
|
||||
const std::string MEDIA_BEG = "<__media_";
|
||||
const std::string MEDIA_END = "__>";
|
||||
|
||||
std::vector<common_chat_prompt_part> out;
|
||||
for (size_t i = 0; i < prompt.parts.size(); i++) {
|
||||
const auto & part = prompt.parts[i];
|
||||
const bool in_text_block = part.is_input && part.text.find(MEDIA_BEG) != std::string::npos &&
|
||||
!out.empty() && !out.back().is_input && string_ends_with(out.back().text, TEXT) &&
|
||||
i + 1 < prompt.parts.size() && !prompt.parts[i + 1].is_input && string_starts_with(prompt.parts[i + 1].text, END);
|
||||
const size_t header_pos = in_text_block ? out.back().text.rfind("<|message_") : std::string::npos;
|
||||
if (header_pos == std::string::npos) {
|
||||
out.push_back(part);
|
||||
continue;
|
||||
}
|
||||
const std::string header = out.back().text.substr(header_pos, out.back().text.size() - TEXT.size() - header_pos);
|
||||
|
||||
std::string rest = part.text;
|
||||
for (bool first = true; !rest.empty(); first = false) {
|
||||
const size_t beg = rest.find(MEDIA_BEG);
|
||||
const size_t end = beg == std::string::npos ? beg : rest.find(MEDIA_END, beg);
|
||||
const bool is_media = beg == 0 && end != std::string::npos;
|
||||
const size_t len = is_media ? end + MEDIA_END.size() : (end == std::string::npos ? rest.size() : beg);
|
||||
if (!first) {
|
||||
out.push_back({ END + header + (is_media ? "" : TEXT), false });
|
||||
} else if (is_media) {
|
||||
// mtmd opens the media block
|
||||
out.back().text.resize(out.back().text.size() - TEXT.size());
|
||||
}
|
||||
out.push_back({ rest.substr(0, len), true });
|
||||
rest.erase(0, len);
|
||||
}
|
||||
}
|
||||
prompt.parts = std::move(out);
|
||||
}
|
||||
|
||||
// Inkling 1 and 2 - the assistant turn is a list of blocks, all closed by <|end_message|>:
|
||||
// <|message_model|><|content_thinking|>...<|end_message|>
|
||||
// <|message_model|><|content_text|>...<|end_message|>
|
||||
// <|message_model|>NAME<|content_invoke_tool_json|>{"name":"NAME","args":{...}}<|end_message|>
|
||||
common_chat_params common_chat_params_init_inkling(const common_chat_template & tmpl,
|
||||
const autoparser::generation_params & inputs) {
|
||||
common_chat_params data;
|
||||
|
||||
data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs);
|
||||
data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs);
|
||||
data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;
|
||||
data.supports_thinking = true;
|
||||
|
||||
inkling_split_media(data.prompt);
|
||||
|
||||
const std::string HEADER = "<|message_model|>";
|
||||
const std::string THINK = "<|content_thinking|>";
|
||||
const std::string TEXT = "<|content_text|>";
|
||||
const std::string TOOL_JSON = "<|content_invoke_tool_json|>";
|
||||
const std::string END = "<|end_message|>";
|
||||
|
||||
data.preserved_tokens = { HEADER, THINK, TEXT, TOOL_JSON, END };
|
||||
|
||||
data.thinking_start_tag = THINK;
|
||||
data.thinking_end_tags = { END };
|
||||
|
||||
data.message_delimiters = {
|
||||
{ COMMON_CHAT_ROLE_ASSISTANT, HEADER },
|
||||
{ COMMON_CHAT_ROLE_USER, "<|message_user|>" },
|
||||
{ COMMON_CHAT_ROLE_SYSTEM, "<|message_system|>" },
|
||||
{ COMMON_CHAT_ROLE_TOOL, "<|message_tool|>" },
|
||||
};
|
||||
|
||||
if (inputs.has_continuation()) {
|
||||
const auto & msg = inputs.continue_msg;
|
||||
|
||||
data.generation_prompt.clear();
|
||||
if (!msg.reasoning_content.empty()) {
|
||||
data.generation_prompt = HEADER + THINK + msg.reasoning_content;
|
||||
}
|
||||
if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
|
||||
if (!data.generation_prompt.empty()) {
|
||||
data.generation_prompt += END;
|
||||
}
|
||||
data.generation_prompt += HEADER + TEXT + msg.render_content();
|
||||
}
|
||||
|
||||
data.prompt.append(data.generation_prompt);
|
||||
}
|
||||
|
||||
auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
|
||||
auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object();
|
||||
auto use_tools = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;
|
||||
auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
|
||||
|
||||
data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
|
||||
auto header = p.literal(HEADER);
|
||||
auto end = p.literal(END);
|
||||
|
||||
auto think = p.literal(THINK);
|
||||
auto thinking = p.rule("thinking", extract_reasoning ? think + p.reasoning(p.until(END)) + end :
|
||||
p.atomic(p.content(think)) + p.content(p.until(END) + end));
|
||||
auto text = p.rule("text", p.literal(TEXT) + p.content(p.until(END)) + end);
|
||||
|
||||
if (has_response_format) {
|
||||
auto response_format = p.rule("response-format", p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema)));
|
||||
return p.zero_or_more(header + thinking) + header + p.literal(TEXT) + response_format + end;
|
||||
}
|
||||
|
||||
auto blocks = p.zero_or_more(header + (thinking | text));
|
||||
|
||||
if (!use_tools) {
|
||||
return blocks;
|
||||
}
|
||||
|
||||
auto tool_choice = p.choice();
|
||||
foreach_function(inputs.tools, [&](size_t tool_index, const json & tool) {
|
||||
const auto & function = tool.at("function");
|
||||
std::string name = function.at("name");
|
||||
|
||||
auto open = p.tool_name(p.literal(name)) + p.literal(TOOL_JSON) + p.literal("{\"name\":" + json(name).dump() + ",\"args\":");
|
||||
auto args = p.tool_args(p.json());
|
||||
|
||||
tool_choice |= p.rule("tool-" + std::to_string(tool_index), p.tool(p.tool_open(open) + args + p.tool_close(p.literal("}")) + end));
|
||||
});
|
||||
|
||||
// tool calls are the last blocks of the turn
|
||||
auto calls = inputs.parallel_tool_calls ? tool_choice + p.zero_or_more(header + tool_choice) : tool_choice;
|
||||
auto tool_calls = header + p.trigger_rule("tool-calls", calls);
|
||||
|
||||
if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) {
|
||||
return blocks + tool_calls;
|
||||
}
|
||||
|
||||
return blocks + p.optional(tool_calls);
|
||||
});
|
||||
|
||||
if (has_response_format || use_tools) {
|
||||
data.grammar_lazy = !has_response_format && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;
|
||||
data.grammar = build_grammar([&](const common_grammar_builder & builder) {
|
||||
data.parser.build_grammar(builder, data.grammar_lazy);
|
||||
});
|
||||
|
||||
if (data.grammar_lazy) {
|
||||
// a header followed by text and not by a content token is a tool call, the grammar starts at the name
|
||||
data.grammar_triggers = {
|
||||
{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, "<\\|message_model\\|>([^<])" },
|
||||
};
|
||||
// Inkling 1: the first header is in the generation prompt, and a lazy grammar does not get the prompt.
|
||||
// an output that starts with no content token is a tool call. not for a continuation, it starts with text
|
||||
if (!inputs.has_continuation()) {
|
||||
data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL, "[^<]+" });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
#include "parsers.h"
|
||||
|
||||
bool is_inkling2_template(const std::string & src) {
|
||||
return src.find("<|message_user|>") != std::string::npos &&
|
||||
src.find("<|content_image|>") != std::string::npos;
|
||||
}
|
||||
|
||||
// the template makes one message block per content part, but a media marker comes inside the text of a message
|
||||
// <header><|content_text|>{media}text<|end_message|> --> <header>{media}<|end_message|><header><|content_text|>text<|end_message|>
|
||||
void common_chat_inkling2_split_media(common_chat_prompt & prompt) {
|
||||
const std::string TEXT = "<|content_text|>";
|
||||
const std::string END = "<|end_message|>";
|
||||
const std::string MEDIA_BEG = "<__media_";
|
||||
const std::string MEDIA_END = "__>";
|
||||
|
||||
std::vector<common_chat_prompt_part> out;
|
||||
for (size_t i = 0; i < prompt.parts.size(); i++) {
|
||||
const auto & part = prompt.parts[i];
|
||||
const bool in_text_block = part.is_input && part.text.find(MEDIA_BEG) != std::string::npos &&
|
||||
!out.empty() && !out.back().is_input && string_ends_with(out.back().text, TEXT) &&
|
||||
i + 1 < prompt.parts.size() && !prompt.parts[i + 1].is_input && string_starts_with(prompt.parts[i + 1].text, END);
|
||||
const size_t header_pos = in_text_block ? out.back().text.rfind("<|message_") : std::string::npos;
|
||||
if (header_pos == std::string::npos) {
|
||||
out.push_back(part);
|
||||
continue;
|
||||
}
|
||||
const std::string header = out.back().text.substr(header_pos, out.back().text.size() - TEXT.size() - header_pos);
|
||||
|
||||
std::string rest = part.text;
|
||||
for (bool first = true; !rest.empty(); first = false) {
|
||||
const size_t beg = rest.find(MEDIA_BEG);
|
||||
const size_t end = beg == std::string::npos ? beg : rest.find(MEDIA_END, beg);
|
||||
const bool is_media = beg == 0 && end != std::string::npos;
|
||||
const size_t len = is_media ? end + MEDIA_END.size() : (end == std::string::npos ? rest.size() : beg);
|
||||
if (!first) {
|
||||
out.push_back({ END + header + (is_media ? "" : TEXT), false });
|
||||
} else if (is_media) {
|
||||
// mtmd opens the media block
|
||||
out.back().text.resize(out.back().text.size() - TEXT.size());
|
||||
}
|
||||
out.push_back({ rest.substr(0, len), true });
|
||||
rest.erase(0, len);
|
||||
}
|
||||
}
|
||||
prompt.parts = std::move(out);
|
||||
}
|
||||
@@ -41,10 +41,7 @@ std::string common_chat_template_generation_prompt_impl(
|
||||
|
||||
bool is_lfm2_template(const std::string & src);
|
||||
|
||||
bool is_inkling2_template(const std::string & src);
|
||||
|
||||
// give each media marker of a text block its own message block
|
||||
void common_chat_inkling2_split_media(common_chat_prompt & prompt);
|
||||
bool is_inkling_template(const std::string & src);
|
||||
|
||||
common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
@@ -60,6 +57,8 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &
|
||||
|
||||
common_chat_params common_chat_params_init_harmony_xml(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_inkling(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_k2_horizon(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
common_chat_params common_chat_params_init_kimi_k2(const common_chat_template & tmpl, const autoparser::generation_params & inputs);
|
||||
|
||||
@@ -10,7 +10,7 @@ set(LLAMA_CHAT_PARSERS_SOURCES
|
||||
${CMAKE_CURRENT_LIST_DIR}/gigachat-v3.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/gpt-oss.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/harmony-xml.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/inkling2.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/inkling.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/k2-horizon.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/kimi-k2.cpp
|
||||
${CMAKE_CURRENT_LIST_DIR}/kimi-k3.cpp
|
||||
|
||||
@@ -284,6 +284,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
|
||||
"PocketTTSModel": "pockettts",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
|
||||
@@ -336,6 +337,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"XverseForCausalLM": "xverse",
|
||||
"YoutuForCausalLM": "deepseek",
|
||||
"YoutuVLForConditionalGeneration": "deepseek",
|
||||
"fish_qwen3_omni": "fish_s2",
|
||||
"modeling_grove_moe.GroveMoeForCausalLM": "grovemoe",
|
||||
"modeling_sarvam_moe.SarvamMoEForCausalLM": "bailingmoe",
|
||||
}
|
||||
@@ -360,6 +362,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"Exaone4_5_ForConditionalGeneration": "exaone",
|
||||
"FishOmniModel": "drama3",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"Gemma3ForConditionalGeneration": "gemma",
|
||||
"Gemma3nForConditionalGeneration": "gemma",
|
||||
"Gemma4ForConditionalGeneration": "gemma",
|
||||
@@ -438,6 +441,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"VoxtralForConditionalGeneration": "ultravox",
|
||||
"VoxtralRealtimeForConditionalGeneration": "ultravox",
|
||||
"YoutuVLForConditionalGeneration": "youtuvl",
|
||||
"fish_qwen3_omni": "fish_s2",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -3488,6 +3488,9 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st
|
||||
# some models (llava_onevision2) only declare the architecture inside text_config
|
||||
if arch is None and text_config.get("architectures") is not None:
|
||||
arch = text_config["architectures"][0]
|
||||
# some configs only have a model_type (fishaudio/s2-pro)
|
||||
if arch is None:
|
||||
arch = hparams.get("model_type")
|
||||
if arch is None:
|
||||
raise ValueError("Failed to detect model architecture")
|
||||
return arch
|
||||
|
||||
+32
-19
@@ -18,6 +18,30 @@ _TOWER_SHARED = "audio_decoder_mtp_shared.layers."
|
||||
_TOWER_RESIDUAL = "audio_decoder_mtp.layers."
|
||||
|
||||
|
||||
def load_codec(dir_model) -> dict[str, Tensor]:
|
||||
sd = torch.load(dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
|
||||
return sd.get("state_dict", sd)
|
||||
|
||||
|
||||
def fold_conv(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# fold weight norm
|
||||
if prefix + ".conv.weight" in sd:
|
||||
return sd[prefix + ".conv.weight"].float()
|
||||
g = sd[prefix + ".conv.parametrizations.weight.original0"].float()
|
||||
v = sd[prefix + ".conv.parametrizations.weight.original1"].float()
|
||||
return g * v / v.norm(dim=(1, 2), keepdim=True)
|
||||
|
||||
|
||||
def fold_wn(g: Tensor, v: Tensor) -> Tensor:
|
||||
# fold old-style weight norm of a 1x1 conv, returns [out, in]
|
||||
return (g.float() * v.float() / v.float().norm(dim=(1, 2), keepdim=True)).squeeze(-1)
|
||||
|
||||
|
||||
def fold_snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
|
||||
alpha = alpha.float().flatten()
|
||||
return alpha, 1.0 / (alpha + 1e-9)
|
||||
|
||||
|
||||
@ModelBase.register("FishOmniModel")
|
||||
@ModelBase.example("fishaudio/drama-3")
|
||||
class Drama3TextModel(Qwen3_5MoeTextModel):
|
||||
@@ -36,6 +60,7 @@ class Drama3TextModel(Qwen3_5MoeTextModel):
|
||||
class Drama3MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
gen_projector_type = gguf.VisionProjectorType.DRAMA3_GEN
|
||||
|
||||
_LAYER_MAP = {
|
||||
"enorm.weight": (gguf.MODEL_TENSOR.A_GEN_CODE_ENORM, True),
|
||||
@@ -69,7 +94,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
cfg = self.hparams
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.DRAMA3_GEN)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(self.gen_projector_type)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(cfg["dim"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(cfg["intermediate_size"])
|
||||
@@ -153,32 +178,20 @@ class Drama3MmprojModel(MmprojModel):
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# only the DAC decoder, the encoder is for voice cloning (not supported yet)
|
||||
sd = torch.load(self.dir_model / "codec.pth", map_location="cpu", mmap=True, weights_only=True)
|
||||
sd = sd.get("state_dict", sd)
|
||||
sd = load_codec(self.dir_model)
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
def get(name: str) -> Tensor:
|
||||
return sd[name].float()
|
||||
|
||||
def conv(prefix: str) -> Tensor:
|
||||
# fold weight norm
|
||||
if prefix + ".conv.weight" in sd:
|
||||
return get(prefix + ".conv.weight")
|
||||
g = get(prefix + ".conv.parametrizations.weight.original0")
|
||||
v = get(prefix + ".conv.parametrizations.weight.original1")
|
||||
return g * v / v.norm(dim=(1, 2), keepdim=True)
|
||||
return fold_conv(sd, prefix)
|
||||
|
||||
def quant_table(prefix: str) -> Tensor:
|
||||
# fold out_proj into the codebook, so decoding is a row lookup
|
||||
g = get(prefix + ".out_proj.weight_g")
|
||||
v = get(prefix + ".out_proj.weight_v")
|
||||
w = (g * v / v.norm(dim=(1, 2), keepdim=True)).squeeze(-1)
|
||||
w = fold_wn(sd[prefix + ".out_proj.weight_g"], sd[prefix + ".out_proj.weight_v"])
|
||||
return get(prefix + ".codebook.weight") @ w.T + get(prefix + ".out_proj.bias")
|
||||
|
||||
def snake(alpha: Tensor) -> tuple[Tensor, Tensor]:
|
||||
alpha = alpha.flatten()
|
||||
return alpha, 1.0 / (alpha + 1e-9)
|
||||
|
||||
q = "quantizer."
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), quant_table(q + "semantic_quantizer.quantizers.0"))
|
||||
n_rest = sum(1 for k in sd if k.startswith(q + "quantizer.quantizers.") and k.endswith(".codebook.weight"))
|
||||
@@ -228,7 +241,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get(d + "0.conv.bias"))
|
||||
for bid in range(4):
|
||||
p = f"{d}{bid + 1}.block."
|
||||
a, b = snake(get(p + "0.alpha"))
|
||||
a, b = fold_snake(get(p + "0.alpha"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), conv(p + "1"))
|
||||
@@ -236,7 +249,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
for xid in range(3):
|
||||
r = f"{p}{xid + 2}.block."
|
||||
for act, tensor in (("0", T.A_GEN_WAV_DAC_RES_ACT1), ("2", T.A_GEN_WAV_DAC_RES_ACT2)):
|
||||
a, b = snake(get(r + act + ".alpha"))
|
||||
a, b = fold_snake(get(r + act + ".alpha"))
|
||||
name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
|
||||
yield (name + ".alpha", a)
|
||||
yield (name + ".beta", b)
|
||||
@@ -244,7 +257,7 @@ class Drama3MmprojModel(MmprojModel):
|
||||
name = gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
|
||||
yield (name + ".weight", conv(r + cv))
|
||||
yield (name + ".bias", get(r + cv + ".conv.bias"))
|
||||
a, b = snake(get(d + "5.alpha"))
|
||||
a, b = fold_snake(get(d + "5.alpha"))
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b)
|
||||
yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), conv(d + "6"))
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, gguf
|
||||
from .drama3 import Drama3MmprojModel, fold_conv, fold_snake, fold_wn, load_codec
|
||||
from .qwen import Qwen3Model
|
||||
|
||||
# Fish Audio S2 Pro (dual-AR)
|
||||
# text model: Qwen3 backbone (slow AR), samples the semantic code of each frame
|
||||
# mmproj: fast AR code predictor (other codebooks) and Fish DAC decoder (codec.pth, same as drama3)
|
||||
|
||||
# residual codebook size of codec.pth, the fast AR head is cut to it
|
||||
_N_RESIDUAL_CODES = 1024
|
||||
|
||||
# codec encoder strides (encoder_rates of the codec config), the quantizer then downsamples 2x2
|
||||
_CODEC_ENC_STRIDES = (2, 4, 8, 8)
|
||||
|
||||
# codec encoder transformers, both 1024 wide with 16 heads of 64
|
||||
# generic audio layers a.blk.0..3: last encoder block, a.blk.4..11: quantizer pre_module
|
||||
_N_ENC_TFM_LAYERS = 4
|
||||
_N_PRE_TFM_LAYERS = 8
|
||||
_CODEC_HEAD_DIM = 64
|
||||
|
||||
|
||||
def _load_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
|
||||
cfg = hparams["text_config"]
|
||||
hparams["text_config"] = {
|
||||
"hidden_size": cfg["dim"],
|
||||
"num_hidden_layers": cfg["n_layer"],
|
||||
"num_attention_heads": cfg["n_head"],
|
||||
"num_key_value_heads": cfg["n_local_heads"],
|
||||
"head_dim": cfg["head_dim"],
|
||||
"intermediate_size": cfg["intermediate_size"],
|
||||
"rms_norm_eps": cfg["norm_eps"],
|
||||
"rope_theta": cfg["rope_base"],
|
||||
"max_position_embeddings": cfg["max_seq_len"],
|
||||
"vocab_size": cfg["vocab_size"],
|
||||
"tie_word_embeddings": cfg["tie_word_embeddings"],
|
||||
}
|
||||
return hparams
|
||||
|
||||
|
||||
def _neox_rows(w: Tensor, head_dim: int) -> Tensor:
|
||||
# checkpoint uses interleaved RoPE, reorder the rows of each head so NEOX RoPE gives the same result
|
||||
order = torch.cat([torch.arange(0, head_dim, 2), torch.arange(1, head_dim, 2)])
|
||||
return w.reshape(-1, head_dim, *w.shape[1:])[:, order].reshape(w.shape)
|
||||
|
||||
|
||||
@ModelBase.register("fish_qwen3_omni")
|
||||
@ModelBase.example("fishaudio/s2-pro")
|
||||
class FishS2TextModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# the rest is mapped by tensor_mapping.py
|
||||
name = name.removeprefix("text_model.")
|
||||
T = gguf.MODEL_TENSOR
|
||||
head_dim = self.hparams["head_dim"]
|
||||
if name.endswith(".attention.wqkv.weight"):
|
||||
n_q = self.hparams["num_attention_heads"] * head_dim
|
||||
n_kv = self.hparams["num_key_value_heads"] * head_dim
|
||||
q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
|
||||
yield (self.format_tensor_name(T.ATTN_Q, bid), _neox_rows(q, head_dim))
|
||||
yield (self.format_tensor_name(T.ATTN_K, bid), _neox_rows(k, head_dim))
|
||||
yield (self.format_tensor_name(T.ATTN_V, bid), v)
|
||||
return
|
||||
if name.endswith(".attention.q_norm.weight"):
|
||||
yield (self.format_tensor_name(T.ATTN_Q_NORM, bid), _neox_rows(data_torch, head_dim))
|
||||
return
|
||||
if name.endswith(".attention.k_norm.weight"):
|
||||
yield (self.format_tensor_name(T.ATTN_K_NORM, bid), _neox_rows(data_torch, head_dim))
|
||||
return
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
@ModelBase.register("fish_qwen3_omni")
|
||||
@ModelBase.example("fishaudio/s2-pro")
|
||||
class FishS2MmprojModel(Drama3MmprojModel):
|
||||
has_audio_encoder = True # codec encoder, for voice cloning
|
||||
gen_projector_type = gguf.VisionProjectorType.FISH_S2_GEN
|
||||
|
||||
_FAST_AR_MAP = {
|
||||
"attention_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
|
||||
"attention.wo.weight": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
|
||||
"ffn_norm.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
|
||||
"feed_forward.w1.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
|
||||
"feed_forward.w3.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP,
|
||||
"feed_forward.w2.weight": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
|
||||
}
|
||||
|
||||
def __init__(self, dir_model: Path, *args, **kwargs):
|
||||
kwargs.pop("hparams", None)
|
||||
super().__init__(dir_model, *args, hparams=_load_hparams(dir_model), **kwargs)
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
cfg = dict(self.global_config["audio_decoder_config"])
|
||||
cfg["num_hidden_layers"] = cfg["n_layer"]
|
||||
return cfg
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
# codec encoder: raw 44.1 kHz waveform -> backbone input rows, other hparams are hardcoded in tools/mtmd
|
||||
self.gguf_writer.add_clip_has_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.FISH_S2_SPKENC)
|
||||
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_audio_num_mel_bins(1)
|
||||
self.gguf_writer.add_audio_block_count(_N_ENC_TFM_LAYERS + _N_PRE_TFM_LAYERS)
|
||||
self.gguf_writer.add_audio_embedding_length(1024)
|
||||
self.gguf_writer.add_audio_feed_forward_length(3072)
|
||||
self.gguf_writer.add_audio_head_count(1024 // _CODEC_HEAD_DIM)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# the VQ picks the nearest codebook entry, small errors in the encoder change the codes
|
||||
if new_name.startswith("a.codec."):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# codec transformers: F16 gives codes closer to the reference than BF16
|
||||
if new_name.startswith("a.blk.") and n_dims == 2:
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name = item[0]
|
||||
if name.startswith("audio_decoder.") or name == "text_model.model.embeddings.weight":
|
||||
return item
|
||||
return None
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
T = gguf.MODEL_TENSOR
|
||||
cfg = self.hparams
|
||||
|
||||
# already named by generate_extra_tensors()
|
||||
if name.startswith("a."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
if name == "text_model.model.embeddings.weight":
|
||||
# semantic band of the backbone embeddings
|
||||
begin = self.global_config["semantic_start_token_id"]
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch[begin:begin + cfg["vocab_size"]])
|
||||
return
|
||||
|
||||
if name == "audio_decoder.codebook_embeddings.weight":
|
||||
# backbone input of a speech row, one table per codebook
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), data_torch.reshape(cfg["num_codebooks"], -1, data_torch.shape[-1]))
|
||||
return
|
||||
|
||||
if name == "audio_decoder.embeddings.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_TOK_EMBD), data_torch)
|
||||
return
|
||||
|
||||
if name == "audio_decoder.output.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_HEAD), data_torch[:_N_RESIDUAL_CODES])
|
||||
return
|
||||
|
||||
if name == "audio_decoder.norm.weight":
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_OUTPUT_NORM), data_torch)
|
||||
return
|
||||
|
||||
assert bid is not None and name.startswith(f"audio_decoder.layers.{bid}.")
|
||||
key = name.removeprefix(f"audio_decoder.layers.{bid}.")
|
||||
if key == "attention.wqkv.weight":
|
||||
head_dim = cfg["head_dim"]
|
||||
n_q = cfg["n_head"] * head_dim
|
||||
n_kv = cfg["n_local_heads"] * head_dim
|
||||
q, k, v = data_torch.split([n_q, n_kv, n_kv], dim=0)
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_Q, bid), _neox_rows(q, head_dim))
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_K, bid), _neox_rows(k, head_dim))
|
||||
yield (self.format_tensor_name(T.A_GEN_CODE_ATTN_V, bid), v)
|
||||
return
|
||||
yield (self.format_tensor_name(self._FAST_AR_MAP[key], bid), data_torch)
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
yield from super().generate_extra_tensors()
|
||||
|
||||
sd = load_codec(self.dir_model)
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
def get(name: str) -> Tensor:
|
||||
return sd[name].float()
|
||||
|
||||
def tn(tensor: gguf.MODEL_TENSOR, bid: int | None = None, xid: int | None = None) -> str:
|
||||
return gguf.TENSOR_NAMES[tensor].format(bid=bid, xid=xid)
|
||||
|
||||
def conv_wb(tensor: gguf.MODEL_TENSOR, prefix: str, bid: int | None = None, xid: int | None = None, stride: int = 1):
|
||||
# taps-major [K / stride, OC, stride * IC], see conv() in tools/mtmd/models/fish-s2-spkenc.cpp
|
||||
w = fold_conv(sd, prefix)
|
||||
n_out, n_in, k = w.shape
|
||||
w = w.reshape(n_out, n_in, k // stride, stride).permute(2, 0, 3, 1).reshape(k // stride, n_out, stride * n_in)
|
||||
yield (tn(tensor, bid, xid) + ".weight", w)
|
||||
yield (tn(tensor, bid, xid) + ".bias", get(prefix + ".conv.bias"))
|
||||
|
||||
def snake_ab(tensor: gguf.MODEL_TENSOR, name: str, bid: int | None = None, xid: int | None = None):
|
||||
a, b = fold_snake(get(name))
|
||||
yield (tn(tensor, bid, xid) + ".alpha", a)
|
||||
yield (tn(tensor, bid, xid) + ".beta", b)
|
||||
|
||||
def tfm_layers(prefix: str, n_layer: int, offset: int):
|
||||
for il in range(n_layer):
|
||||
p = f"{prefix}layers.{il}."
|
||||
bid = offset + il
|
||||
wq, wk, wv = get(p + "attention.wqkv.weight").chunk(3, dim=0)
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_Q, bid), _neox_rows(wq, _CODEC_HEAD_DIM))
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_K, bid), _neox_rows(wk, _CODEC_HEAD_DIM))
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_V, bid), wv)
|
||||
yield (self.format_tensor_name(T.A_ENC_OUTPUT, bid), get(p + "attention.wo.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_INPUT_NORM, bid), get(p + "attention_norm.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_ATTN_SCALE, bid), get(p + "attention_layer_scale.gamma"))
|
||||
yield (self.format_tensor_name(T.A_ENC_OUTPUT_NORM, bid), get(p + "ffn_norm.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_GATE, bid), get(p + "feed_forward.w1.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_UP, bid), get(p + "feed_forward.w3.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_DOWN, bid), get(p + "feed_forward.w2.weight"))
|
||||
yield (self.format_tensor_name(T.A_ENC_FFN_SCALE_LS, bid), get(p + "ffn_layer_scale.gamma"))
|
||||
|
||||
# encoder: entry conv, 4x (3 residual units, Snake, strided conv), Snake, post conv
|
||||
e = "encoder.block."
|
||||
yield from conv_wb(T.A_CODEC_ENC_ENTRY, e + "0")
|
||||
for bid, stride in enumerate(_CODEC_ENC_STRIDES):
|
||||
p = f"{e}{bid + 1}.block."
|
||||
for xid in range(3):
|
||||
r = f"{p}{xid}.block."
|
||||
yield from snake_ab(T.A_CODEC_ENC_RES_ACT1, r + "0.alpha", bid, xid)
|
||||
yield from conv_wb(T.A_CODEC_ENC_RES_CONV1, r + "1", bid, xid)
|
||||
yield from snake_ab(T.A_CODEC_ENC_RES_ACT2, r + "2.alpha", bid, xid)
|
||||
yield from conv_wb(T.A_CODEC_ENC_RES_CONV2, r + "3", bid, xid)
|
||||
yield from snake_ab(T.A_CODEC_ENC_SNAKE, p + "3.alpha", bid)
|
||||
yield from conv_wb(T.A_CODEC_ENC_CONV, p + "4", bid, stride=stride)
|
||||
# the last block ends with a windowed transformer
|
||||
yield from tfm_layers(f"{e}4.block.5.", _N_ENC_TFM_LAYERS, 0)
|
||||
yield (self.format_tensor_name(T.A_POST_NORM), get(f"{e}4.block.5.norm.weight"))
|
||||
yield from snake_ab(T.A_CODEC_ENC_POST_SNAKE, e + "5.alpha")
|
||||
yield from conv_wb(T.A_CODEC_ENC_POST_CONV, e + "6")
|
||||
|
||||
# quantizer: 2x (strided conv + ConvNeXt), windowed transformer, then semantic + residual VQ
|
||||
q = "quantizer."
|
||||
for bid in range(2):
|
||||
p = f"{q}downsample.{bid}."
|
||||
yield from conv_wb(T.A_CODEC_DOWN_CONV, p + "0", bid, stride=2)
|
||||
# depthwise: one [C] row per tap
|
||||
yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".weight", fold_conv(sd, p + "1.dwconv").squeeze(1).T.contiguous())
|
||||
yield (tn(T.A_CODEC_DOWN_DWCONV, bid) + ".bias", get(p + "1.dwconv.conv.bias"))
|
||||
for key, tensor in (("norm", T.A_CODEC_DOWN_NORM), ("pwconv1", T.A_CODEC_DOWN_PW1), ("pwconv2", T.A_CODEC_DOWN_PW2)):
|
||||
yield (tn(tensor, bid) + ".weight", get(f"{p}1.{key}.weight"))
|
||||
yield (tn(tensor, bid) + ".bias", get(f"{p}1.{key}.bias"))
|
||||
yield (tn(T.A_CODEC_DOWN_GAMMA, bid), get(p + "1.gamma"))
|
||||
yield from tfm_layers(q + "pre_module.", _N_PRE_TFM_LAYERS, _N_ENC_TFM_LAYERS)
|
||||
yield (self.format_tensor_name(T.A_CODEC_PRE_NORM), get(q + "pre_module.norm.weight"))
|
||||
|
||||
# nearest codebook entry by cosine similarity, see VectorQuantize.decode_latents()
|
||||
def vq(prefix: str) -> tuple[Tensor, Tensor, Tensor]:
|
||||
w = fold_wn(sd[prefix + ".in_proj.weight_g"], sd[prefix + ".in_proj.weight_v"])
|
||||
cb = torch.nn.functional.normalize(get(prefix + ".codebook.weight"), dim=-1)
|
||||
return w, get(prefix + ".in_proj.bias"), cb
|
||||
|
||||
w, b, cb = vq(q + "semantic_quantizer.quantizers.0")
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".weight"), w)
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_IN, suffix=".bias"), b)
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_FIRST_CB), cb)
|
||||
n_rest = self.hparams["num_codebooks"] - 1
|
||||
rest = [vq(f"{q}quantizer.quantizers.{i}") for i in range(n_rest)]
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".weight"), torch.stack([r[0] for r in rest]))
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_IN, suffix=".bias"), torch.stack([r[1] for r in rest]))
|
||||
yield (self.format_tensor_name(T.A_CODEC_VQ_REST_CB), torch.stack([r[2] for r in rest]))
|
||||
@@ -302,6 +302,10 @@ class Inkling2VisionModel(MmprojModel):
|
||||
w.add_vision_attention_layernorm_eps(config["layer_norm_eps"])
|
||||
w.add_vision_spatial_merge_size(config["spatial_merge_size"])
|
||||
|
||||
def dequant_model(self):
|
||||
# vision tensors are BF16, only the text experts are quantized
|
||||
return
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item):
|
||||
name, gen = item
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, MmprojModel, gguf, logger
|
||||
from .qwen import QwenModel, Qwen3Model
|
||||
|
||||
# token ids of the yue2_infer protocol (protocol.py)
|
||||
EOD = 151643
|
||||
ABC_START, ABC_END = 151847, 151848
|
||||
MUSIC_START, MUSIC_END = 151851, 151852
|
||||
CODEC_OFFSET, CODEC_SIZE = 151853, 32768
|
||||
LATENT_START = 184621
|
||||
|
||||
# the VAE is a separate repo (m-a-p/YuE2-Vae), downloaded into this subdir of the model dir
|
||||
VAE_DIR = "vae"
|
||||
|
||||
INSTRUCTIONS = {
|
||||
"off": "Generate music with codec tokens from the given conditions.",
|
||||
"melody": "Generate a melody-only ABC transcription without chord symbols, then generate music with codec tokens from the given conditions.",
|
||||
"full": "Generate a chord-annotated ABC transcription, then generate music with codec tokens from the given conditions.",
|
||||
}
|
||||
|
||||
# the text is "[Tags]\n<style>\n[Lyrics]\n<lyrics>", optionally followed by "\n[ABC]\n<score>" (cover mode)
|
||||
# voice picks the planning mode: full (default), melody or off
|
||||
TTS_TEMPLATE = (
|
||||
"{%- set text = messages[0]['content'] -%}"
|
||||
"{%- set mode = (voice | lower) if voice is defined and voice and (voice | lower) in ['full', 'melody', 'off'] else 'full' -%}"
|
||||
"{%- set parts = text.split('\\n[ABC]\\n') -%}"
|
||||
"{%- set instructions = {"
|
||||
+ ", ".join(f"'{k}': '{v}'" for k, v in INSTRUCTIONS.items())
|
||||
+ "} -%}"
|
||||
"{{- '<|endoftext|>' + instructions[mode] + '\\n' + parts[0].rstrip() + '\\n' -}}"
|
||||
"{%- if mode == 'off' -%}"
|
||||
"{{- '<abc></abc><|music_start|>' -}}"
|
||||
"{%- elif parts | length > 1 -%}"
|
||||
"{{- '<abc>' + parts[1] + '</abc><|music_start|>' -}}"
|
||||
"{%- else -%}"
|
||||
"{{- '<abc>' -}}"
|
||||
"{%- endif -%}"
|
||||
)
|
||||
|
||||
|
||||
def _is_nar_tensor(name: str) -> bool:
|
||||
return ".nar_" in name or name.startswith(("llm2vae.", "vae2llm.", "time_embedder.", "latent_pos_embed."))
|
||||
|
||||
|
||||
@ModelBase.register("YuE2ForCausalLM")
|
||||
class YuE2TextModel(Qwen3Model):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN3
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
# the NAR path goes to the mmproj
|
||||
if _is_nar_tensor(item[0]):
|
||||
return None
|
||||
return super().filter_tensors(item)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
# the codes are sampled with classifier-free guidance, the uncond prompt runs as a second sequence
|
||||
self.gguf_writer.add_classifier_free_guidance(True)
|
||||
|
||||
def set_vocab(self):
|
||||
# checkpoint-native tiktoken file, see tokenization_yue2.py
|
||||
ranks: dict[bytes, int] = {}
|
||||
with open(self.dir_model / "qwen.tiktoken", "rb") as f:
|
||||
for line in f.read().splitlines():
|
||||
if line:
|
||||
tok, rank = line.split()
|
||||
ranks[base64.b64decode(tok)] = int(rank)
|
||||
if len(ranks) != EOD:
|
||||
raise ValueError(f"YuE2: expected {EOD} BPE tokens, got {len(ranks)}")
|
||||
|
||||
vocab_size = self.hparams["vocab_size"]
|
||||
tokens: list[str] = [""] * vocab_size
|
||||
toktypes: list[int] = [gguf.TokenType.UNUSED] * vocab_size
|
||||
merges: list[str] = []
|
||||
for tok, rank in ranks.items():
|
||||
tokens[rank] = QwenModel.token_bytes_to_string(tok)
|
||||
toktypes[rank] = gguf.TokenType.NORMAL
|
||||
if len(tok) > 1:
|
||||
merged = QwenModel.bpe(ranks, tok, max_rank=rank)
|
||||
assert len(merged) == 2
|
||||
merges.append(" ".join(map(QwenModel.token_bytes_to_string, merged)))
|
||||
|
||||
specials = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<R>", "<S>", "<X>", "<mask>", "<sep>"]
|
||||
specials += [f"<extra_{i}>" for i in range(200)]
|
||||
specials[204:206] = ["<abc>", "</abc>"]
|
||||
named = {EOD + i: s for i, s in enumerate(specials)}
|
||||
named[MUSIC_START] = "<|music_start|>"
|
||||
named[MUSIC_END] = "<|music_end|>"
|
||||
for i in range(CODEC_SIZE):
|
||||
named[CODEC_OFFSET + i] = f"<|codec_{i}|>"
|
||||
for i, s in enumerate(["<|latent_start|>", "<|latent_end|>", "<|latent_pad|>"]):
|
||||
named[LATENT_START + i] = s
|
||||
for i in range(EOD, vocab_size):
|
||||
tokens[i] = named.get(i, f"[PAD{i}]")
|
||||
toktypes[i] = gguf.TokenType.CONTROL if i in named else gguf.TokenType.UNUSED
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("gpt2")
|
||||
self.gguf_writer.add_tokenizer_pre("qwen2")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
self.gguf_writer.add_token_merges(merges)
|
||||
self.gguf_writer.add_bos_token_id(EOD)
|
||||
self.gguf_writer.add_eos_token_id(MUSIC_END)
|
||||
self.gguf_writer.add_pad_token_id(EOD)
|
||||
self.gguf_writer.add_add_bos_token(False)
|
||||
self.gguf_writer.add_chat_template([{"name": "tts", "template": TTS_TEMPLATE}])
|
||||
|
||||
|
||||
@ModelBase.register("YuE2ForCausalLM")
|
||||
class YuE2MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
# the NAR path: same shapes as the backbone layers
|
||||
return {k: self.global_config[k] for k in ("num_hidden_layers", "hidden_size", "intermediate_size",
|
||||
"num_attention_heads", "num_key_value_heads", "rms_norm_eps")}
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name = item[0]
|
||||
if name == "latent_pos_embed.pe":
|
||||
return None # sinusoidal, computed at runtime
|
||||
if _is_nar_tensor(name) or name == "model.norm.weight":
|
||||
return item
|
||||
return None
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hp = self.hparams
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.YUE2_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(hp["hidden_size"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(hp["intermediate_size"])
|
||||
self.gguf_writer.add_gen_audio_block_count(hp["num_hidden_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(hp["num_attention_heads"])
|
||||
self.gguf_writer.add_gen_audio_head_count_kv(hp["num_key_value_heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(hp["rms_norm_eps"])
|
||||
# rope theta, the ODE solver and the VAE hparams are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# read on the host, or small convs and projections of the flow / VAE graphs
|
||||
if not new_name.startswith("a.gen.nar.blk.") or n_dims < 2:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@staticmethod
|
||||
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
|
||||
return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
from safetensors.torch import load_file
|
||||
path = self.dir_model / VAE_DIR / "model.safetensors"
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"YuE2: VAE not found at {path}, download m-a-p/YuE2-Vae into it")
|
||||
logger.info(f"gguf: loading VAE decoder from '{path}'")
|
||||
sd = {k[len("decoder.layers."):]: v for k, v in load_file(str(path)).items() if k.startswith("decoder.layers.")}
|
||||
p = "a.gen.vae"
|
||||
|
||||
# SnakeBeta with log-scale params: x + 1/(exp(beta) + 1e-9) * sin(exp(alpha) * x)^2
|
||||
def snake(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
yield (f"{dst}.alpha", torch.exp(sd[src + ".alpha"].float()))
|
||||
yield (f"{dst}.beta", 1.0 / (torch.exp(sd[src + ".beta"].float()) + 1e-9))
|
||||
|
||||
yield (f"{p}.conv_in.weight", self._wn(sd, "0"))
|
||||
yield (f"{p}.conv_in.bias", sd["0.bias"])
|
||||
n_blk = sum(1 for k in sd if k.endswith(".layers.1.weight_g") and k.count(".") == 3)
|
||||
for b in range(n_blk):
|
||||
s = f"{b + 1}.layers"
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.0")
|
||||
yield (f"{p}.blk.{b}.up.weight", self._wn(sd, f"{s}.1"))
|
||||
yield (f"{p}.blk.{b}.up.bias", sd[f"{s}.1.bias"])
|
||||
for r in range(3):
|
||||
rs = f"{s}.{r + 2}.layers"
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake1", f"{rs}.0")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.weight", self._wn(sd, f"{rs}.1"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv1.bias", sd[f"{rs}.1.bias"])
|
||||
yield from snake(f"{p}.blk.{b}.res.{r}.snake2", f"{rs}.2")
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.weight", self._wn(sd, f"{rs}.3"))
|
||||
yield (f"{p}.blk.{b}.res.{r}.conv2.bias", sd[f"{rs}.3.bias"])
|
||||
yield from snake(f"{p}.post_snake", f"{n_blk + 1}")
|
||||
yield (f"{p}.conv_out.weight", self._wn(sd, f"{n_blk + 2}"))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if name.startswith("a.gen.vae."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
p = "a.gen.nar"
|
||||
aux = {
|
||||
"model.norm.weight": f"{p}.output_norm.weight",
|
||||
"vae2llm.weight": f"{p}.inp.weight",
|
||||
"vae2llm.bias": f"{p}.inp.bias",
|
||||
"llm2vae.weight": f"{p}.out.weight",
|
||||
"llm2vae.bias": f"{p}.out.bias",
|
||||
"time_embedder.mlp.0.weight": f"{p}.time.0.weight",
|
||||
"time_embedder.mlp.0.bias": f"{p}.time.0.bias",
|
||||
"time_embedder.mlp.2.weight": f"{p}.time.2.weight",
|
||||
"time_embedder.mlp.2.bias": f"{p}.time.2.bias",
|
||||
}
|
||||
if name in aux:
|
||||
yield (aux[name], data_torch)
|
||||
return
|
||||
|
||||
# the NAR layers keep the libllama names, mtmd loads them as a text model
|
||||
assert bid is not None, name
|
||||
layer = {
|
||||
"nar_input_layernorm.weight": "attn_norm.weight",
|
||||
"nar_self_attn.q_proj.weight": "attn_q.weight",
|
||||
"nar_self_attn.k_proj.weight": "attn_k.weight",
|
||||
"nar_self_attn.v_proj.weight": "attn_v.weight",
|
||||
"nar_self_attn.o_proj.weight": "attn_output.weight",
|
||||
"nar_self_attn.q_norm.weight": "attn_q_norm.weight",
|
||||
"nar_self_attn.k_norm.weight": "attn_k_norm.weight",
|
||||
"nar_pre_mlp_layernorm.weight": "ffn_norm.weight",
|
||||
"nar_mlp.gate_proj.weight": "ffn_gate.weight",
|
||||
"nar_mlp.up_proj.weight": "ffn_up.weight",
|
||||
"nar_mlp.down_proj.weight": "ffn_down.weight",
|
||||
}
|
||||
key = name.split(".", 3)[3] # strip "model.layers.{bid}."
|
||||
yield (f"{p}.blk.{bid}.{layer[key]}", data_torch)
|
||||
@@ -692,7 +692,7 @@ llama-openvino:server --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_
|
||||
# Or Using llama-server executable
|
||||
./build/ReleaseOV/bin/llama-server -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --port 8080 -c 1024
|
||||
|
||||
# Option 1: Open your browser to http://localhost:8080 to access the web UI for the llama.cpp server.
|
||||
# Option 1: Open your browser to http://localhost:8080 to open Llama Chat, the built-in chat app of the llama.cpp server.
|
||||
# Option 2: In a NEW terminal, test the server with curl
|
||||
|
||||
# If you are behind a proxy, make sure to set NO_PROXY to avoid proxy for localhost
|
||||
|
||||
@@ -12,6 +12,8 @@ Supported models:
|
||||
- [Chatterbox](#chatterbox)
|
||||
- [Drama 3](#drama-3)
|
||||
- [ZONOS3](#zonos3)
|
||||
- [Fish Audio S2 Pro](#fish-audio-s2-pro)
|
||||
- [YuE2](#yue2) (songs)
|
||||
|
||||
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
|
||||
|
||||
@@ -215,3 +217,92 @@ Convert the root of the release repository, not the `lm/` subdirectory. The mmpr
|
||||
python convert_hf_to_gguf.py path/to/ZONOS3-Mini --outtype q8_0 --outfile zonos3-mini.gguf
|
||||
python convert_hf_to_gguf.py path/to/ZONOS3-Mini --mmproj --outtype f32 --outfile mmproj-zonos3-mini.gguf
|
||||
```
|
||||
|
||||
## Fish Audio S2 Pro
|
||||
|
||||
[Fish Audio S2 Pro](https://huggingface.co/fishaudio/s2-pro): a Qwen3 backbone (4B) samples each frame's semantic token, a 4-layer fast AR predicts the 9 other codes, and the Fish DAC (same as Drama 3) decodes them to 44.1 kHz audio.
|
||||
|
||||
Available params:
|
||||
- Speaker tags `<|speaker:0|>`, `<|speaker:1|>`, ... in the text give each character its own voice, a text without a tag starts with `<|speaker:0|>`
|
||||
- Inline instructions in brackets control the delivery, for example `[laughing]`, `[whisper]` or `[pause]`
|
||||
- The language is the one of the text, `--tts-lang` is not used
|
||||
- Voice cloning: `--tts-speaker-file` with the reference clip, optionally `--tts-speaker-text` with its transcript (it is put in the prompt as the reference does). The clip is encoded by the codec encoder half of `codec.pth`, it is truncated to the first 30 s
|
||||
- Note: `--temp`, `--top-k` and `--top-p` are not used (only `--seed`), frames are sampled with the settings of the reference (semantic token: repetition penalty 1.1 over the last 16 frames, top-k 30, top-p 0.8, temperature 0.8, or top-p 0.9 and temperature 1.0 when the last 4 frames repeat a token; other codes: greedy)
|
||||
- Each frame is 2048 samples (about 46 ms), generation stops at the end-of-speech token
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m s2-pro.gguf \
|
||||
-mm mmproj-s2-pro.gguf \
|
||||
-p "<|speaker:0|>Did you remember to buy the bread? <|speaker:1|>Of course I did [laughing] it is on the kitchen table." \
|
||||
-n 1000 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Voice cloning (`--tts-speaker-text` can be left out):
|
||||
|
||||
```sh
|
||||
llama-tts -m s2-pro.gguf \
|
||||
-mm mmproj-s2-pro.gguf \
|
||||
--tts-speaker-file reference.wav \
|
||||
--tts-speaker-text "The words spoken in reference.wav." \
|
||||
-p "Hello, this is a cloned voice." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The mmproj holds the fast AR and `codec.pth` (the encoder half for voice cloning, the decoder half for the audio), it is best kept in BF16. The codec encoder is always stored in F16/F32, small errors in it change the codes picked by its VQ:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/s2-pro --outtype q8_0 --outfile s2-pro.gguf
|
||||
python convert_hf_to_gguf.py path/to/s2-pro --mmproj --outtype bf16 --outfile mmproj-s2-pro.gguf
|
||||
```
|
||||
|
||||
## YuE2
|
||||
|
||||
[m-a-p/YuE2-3B](https://huggingface.co/m-a-p/YuE2-3B) generates songs (vocals and accompaniment) from a style prompt and lyrics, as 48 kHz stereo. The backbone (a Qwen3 text model) writes an ABC score, then one semantic code per 40 ms frame; the NAR half of the checkpoint turns the codes into VAE latents by flow matching, attending to the keys and values of the backbone, and the [YuE2-Vae](https://huggingface.co/m-a-p/YuE2-Vae) decoder makes the audio.
|
||||
|
||||
The prompt is the text of the reference request:
|
||||
|
||||
```
|
||||
[Tags]
|
||||
City Pop, upbeat, danceable, groovy bass, electric guitar, synth
|
||||
[Lyrics]
|
||||
[Verse]
|
||||
...
|
||||
```
|
||||
|
||||
Available params:
|
||||
- `--tts-voice` picks the planning mode of the reference (`cot`): `full` (default, a chord-annotated score), `melody` (a melody-only score) or `off` (no score)
|
||||
- Cover mode: append the score to the prompt as `\n[ABC]\n<score>`, with `--tts-voice melody` or `full`; the model then goes straight to the codes
|
||||
- The generation stops at the limits of the reference (4096 score tokens, then 9000 codes, 6 minutes); `-n` sets a lower limit on the score tokens and the codes together
|
||||
- The context must keep the default size (24576 positions, the one of the reference): a smaller `-c` is rejected when the model is loaded
|
||||
- Note: `--top-k` and `--top-p` are not used, both phases are sampled with the settings of the reference (`yue2_generation_config.json`); `--temp` changes the temperature of the codes
|
||||
- Note: the backbone context must use an F16 KV cache (default): the NAR pass loads its keys and values
|
||||
- The codes are sampled with classifier-free guidance like the reference: scale 1.01 with `--tts-voice off`, off (1.0) otherwise. `llama-server` takes `cfg_scale` to change it, the reference suggests 1.2 for a stronger text guidance. The uncond prompt runs as a second sequence, so the context must also hold it (the instruction, the score and the codes once more)
|
||||
- The NAR pass solves the flow-matching ODE in 32 midpoint steps like the reference; `llama-server` takes `flow_steps` to change it. The NAR time is about proportional to it: 16 steps are about 35% faster for the NAR and the VAE together, with a small deviation from the reference (about 0.2 dB of spectral difference); it grows with fewer steps (about 0.4 dB at 8, 0.8 dB at 4)
|
||||
- The NAR pass runs on chunks of codes like the reference: one chunk for a song that fits in 24576 positions, so the audio arrives after the last code. When streaming, each 30 s of codes is a chunk, processed during the generation, so the audio arrives in 30 s steps; the chunks are independent (each sees the prompt, the score and its own codes), which may be audible at the boundaries
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m yue2-3b.gguf \
|
||||
-mm mmproj-yue2-3b.gguf \
|
||||
-f song.txt \
|
||||
--output song.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The VAE is a separate repository, download it into the `vae` subdirectory of the model:
|
||||
|
||||
```sh
|
||||
hf download m-a-p/YuE2-3B --local-dir YuE2-3B
|
||||
hf download m-a-p/YuE2-Vae --local-dir YuE2-3B/vae
|
||||
python convert_hf_to_gguf.py YuE2-3B --outfile yue2-3b.gguf
|
||||
python convert_hf_to_gguf.py YuE2-3B --mmproj --outfile mmproj-yue2-3b.gguf
|
||||
```
|
||||
|
||||
The mmproj holds the NAR transformer layers (loaded with libllama as a Qwen3 text model, in the `--outtype` type), and the latent projections and the VAE decoder, always in F32.
|
||||
|
||||
@@ -87,36 +87,32 @@ for id: llama_token in tokens {
|
||||
|
||||
print("\n")
|
||||
|
||||
var batch = llama_batch_init(max(Int32(tokens.count), Int32(n_parallel)), 0, 1)
|
||||
let batch = llama_batch_ext_init(context)
|
||||
defer {
|
||||
llama_batch_free(batch)
|
||||
llama_batch_ext_free(batch)
|
||||
}
|
||||
|
||||
// evaluate the initial prompt
|
||||
batch.n_tokens = Int32(tokens.count)
|
||||
|
||||
for (i, token) in tokens.enumerated() {
|
||||
batch.token[i] = token
|
||||
batch.pos[i] = Int32(i)
|
||||
batch.n_seq_id[i] = 1
|
||||
// batch.seq_id[i][0] = 0
|
||||
// TODO: is this the proper way to do this?
|
||||
if let seq_id = batch.seq_id[i] {
|
||||
seq_id[0] = 0
|
||||
let idx = llama_batch_ext_add_token(batch, 0, token)
|
||||
if idx < 0 {
|
||||
print("llama_batch_ext_add_token() failed: \(idx)")
|
||||
exit(1)
|
||||
}
|
||||
batch.logits[i] = 0
|
||||
var pos = llama_pos(i)
|
||||
llama_batch_ext_set_pos(batch, idx, &pos)
|
||||
}
|
||||
|
||||
// llama_decode will output logits only for the last token of the prompt
|
||||
batch.logits[Int(batch.n_tokens) - 1] = 1
|
||||
// llama_process will output logits only for the last token of the prompt
|
||||
llama_batch_ext_set_output_logits(batch, Int32(tokens.count) - 1, true)
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed")
|
||||
exit(1)
|
||||
}
|
||||
|
||||
for i in 1 ..< n_parallel {
|
||||
llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, batch.n_tokens)
|
||||
llama_memory_seq_cp(llama_get_memory(context), 0, Int32(i), 0, Int32(tokens.count))
|
||||
}
|
||||
|
||||
if n_parallel > 1 {
|
||||
@@ -125,16 +121,17 @@ if n_parallel > 1 {
|
||||
|
||||
var streams: [String] = .init(repeating: "", count: n_parallel)
|
||||
var streamBuffers: [[CChar]] = .init(repeating: [], count: n_parallel)
|
||||
var i_batch = [Int32](repeating: batch.n_tokens - 1, count: n_parallel)
|
||||
var i_batch = [Int32](repeating: Int32(tokens.count) - 1, count: n_parallel)
|
||||
|
||||
var n_cur = batch.n_tokens
|
||||
var n_cur = Int32(tokens.count)
|
||||
var n_decode = 0
|
||||
|
||||
let t_main_start = ggml_time_us()
|
||||
|
||||
while n_cur <= n_len {
|
||||
// prepare the next batch
|
||||
batch.n_tokens = 0
|
||||
llama_batch_ext_clear(batch)
|
||||
var n_tokens: Int32 = 0
|
||||
|
||||
// sample the next token for each parallel sequence / stream
|
||||
for i in 0 ..< n_parallel {
|
||||
@@ -165,31 +162,32 @@ while n_cur <= n_len {
|
||||
streams[i] += nextStringPiece
|
||||
|
||||
// push this new token for next evaluation
|
||||
batch.token[Int(batch.n_tokens)] = new_token_id
|
||||
batch.pos[Int(batch.n_tokens)] = n_cur
|
||||
batch.n_seq_id[Int(batch.n_tokens)] = 1
|
||||
if let seq_id = batch.seq_id[Int(batch.n_tokens)] {
|
||||
seq_id[0] = Int32(i)
|
||||
let idx = llama_batch_ext_add_token(batch, Int32(i), new_token_id)
|
||||
if idx < 0 {
|
||||
print("llama_batch_ext_add_token() failed: \(idx)")
|
||||
exit(1)
|
||||
}
|
||||
batch.logits[Int(batch.n_tokens)] = 1
|
||||
var pos = n_cur
|
||||
llama_batch_ext_set_pos(batch, idx, &pos)
|
||||
llama_batch_ext_set_output_logits(batch, idx, true)
|
||||
|
||||
i_batch[i] = batch.n_tokens
|
||||
i_batch[i] = idx
|
||||
|
||||
batch.n_tokens += 1
|
||||
n_tokens += 1
|
||||
|
||||
n_decode += 1
|
||||
}
|
||||
|
||||
// all streams are finished
|
||||
if batch.n_tokens == 0 {
|
||||
if n_tokens == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
n_cur += 1
|
||||
|
||||
// evaluate the current batch with the transformer model
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed")
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,20 +5,16 @@ enum LlamaError: Error {
|
||||
case couldNotInitializeContext
|
||||
}
|
||||
|
||||
func llama_batch_clear(_ batch: inout llama_batch) {
|
||||
batch.n_tokens = 0
|
||||
}
|
||||
|
||||
func llama_batch_add(_ batch: inout llama_batch, _ id: llama_token, _ pos: llama_pos, _ seq_ids: [llama_seq_id], _ logits: Bool) {
|
||||
batch.token [Int(batch.n_tokens)] = id
|
||||
batch.pos [Int(batch.n_tokens)] = pos
|
||||
batch.n_seq_id[Int(batch.n_tokens)] = Int32(seq_ids.count)
|
||||
for i in 0..<seq_ids.count {
|
||||
batch.seq_id[Int(batch.n_tokens)]![Int(i)] = seq_ids[i]
|
||||
@discardableResult
|
||||
func llama_batch_add(_ batch: OpaquePointer, _ id: llama_token, _ pos: llama_pos, _ seq_id: llama_seq_id, _ logits: Bool) -> Int32 {
|
||||
let idx = llama_batch_ext_add_token(batch, seq_id, id)
|
||||
if idx < 0 {
|
||||
return idx
|
||||
}
|
||||
batch.logits [Int(batch.n_tokens)] = logits ? 1 : 0
|
||||
|
||||
batch.n_tokens += 1
|
||||
var pos = pos
|
||||
llama_batch_ext_set_pos(batch, idx, &pos)
|
||||
llama_batch_ext_set_output_logits(batch, idx, logits)
|
||||
return idx
|
||||
}
|
||||
|
||||
actor LlamaContext {
|
||||
@@ -26,7 +22,7 @@ actor LlamaContext {
|
||||
private var context: OpaquePointer
|
||||
private var vocab: OpaquePointer
|
||||
private var sampling: UnsafeMutablePointer<llama_sampler>
|
||||
private var batch: llama_batch
|
||||
private var batch: OpaquePointer
|
||||
private var tokens_list: [llama_token]
|
||||
var is_done: Bool = false
|
||||
|
||||
@@ -42,7 +38,7 @@ actor LlamaContext {
|
||||
self.model = model
|
||||
self.context = context
|
||||
self.tokens_list = []
|
||||
self.batch = llama_batch_init(512, 0, 1)
|
||||
self.batch = llama_batch_ext_init(context)
|
||||
self.temporary_invalid_cchars = []
|
||||
let sparams = llama_sampler_chain_default_params()
|
||||
self.sampling = llama_sampler_chain_init(sparams)
|
||||
@@ -53,7 +49,7 @@ actor LlamaContext {
|
||||
|
||||
deinit {
|
||||
llama_sampler_free(sampling)
|
||||
llama_batch_free(batch)
|
||||
llama_batch_ext_free(batch)
|
||||
llama_model_free(model)
|
||||
llama_free(context)
|
||||
llama_backend_free()
|
||||
@@ -110,10 +106,6 @@ actor LlamaContext {
|
||||
return SwiftString
|
||||
}
|
||||
|
||||
func get_n_tokens() -> Int32 {
|
||||
return batch.n_tokens;
|
||||
}
|
||||
|
||||
func completion_init(text: String) {
|
||||
print("attempting to complete \"\(text)\"")
|
||||
|
||||
@@ -133,25 +125,27 @@ actor LlamaContext {
|
||||
print(String(cString: token_to_piece(token: id) + [0]))
|
||||
}
|
||||
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_ext_clear(batch)
|
||||
|
||||
for i1 in 0..<tokens_list.count {
|
||||
let i = Int(i1)
|
||||
llama_batch_add(&batch, tokens_list[i], Int32(i), [0], false)
|
||||
}
|
||||
batch.logits[Int(batch.n_tokens) - 1] = 1 // true
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed")
|
||||
let idx = llama_batch_add(batch, tokens_list[i], Int32(i), 0, i == tokens_list.count - 1)
|
||||
if idx < 0 {
|
||||
print("llama_batch_add() failed: \(idx)")
|
||||
}
|
||||
}
|
||||
|
||||
n_cur = batch.n_tokens
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed")
|
||||
}
|
||||
|
||||
n_cur = Int32(tokens_list.count)
|
||||
}
|
||||
|
||||
func completion_loop() -> String {
|
||||
var new_token_id: llama_token = 0
|
||||
|
||||
new_token_id = llama_sampler_sample(sampling, context, batch.n_tokens - 1)
|
||||
new_token_id = llama_sampler_sample(sampling, context, -1)
|
||||
|
||||
if llama_vocab_is_eog(vocab, new_token_id) || n_cur == n_len {
|
||||
print("\n")
|
||||
@@ -178,13 +172,13 @@ actor LlamaContext {
|
||||
print(new_token_str)
|
||||
// tokens_list.append(new_token_id)
|
||||
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_add(&batch, new_token_id, n_cur, [0], true)
|
||||
llama_batch_ext_clear(batch)
|
||||
llama_batch_add(batch, new_token_id, n_cur, 0, true)
|
||||
|
||||
n_decode += 1
|
||||
n_cur += 1
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("failed to evaluate llama!")
|
||||
}
|
||||
|
||||
@@ -201,21 +195,20 @@ actor LlamaContext {
|
||||
for _ in 0..<nr {
|
||||
// bench prompt processing
|
||||
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_ext_clear(batch)
|
||||
|
||||
let n_tokens = pp
|
||||
|
||||
for i in 0..<n_tokens {
|
||||
llama_batch_add(&batch, 0, Int32(i), [0], false)
|
||||
llama_batch_add(batch, 0, Int32(i), 0, i == n_tokens - 1)
|
||||
}
|
||||
batch.logits[Int(batch.n_tokens) - 1] = 1 // true
|
||||
|
||||
llama_memory_clear(llama_get_memory(context), false)
|
||||
|
||||
let t_pp_start = DispatchTime.now().uptimeNanoseconds / 1000;
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed during prompt")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed during prompt")
|
||||
}
|
||||
llama_synchronize(context)
|
||||
|
||||
@@ -228,14 +221,14 @@ actor LlamaContext {
|
||||
let t_tg_start = DispatchTime.now().uptimeNanoseconds / 1000;
|
||||
|
||||
for i in 0..<tg {
|
||||
llama_batch_clear(&batch)
|
||||
llama_batch_ext_clear(batch)
|
||||
|
||||
for j in 0..<pl {
|
||||
llama_batch_add(&batch, 0, Int32(i), [Int32(j)], true)
|
||||
llama_batch_add(batch, 0, Int32(i), Int32(j), true)
|
||||
}
|
||||
|
||||
if llama_decode(context, batch) != 0 {
|
||||
print("llama_decode() failed during text generation")
|
||||
if llama_process(context, LLAMA_PROCESS_TYPE_DECODE, batch) != 0 {
|
||||
print("llama_process() failed during text generation")
|
||||
}
|
||||
llama_synchronize(context)
|
||||
}
|
||||
|
||||
@@ -9,11 +9,16 @@ constexpr int D = 128;
|
||||
using bf16 = nv_bfloat16;
|
||||
|
||||
template <int R, int C> struct matrix {
|
||||
bf16 hi[R * C], lo[R * C];
|
||||
#ifdef GGML_USE_HIP
|
||||
static constexpr int alloc = C + 4;
|
||||
#else
|
||||
static constexpr int alloc = C;
|
||||
#endif // GGML_USE_HIP
|
||||
bf16 hi[R * alloc], lo[R * alloc];
|
||||
|
||||
static __device__ __forceinline__ int index(int r, int c) {
|
||||
#ifdef GGML_USE_HIP
|
||||
return r * C + c;
|
||||
return r * alloc + c;
|
||||
#else
|
||||
return 2 * ggml_cuda_mma::swizzle<C / 2, nv_bfloat162>(r * (C / 2) + c / 2, r) + c % 2;
|
||||
#endif // GGML_USE_HIP
|
||||
@@ -56,11 +61,11 @@ template <int C, int V> struct alignas(128) shared {
|
||||
static __device__ __forceinline__ int lower_index(int r, int c) { return r * C + c; }
|
||||
};
|
||||
|
||||
#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
|
||||
#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
|
||||
using namespace ggml_cuda_mma;
|
||||
#ifdef GGML_USE_HIP
|
||||
using acc = tile<16, 16, float, DATA_LAYOUT_J_MAJOR>;
|
||||
using a_tile = tile<16, 8, nv_bfloat162, DATA_LAYOUT_I_MAJOR_MIRRORED>;
|
||||
using a_tile = tile<16, 8, nv_bfloat162, get_input_data_layout()>;
|
||||
using b_tile = a_tile;
|
||||
#else
|
||||
using acc = tile<16, 8, float>;
|
||||
@@ -73,6 +78,23 @@ __device__ __forceinline__ tile_t load(const matrix<R, C> & m, int row, int col,
|
||||
tile_t t;
|
||||
const bf16 * data = low ? m.lo : m.hi;
|
||||
#ifdef GGML_USE_HIP
|
||||
#ifdef AMD_MFMA_AVAILABLE
|
||||
if constexpr (!TRANS) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < tile_t::ne; ++i) {
|
||||
const int r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i);
|
||||
ggml_cuda_memcpy_1<4>(t.x + i, data + m.index(r, c));
|
||||
}
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < tile_t::ne; ++i) {
|
||||
const int r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i);
|
||||
const bf16 x = data[m.index(c, r)];
|
||||
const bf16 y = data[m.index(c + 1, r)];
|
||||
t.x[i] = __float22bfloat162_rn(make_float2(__bfloat162float(x), __bfloat162float(y)));
|
||||
}
|
||||
}
|
||||
#else
|
||||
#pragma unroll
|
||||
for (int i = 0; i < tile_t::ne; ++i) {
|
||||
const int r = row + tile_t::get_i(i), c = col + 2 * tile_t::get_j(i);
|
||||
@@ -80,6 +102,7 @@ __device__ __forceinline__ tile_t load(const matrix<R, C> & m, int row, int col,
|
||||
const bf16 y = data[TRANS ? m.index(c + 1, r) : m.index(r, c + 1)];
|
||||
t.x[i] = __float22bfloat162_rn(make_float2(__bfloat162float(x), __bfloat162float(y)));
|
||||
}
|
||||
#endif // AMD_MFMA_AVAILABLE
|
||||
#else
|
||||
const auto * packed = reinterpret_cast<const nv_bfloat162 *>(data);
|
||||
if constexpr (TRANS) {
|
||||
@@ -118,15 +141,17 @@ template <int R, int C> __device__ __forceinline__ void store(matrix<R, C> & m,
|
||||
}
|
||||
#endif // GGML_USE_HIP
|
||||
}
|
||||
#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
|
||||
#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
|
||||
|
||||
// Each block owns V independent value columns and advances by C tokens.
|
||||
// Keep recurrent state in FP32; pack high/residual BF16 operands inside the block.
|
||||
template <int C, int V, int BLOCKS = 1>
|
||||
static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_mma_args a) {
|
||||
#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
|
||||
static_assert(ggml_cuda_get_physical_warp_size() == 32, "GDN MMA requires 32-lane warps");
|
||||
constexpr int WARPS = 8;
|
||||
#if defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
static_assert(256 % warp_size == 0, "block must span a whole number of waves");
|
||||
constexpr int warps = 256 / warp_size;
|
||||
constexpr int input_rows = warp_size / 32; // input rows a wave stages per step
|
||||
#ifdef GGML_USE_HIP
|
||||
constexpr int N = 16;
|
||||
#else
|
||||
@@ -134,34 +159,37 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
#endif // GGML_USE_HIP
|
||||
static_assert(C >= 16 && (C & (C - 1)) == 0 && V % N == 0 && D % V == 0, "invalid GDN tile");
|
||||
extern __shared__ __align__(128) unsigned char bytes[];
|
||||
auto & s = *reinterpret_cast<shared<C, V> *>(bytes);
|
||||
const int lane = threadIdx.x, warp = threadIdx.y, tid = warp * 32 + lane;
|
||||
const int slice = blockIdx.x % (D / V);
|
||||
const int h = (blockIdx.x / (D / V)) % a.H;
|
||||
const int seq = blockIdx.x / ((D / V) * a.H);
|
||||
const int v0 = slice * V;
|
||||
const int64_t soff = ((int64_t) seq * a.H + h) * D * D;
|
||||
const int64_t qoff = (seq / a.rq3) * a.sq3 + (h % a.H_k) * a.sq1;
|
||||
const int64_t voff = seq * a.sv3 + h * a.sv1 + v0;
|
||||
const int64_t goff = seq * a.sb3 + h * a.sb1;
|
||||
float * out = a.dst + ((int64_t) seq * a.n_tokens * a.H + h) * D + v0;
|
||||
constexpr int STATE_TILES = (D / 16) * (V / N);
|
||||
constexpr int PER_WARP = STATE_TILES / WARPS;
|
||||
static_assert(STATE_TILES % WARPS == 0, "state must divide across warps");
|
||||
auto & s = *reinterpret_cast<shared<C, V> *>(bytes);
|
||||
const int lane = threadIdx.x;
|
||||
const int warp = threadIdx.y;
|
||||
const int tid = warp * warp_size + lane;
|
||||
const int sub = warp_size > 32 ? lane / 32 : 0;
|
||||
const int slane = warp_size > 32 ? lane % 32 : lane;
|
||||
const int slice = blockIdx.x % (D / V);
|
||||
const int h = (blockIdx.x / (D / V)) % a.H;
|
||||
const int seq = blockIdx.x / ((D / V) * a.H);
|
||||
const int v0 = slice * V;
|
||||
const int64_t soff = ((int64_t) seq * a.H + h) * D * D;
|
||||
const int64_t qoff = (seq / a.rq3) * a.sq3 + (h % a.H_k) * a.sq1;
|
||||
const int64_t voff = seq * a.sv3 + h * a.sv1 + v0;
|
||||
const int64_t goff = seq * a.sb3 + h * a.sb1;
|
||||
float * out = a.dst + ((int64_t) seq * a.n_tokens * a.H + h) * D + v0;
|
||||
constexpr int state_tiles = (D / 16) * (V / N);
|
||||
constexpr int PER_WARP = state_tiles / warps;
|
||||
static_assert(state_tiles % warps == 0, "state must divide across warps");
|
||||
acc state[PER_WARP];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < PER_WARP; ++j) {
|
||||
const int tile_index = warp + j * WARPS, r = (tile_index / (V / N)) * 16, c = (tile_index % (V / N)) * N;
|
||||
const int tile_index = warp + j * warps, r = (tile_index / (V / N)) * 16, c = (tile_index % (V / N)) * N;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
state[j].x[i] = a.state[soff + (v0 + c + acc::get_j(i)) * D + r + acc::get_i(i)];
|
||||
}
|
||||
}
|
||||
auto read_qk = [&](int64_t t, float4 & q, float4 & k) {
|
||||
t += warp;
|
||||
q = k = make_float4(0, 0, 0, 0);
|
||||
if (t < a.n_tokens) {
|
||||
const int64_t off = qoff + t * a.sq2 + 4 * lane;
|
||||
const int64_t off = qoff + t * a.sq2 + 4 * slane;
|
||||
if ((((uintptr_t) (a.q + off) | (uintptr_t) (a.k + off)) & 15) == 0) {
|
||||
q = *reinterpret_cast<const float4 *>(a.q + off);
|
||||
k = *reinterpret_cast<const float4 *>(a.k + off);
|
||||
@@ -172,22 +200,26 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
}
|
||||
};
|
||||
// Carry the first eight input rows across chunks to hide global-load latency.
|
||||
constexpr int QK_ROWS = warps * input_rows;
|
||||
constexpr bool PREFETCH = C == 16 && V == 32;
|
||||
float4 next_q, next_k;
|
||||
float4 next_q;
|
||||
float4 next_k;
|
||||
if constexpr (PREFETCH) {
|
||||
read_qk(0, next_q, next_k);
|
||||
read_qk(warp * input_rows + sub, next_q, next_k);
|
||||
}
|
||||
for (int64_t t0 = 0; t0 < a.n_tokens; t0 += C) {
|
||||
const int valid = min((int64_t) C, a.n_tokens - t0);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < C / 8; ++j) {
|
||||
const int t = warp + 8 * j, d = 4 * lane;
|
||||
float4 q, k;
|
||||
for (int j = 0; j < C / QK_ROWS; ++j) {
|
||||
const int t = warp * input_rows + sub + QK_ROWS * j;
|
||||
const int d = 4 * slane;
|
||||
float4 q;
|
||||
float4 k;
|
||||
if (PREFETCH && j == 0) {
|
||||
q = next_q;
|
||||
k = next_k;
|
||||
} else {
|
||||
read_qk(t0 + 8 * j, q, k);
|
||||
read_qk(t0 + t, q, k);
|
||||
}
|
||||
s.q.store2(t, d, make_float2(q.x * a.scale, q.y * a.scale));
|
||||
s.q.store2(t, d + 2, make_float2(q.z * a.scale, q.w * a.scale));
|
||||
@@ -197,13 +229,13 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
// Load gates in parallel before the FP32 prefix scan.
|
||||
if (warp == 0) {
|
||||
float carry = 0.f;
|
||||
for (int base = 0; base < C; base += 32) {
|
||||
for (int base = 0; base < C; base += warp_size) {
|
||||
const int t = base + lane;
|
||||
float g = t < valid ? a.g[goff + (t0 + t) * a.sb2] : 0.f;
|
||||
const float beta = t < valid ? a.beta[goff + (t0 + t) * a.sb2] : 0.f;
|
||||
#pragma unroll
|
||||
for (int offset = 1; offset < 32; offset *= 2) {
|
||||
const float previous = ggml_cuda_shfl_up(g, offset, 32);
|
||||
for (int offset = 1; offset < warp_size; offset *= 2) {
|
||||
const float previous = ggml_cuda_shfl_up(g, offset, warp_size);
|
||||
if (lane >= offset) {
|
||||
g += previous;
|
||||
}
|
||||
@@ -213,23 +245,23 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
s.prefix[t] = g;
|
||||
s.beta[t] = beta;
|
||||
}
|
||||
carry = ggml_cuda_shfl(g, 31, 32);
|
||||
carry = ggml_cuda_shfl(g, warp_size - 1, warp_size);
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < PER_WARP; ++j) {
|
||||
const int ti = warp + j * WARPS;
|
||||
const int ti = warp + j * warps;
|
||||
store(s.scratch.state, state[j], (ti / (V / N)) * 16, (ti % (V / N)) * N);
|
||||
}
|
||||
__syncthreads();
|
||||
if constexpr (PREFETCH) {
|
||||
if (t0 + C < a.n_tokens) {
|
||||
read_qk(t0 + C, next_q, next_k);
|
||||
read_qk(t0 + C + warp * input_rows + sub, next_q, next_k);
|
||||
}
|
||||
}
|
||||
// V32 overlaps Gram work with one output tile per projection warp.
|
||||
constexpr bool SPLIT_PROJECTION = C == 16 && V == 32;
|
||||
acc base_output[((C / 16) * (V / N) + WARPS - 1) / WARPS];
|
||||
acc base_output[((C / 16) * (V / N) + warps - 1) / warps];
|
||||
// delta = beta * (v - decay * K S), base output = decay * Q S
|
||||
auto project = [&]() {
|
||||
if constexpr (V == 32) {
|
||||
@@ -237,23 +269,26 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
return;
|
||||
}
|
||||
}
|
||||
for (int ti = (V == 32 ? warp - 4 : warp); ti < (C / 16) * (V / N); ti += (V == 32 ? 4 : WARPS)) {
|
||||
const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
acc qs, ks;
|
||||
for (int ti = (V == 32 ? warp - 4 : warp); ti < (C / 16) * (V / N); ti += (V == 32 ? 4 : warps)) {
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
acc qs;
|
||||
acc ks;
|
||||
product<D, false, true>(qs, s.q, s.scratch.state, r, c);
|
||||
product<D, false, true>(ks, s.k, s.scratch.state, r, c);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
const int t = r + acc::get_i(i), v = c + acc::get_j(i);
|
||||
const int t = r + acc::get_i(i);
|
||||
const int v = c + acc::get_j(i);
|
||||
const float decay = expf(s.prefix[t]);
|
||||
const float vv = t < valid ? a.v[voff + (t0 + t) * a.sv2 + v] : 0.f;
|
||||
s.delta.store(t, v, s.beta[t] * (vv - decay * ks.x[i]));
|
||||
base_output[ti / WARPS].x[i] = decay * qs.x[i];
|
||||
base_output[ti / warps].x[i] = decay * qs.x[i];
|
||||
}
|
||||
}
|
||||
};
|
||||
// A = beta * strict_tril(K K^T * decay), P = tril(Q K^T * decay)
|
||||
for (int ti = warp; ti < 2 * (C / 16) * (C / N) && (V != 32 || warp < 4); ti += (V == 32 ? 4 : WARPS)) {
|
||||
for (int ti = warp; ti < 2 * (C / 16) * (C / N) && (V != 32 || warp < 4); ti += (V == 32 ? 4 : warps)) {
|
||||
const bool gram = ti < (C / 16) * (C / N);
|
||||
const int t = ti % ((C / 16) * (C / N)), r = (t / (C / N)) * 16, c = (t % (C / N)) * N;
|
||||
acc x;
|
||||
@@ -280,27 +315,27 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
}
|
||||
__syncthreads();
|
||||
// inverse = (I + A)^-1 by forward substitution
|
||||
for (int row = warp; row < C; row += WARPS) {
|
||||
float x[(C + 31) / 32];
|
||||
for (int row = warp; row < C; row += warps) {
|
||||
float x[(C + warp_size - 1) / warp_size];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < (C + 31) / 32; ++j) {
|
||||
const int col = lane + 32 * j;
|
||||
x[j] = col == row ? 1.f : (col < row ? -s.lower[s.lower_index(row, col)] : 0.f);
|
||||
for (int j = 0; j < (C + warp_size - 1) / warp_size; ++j) {
|
||||
const int col = lane + warp_size * j;
|
||||
x[j] = col == row ? 1.f : (col < row ? -s.lower[s.lower_index(row, col)] : 0.f);
|
||||
}
|
||||
for (int pivot = row - 1; pivot > 0; --pivot) {
|
||||
const float xp = ggml_cuda_shfl(x[pivot / 32], pivot % 32, 32);
|
||||
const float xp = ggml_cuda_shfl(x[pivot / warp_size], pivot % warp_size, warp_size);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < (C + 31) / 32; ++j) {
|
||||
const int col = lane + 32 * j;
|
||||
for (int j = 0; j < (C + warp_size - 1) / warp_size; ++j) {
|
||||
const int col = lane + warp_size * j;
|
||||
if (col < pivot) {
|
||||
x[j] -= xp * s.lower[s.lower_index(pivot, col)];
|
||||
}
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < (C + 31) / 32; ++j) {
|
||||
if (lane + 32 * j < C) {
|
||||
s.inverse.store(row, lane + 32 * j, x[j]);
|
||||
for (int j = 0; j < (C + warp_size - 1) / warp_size; ++j) {
|
||||
if (lane + warp_size * j < C) {
|
||||
s.inverse.store(row, lane + warp_size * j, x[j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -309,8 +344,9 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
}
|
||||
__syncthreads();
|
||||
// solved = inverse * delta
|
||||
for (int ti = warp; ti < (C / 16) * (V / N); ti += WARPS) {
|
||||
const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
for (int ti = warp; ti < (C / 16) * (V / N); ti += warps) {
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
acc x;
|
||||
product<C, false, true>(x, s.inverse, s.delta, r, c);
|
||||
store(s.scratch.solved, x, r, c);
|
||||
@@ -318,21 +354,25 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
__syncthreads();
|
||||
// out = base output + P * solved
|
||||
for (int ti = (SPLIT_PROJECTION ? warp - 4 : warp); ti < (C / 16) * (V / N) && (!SPLIT_PROJECTION || warp >= 4);
|
||||
ti += WARPS) {
|
||||
const int r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
ti += warps) {
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
acc x;
|
||||
product<C, false, true>(x, s.p, s.scratch.solved, r, c);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
const int t = r + acc::get_i(i), v = c + acc::get_j(i);
|
||||
const int t = r + acc::get_i(i);
|
||||
const int v = c + acc::get_j(i);
|
||||
if (t < valid) {
|
||||
out[(t0 + t) * a.H * D + v] = base_output[ti / WARPS].x[i] + x.x[i];
|
||||
out[(t0 + t) * a.H * D + v] = base_output[ti / warps].x[i] + x.x[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
// S = decay(chunk) * S + K^T * (solved * decay to chunk end)
|
||||
for (int i = tid; i < C * V; i += 256) {
|
||||
const int t = i / V, v = i % V, j = s.scratch.solved.index(t, v);
|
||||
for (int i = tid; i < C * V; i += warps * warp_size) {
|
||||
const int t = i / V;
|
||||
const int v = i % V;
|
||||
const int j = s.scratch.solved.index(t, v);
|
||||
const float x = __bfloat162float(s.scratch.solved.hi[j]) + __bfloat162float(s.scratch.solved.lo[j]);
|
||||
s.delta.store(t, v, x * expf(s.prefix[valid - 1] - s.prefix[t]));
|
||||
}
|
||||
@@ -340,7 +380,9 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
const float decay = expf(s.prefix[valid - 1]);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < PER_WARP; ++j) {
|
||||
const int ti = warp + j * WARPS, r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
const int ti = warp + j * warps;
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
state[j].x[i] *= decay;
|
||||
@@ -351,7 +393,9 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < PER_WARP; ++j) {
|
||||
const int ti = warp + j * WARPS, r = (ti / (V / N)) * 16, c = (ti % (V / N)) * N;
|
||||
const int ti = warp + j * warps;
|
||||
const int r = (ti / (V / N)) * 16;
|
||||
const int c = (ti % (V / N)) * N;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < acc::ne; ++i) {
|
||||
a.state_out[soff + (v0 + c + acc::get_j(i)) * D + r + acc::get_i(i)] = state[j].x[i];
|
||||
@@ -360,10 +404,11 @@ static __global__ __launch_bounds__(256, BLOCKS) void gdn_single(ggml_cuda_gdn_m
|
||||
#else
|
||||
GGML_UNUSED_VARS(a);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3))
|
||||
#endif // defined(AMPERE_MMA_AVAILABLE) || (defined(AMD_WMMA_AVAILABLE) && defined(RDNA3)) || defined(AMD_MFMA_AVAILABLE)
|
||||
}
|
||||
|
||||
template <int C, int V, int BLOCKS = 1> bool init(int device) {
|
||||
template <int C, int V, int BLOCKS = 1>
|
||||
static bool init(int device) {
|
||||
static std::once_flag once[GGML_CUDA_MAX_DEVICES];
|
||||
static bool available[GGML_CUDA_MAX_DEVICES] = {};
|
||||
std::call_once(once[device], [device] {
|
||||
@@ -381,7 +426,13 @@ template <int C, int V, int BLOCKS = 1> bool init(int device) {
|
||||
return available[device];
|
||||
}
|
||||
|
||||
enum class gdn_path { ar, slice32_one_block, slice32_two_blocks, slice64, whole_head };
|
||||
enum class gdn_path {
|
||||
ar,
|
||||
slice32_one_block,
|
||||
slice32_two_blocks,
|
||||
slice64,
|
||||
whole_head
|
||||
};
|
||||
|
||||
static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a) {
|
||||
GGML_ASSERT(device >= 0 && device < GGML_CUDA_MAX_DEVICES);
|
||||
@@ -391,10 +442,25 @@ static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a
|
||||
}
|
||||
const auto & info = ggml_cuda_info().devices[device];
|
||||
#ifdef GGML_USE_HIP
|
||||
if (!GGML_CUDA_CC_IS_RDNA3_5(info.cc) || a.n_tokens < 2048) {
|
||||
return gdn_path::ar;
|
||||
if (GGML_CUDA_CC_IS_RDNA3_5(info.cc)) {
|
||||
if (a.n_tokens < 2048) {
|
||||
return gdn_path::ar;
|
||||
}
|
||||
return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar;
|
||||
}
|
||||
return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar;
|
||||
if (GGML_CUDA_CC_IS_CDNA(info.cc)) {
|
||||
if (!amd_mfma_available(info.cc)) {
|
||||
return gdn_path::ar;
|
||||
}
|
||||
const int64_t blocks = a.H * a.n_seqs * 2;
|
||||
const int64_t blocks_nwaves = (blocks + info.nsm - 1) / info.nsm;
|
||||
const int64_t blocks_efficiency = 100 * blocks / (info.nsm * blocks_nwaves);
|
||||
if (blocks_nwaves > 1 && blocks_efficiency < 60) {
|
||||
return gdn_path::ar;
|
||||
}
|
||||
return init<16, 64>(device) ? gdn_path::slice64 : gdn_path::ar;
|
||||
}
|
||||
return gdn_path::ar;
|
||||
#elif defined(GGML_USE_MUSA)
|
||||
GGML_UNUSED_VARS(info);
|
||||
return gdn_path::ar;
|
||||
@@ -428,7 +494,8 @@ bool ggml_cuda_gdn_mma_launch(int device, const ggml_cuda_gdn_mma_args & a, cuda
|
||||
return false;
|
||||
}
|
||||
#ifdef GGML_USE_HIP
|
||||
gdn_single<16, 64><<<a.H * a.n_seqs * 2, dim3(32, 8), sizeof(shared<16, 64>), stream>>>(a);
|
||||
const int warp_size = ggml_cuda_info().devices[device].warp_size;
|
||||
gdn_single<16, 64><<<a.H * a.n_seqs * 2, dim3(warp_size, 256 / warp_size), sizeof(shared<16, 64>), stream>>>(a);
|
||||
#else
|
||||
if (path == gdn_path::slice32_two_blocks) {
|
||||
gdn_single<16, 32, 2><<<a.H * a.n_seqs * 4, dim3(32, 8), sizeof(shared<16, 32>), stream>>>(a);
|
||||
|
||||
@@ -2526,7 +2526,7 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c
|
||||
} else if (src0->type == GGML_TYPE_F8_E4M3 &&
|
||||
ggml_cuda_should_use_mmvf(src0, cc, warp_size, dst->ne[2])) {
|
||||
return false;
|
||||
} else if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) {
|
||||
} else if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc) && src0->ne[0] % 2 == 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -2570,7 +2570,8 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
|
||||
ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst);
|
||||
return;
|
||||
} else {
|
||||
if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc)) {
|
||||
// MMVF reads src0 in pairs of values, so it needs an even number of columns
|
||||
if (src0->type != GGML_TYPE_F8_E4M3 && GGML_CUDA_CC_IS_AMD(cc) && ne00 % 2 == 0) {
|
||||
ggml_cuda_mul_mat_vec_f(ctx, src0, src1, ids, dst);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -378,6 +378,14 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
|
||||
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
||||
}
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_RDNA3_5(cc)) {
|
||||
switch (type) { // tuned on Radeon 8060S (gfx1151)
|
||||
case GGML_TYPE_Q8_0:
|
||||
return ne11 <= 6;
|
||||
default:
|
||||
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
||||
}
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_CDNA(cc)) {
|
||||
if (GGML_CUDA_CC_IS_CDNA1(cc)) {
|
||||
switch (type) {
|
||||
|
||||
Vendored
+2
-2
@@ -240,9 +240,9 @@ typedef uint64_t ggml_cuda_thread_mask_t;
|
||||
#define RDNA2
|
||||
#endif
|
||||
|
||||
#if defined(__gfx1010__) || defined(__gfx1012__)
|
||||
#if defined(__gfx1010__) || defined(__gfx1012__) || defined(__gfx1013__)
|
||||
#define RDNA1
|
||||
#endif // defined(__gfx1010__) || defined(__gfx1012__)
|
||||
#endif // defined(__gfx1010__) || defined(__gfx1012__) || defined(__gfx1013__)
|
||||
|
||||
#if defined(RDNA4) || defined(RDNA4_M) || defined(RDNA3) || defined(RDNA2) || defined(RDNA1)
|
||||
#define RDNA // For the entire family
|
||||
|
||||
@@ -712,6 +712,9 @@ struct ggml_backend_opencl_context {
|
||||
size_t image_max_buffer_size;
|
||||
size_t image2d_max_width;
|
||||
size_t image2d_max_height;
|
||||
size_t image3d_max_width;
|
||||
size_t image3d_max_height;
|
||||
size_t image3d_max_depth;
|
||||
|
||||
cl_device_svm_capabilities svm_caps;
|
||||
|
||||
@@ -1431,6 +1434,7 @@ struct ggml_backend_opencl_context {
|
||||
cl_kernel kernel_gemv_noshuffle_q4_k_f32_mc3; // multi-column (N=3) verify GEMV
|
||||
cl_kernel kernel_gemm_noshuffle_q4_k_f32;
|
||||
cl_kernel kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin;
|
||||
cl_kernel kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin;
|
||||
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin;
|
||||
cl_kernel kernel_gemv_noshuffle_q4_k_f32_32b_trans;
|
||||
cl_kernel kernel_gemm_noshuffle_q4_k_q8_1_dp4a = nullptr; // dp4a (int8) dense prefill GEMM
|
||||
@@ -6701,6 +6705,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
|
||||
backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans = nullptr;
|
||||
backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin = nullptr;
|
||||
backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin = nullptr;
|
||||
backend_ctx->kernel_gemm_noshuffle_q4_k_q8_1_dp4a_ila_a8_bin = nullptr;
|
||||
{
|
||||
if (backend_ctx->has_vector_subgroup_broadcast) {
|
||||
@@ -6745,6 +6750,17 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
CL_CHECK(clReleaseProgram(bin_prog));
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q4_k_f32_32b_trans_ila_a7", &bin_size);
|
||||
if (kernel_bin && bin_size > 0) {
|
||||
cl_program bin_prog =
|
||||
build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size);
|
||||
|
||||
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin =
|
||||
clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7", &err), err));
|
||||
CL_CHECK(clReleaseProgram(bin_prog));
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9258,6 +9274,9 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE_MAX_BUFFER_SIZE, sizeof(size_t), &backend_ctx->image_max_buffer_size, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_WIDTH, sizeof(size_t), &backend_ctx->image2d_max_width, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE2D_MAX_HEIGHT, sizeof(size_t), &backend_ctx->image2d_max_height, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE3D_MAX_WIDTH, sizeof(size_t), &backend_ctx->image3d_max_width, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE3D_MAX_HEIGHT, sizeof(size_t), &backend_ctx->image3d_max_height, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_IMAGE3D_MAX_DEPTH, sizeof(size_t), &backend_ctx->image3d_max_depth, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(size_t), &backend_ctx->max_workgroup_size, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_LOCAL_MEM_SIZE, sizeof(cl_ulong), &backend_ctx->local_mem_size, NULL));
|
||||
CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_SVM_CAPABILITIES, sizeof(cl_device_svm_capabilities), &backend_ctx->svm_caps, 0));
|
||||
@@ -11660,6 +11679,26 @@ static bool use_fa_bin_kernels_prefill(const ggml_backend_opencl_context * backe
|
||||
|
||||
constexpr bool prefill_only = true;
|
||||
|
||||
// The repacked Q/K/V are 3D images: Q {n_q, n_batch*n_head, dk/4}, K {dk, n_kv/4, n_batch*n_head_kv},
|
||||
// V {n_kv, dv/4, n_batch*n_head_kv}. Past the device's 3D image limits clCreateImage fails and the
|
||||
// call asserts (X2-90: V is n_kv wide, so any prefill past ~16k tokens of context), so decline those shapes.
|
||||
const size_t n_kv = (size_t) k->ne[1];
|
||||
const size_t n_bh = (size_t) q->ne[3] * (size_t) q->ne[2];
|
||||
const size_t n_bhkv = (size_t) q->ne[3] * (size_t) k->ne[2];
|
||||
if ((size_t) n_q > backend_ctx->image3d_max_width || n_bh > backend_ctx->image3d_max_height ||
|
||||
(size_t) dk / 4 > backend_ctx->image3d_max_depth ||
|
||||
(size_t) dk > backend_ctx->image3d_max_width || (n_kv + 3) / 4 > backend_ctx->image3d_max_height ||
|
||||
n_kv > backend_ctx->image3d_max_width || (size_t) dv / 4 > backend_ctx->image3d_max_height ||
|
||||
n_bhkv > backend_ctx->image3d_max_depth) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// q8_0 K/V at DK=512 gives wrong results here when V differs from K (f16 K/V is correct).
|
||||
// Decline it, as this backend does without the kernel library.
|
||||
if (is_q8_0 && dk == 512) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return (backend_ctx->gpu_family == GPU_FAMILY::ADRENO &&
|
||||
(is_mixed || is_q8_0) && (dk == dv)
|
||||
&& (dk == 64 || dk == 128 || dk == 256 || dk == 512)
|
||||
@@ -11775,7 +11814,8 @@ inline bool use_q6_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
|
||||
inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
if (!backend_ctx->kernel_gemv_noshuffle_q4_k_f32_32b_trans ||
|
||||
!backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin) {
|
||||
!(backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin ||
|
||||
backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin)) {
|
||||
return false;
|
||||
}
|
||||
// bin kernels require ne2 == 1 and ne3 == 1 for weights
|
||||
@@ -22794,6 +22834,7 @@ static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_te
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
|
||||
|
||||
CL_CHECK(clReleaseMemObject(mem_tex_matrixO_1dbuf));
|
||||
CL_CHECK(clReleaseMemObject(mem_tex_mask_fallback_1dbuf));
|
||||
CL_CHECK(clReleaseMemObject(img_q_wmm));
|
||||
CL_CHECK(clReleaseMemObject(img_k_wmm));
|
||||
CL_CHECK(clReleaseMemObject(img_v_wmm));
|
||||
@@ -27675,6 +27716,8 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const gg
|
||||
const int gemm_tile_n = 64;
|
||||
int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n;
|
||||
|
||||
const bool have_a8 = backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin != nullptr;
|
||||
|
||||
static const char * q4_k_bin_dp4a_env = getenv("GGML_OPENCL_Q4_K_BIN_DP4A");
|
||||
bool q4_k_bin_dp4a_on = q4_k_bin_dp4a_env
|
||||
? (atoi(q4_k_bin_dp4a_env) != 0)
|
||||
@@ -27744,6 +27787,74 @@ static void ggml_cl_mul_mat_q4_k_f32_adreno_ila(ggml_backend_t backend, const gg
|
||||
return;
|
||||
}
|
||||
|
||||
if (!have_a8) {
|
||||
cl_mem b_src_sub = nullptr;
|
||||
cl_mem b_pad_sub = nullptr;
|
||||
cl_mem b_buf = nullptr;
|
||||
region.origin = offset1;
|
||||
region.size = (size_t)K * N * sizeof(float);
|
||||
CL_CHECK((b_src_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
|
||||
if (N_pad == N) {
|
||||
b_buf = b_src_sub;
|
||||
} else {
|
||||
size_t padded_bytes = (size_t)K * N_pad * sizeof(float);
|
||||
backend_ctx->prealloc_act_trans.allocate(context, padded_bytes);
|
||||
region.origin = 0;
|
||||
region.size = padded_bytes;
|
||||
CL_CHECK((b_pad_sub = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
|
||||
|
||||
const float b_pad_zero = 0.0f;
|
||||
CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_pad_sub, &b_pad_zero, sizeof(float),
|
||||
0, padded_bytes, 0, NULL, NULL));
|
||||
CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, b_src_sub, b_pad_sub, 0, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL));
|
||||
b_buf = b_pad_sub;
|
||||
}
|
||||
|
||||
img_fmt = { CL_R, CL_FLOAT };
|
||||
memset(&img_desc, 0, sizeof(img_desc));
|
||||
img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
|
||||
img_desc.image_width = (size_t)K * N_pad;
|
||||
img_desc.buffer = b_buf;
|
||||
cl_mem b_img_a7;
|
||||
CL_CHECK((b_img_a7 = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
|
||||
|
||||
const int c_offset_for_kernel = (int)offsetd;
|
||||
|
||||
int line_stride_B = K * (int)sizeof(float);
|
||||
int line_stride_C = M * (int)sizeof(float);
|
||||
|
||||
kernel = backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a7_bin;
|
||||
cl_uint k_arg = 0;
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->q_img));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->d));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->dm));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q4_k->s));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img_a7));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset1));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_C));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N));
|
||||
|
||||
size_t local_work_size[3] = { 64, 2, 4 };
|
||||
size_t m_tiles = (size_t)CEIL_DIV(M, 64);
|
||||
size_t global_y = ((m_tiles + 1) / 2) * 2;
|
||||
size_t global_work_size[3] = { 64, global_y, (size_t)CEIL_DIV(N_pad, 32) };
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
|
||||
|
||||
CL_CHECK(clReleaseMemObject(b_img_a7));
|
||||
if (b_src_sub) {
|
||||
CL_CHECK(clReleaseMemObject(b_src_sub));
|
||||
}
|
||||
if (b_pad_sub) {
|
||||
CL_CHECK(clReleaseMemObject(b_pad_sub));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
cl_mem b_sub_buf = nullptr;
|
||||
cl_mem b_padded = nullptr;
|
||||
cl_mem b_buf = nullptr;
|
||||
|
||||
@@ -335,7 +335,7 @@ static int ggml_vk_comm_sfd_export(vk_comm_device & d) {
|
||||
int fd = -1;
|
||||
VkSemaphoreGetFdInfoKHR gi{};
|
||||
gi.sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR;
|
||||
gi.semaphore = d.sfd.sem;
|
||||
gi.semaphore = static_cast<VkSemaphore>(d.sfd.sem);
|
||||
gi.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
|
||||
const VkResult result = d.sfd.get_fd((VkDevice) d.ctx->device->device, &gi, &fd);
|
||||
if (result != VK_SUCCESS) {
|
||||
@@ -348,7 +348,7 @@ static int ggml_vk_comm_sfd_export(vk_comm_device & d) {
|
||||
static void ggml_vk_comm_sfd_import(vk_comm_device & d, vk::Semaphore sem, int fd) {
|
||||
VkImportSemaphoreFdInfoKHR isi{};
|
||||
isi.sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR;
|
||||
isi.semaphore = sem;
|
||||
isi.semaphore = static_cast<VkSemaphore>(sem);
|
||||
isi.flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT;
|
||||
isi.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
|
||||
isi.fd = fd;
|
||||
|
||||
@@ -70,7 +70,7 @@ void ggml_vk_buffer_memset(vk_buffer& dst, size_t offset, uint32_t c, size_t siz
|
||||
vk_buffer ggml_vk_buffer_from_host_ptr(vk_device & device, void * ptr, size_t size);
|
||||
|
||||
// pipelines
|
||||
uint64_t vk_tensor_offset(const ggml_tensor * tensor);
|
||||
size_t vk_tensor_offset(const ggml_tensor * tensor);
|
||||
uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t);
|
||||
void ggml_vk_wait_for_fence(ggml_backend_vk_context * ctx);
|
||||
void ggml_pipeline_request_descriptor_sets(ggml_backend_vk_context *ctx, vk_pipeline& pipeline, uint32_t n);
|
||||
|
||||
@@ -466,15 +466,15 @@ struct vk_conv2d_pipeline_state {
|
||||
|
||||
struct vk_conv3d_pipeline_state {
|
||||
vk_conv3d_pipeline_state(uint32_t s0, uint32_t s1, uint32_t s2, uint32_t p0, uint32_t p1, uint32_t p2,
|
||||
uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned)
|
||||
: s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned) {}
|
||||
uint32_t d0, uint32_t d1, uint32_t d2, uint32_t KW, uint32_t KH, uint32_t KD, uint32_t aligned, uint32_t split_k)
|
||||
: s0(s0), s1(s1), s2(s2), p0(p0), p1(p1), p2(p2), d0(d0), d1(d1), d2(d2), KW(KW), KH(KH), KD(KD), aligned(aligned), split_k(split_k) {}
|
||||
|
||||
uint32_t s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD;
|
||||
uint32_t aligned;
|
||||
uint32_t aligned, split_k;
|
||||
|
||||
bool operator<(const vk_conv3d_pipeline_state &b) const {
|
||||
return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned) <
|
||||
std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned);
|
||||
return std::tie(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned, split_k) <
|
||||
std::tie(b.s0, b.s1, b.s2, b.p0, b.p1, b.p2, b.d0, b.d1, b.d2, b.KW, b.KH, b.KD, b.aligned, b.split_k);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -259,7 +259,7 @@ void ggml_vk_print_device_lost_info(const vk_device& device) {
|
||||
}
|
||||
void * const vk_ptr_base = (void *)(uintptr_t) 0x1000; // NOLINT
|
||||
|
||||
uint64_t vk_tensor_offset(const ggml_tensor * tensor) {
|
||||
size_t vk_tensor_offset(const ggml_tensor * tensor) {
|
||||
if (tensor->view_src) {
|
||||
return (uint8_t *) tensor->view_src->data - (uint8_t *) vk_ptr_base;
|
||||
}
|
||||
@@ -1258,10 +1258,14 @@ static vk_fa_tuning_params get_fa_tuning_params_scalar(const vk_device& device,
|
||||
|
||||
result.shmem_staging = (device->vendor_id == VK_VENDOR_ID_NVIDIA && hsk < 256 && hsv < 256) ? 1 : 0;
|
||||
|
||||
if (!reduce_block_rows && !ggml_vk_flash_attn_scalar_shmem_support(device, result, hsk, hsv, f32acc, k_type, v_type)) {
|
||||
if (!reduce_block_rows && result.block_rows > 1 && !ggml_vk_flash_attn_scalar_shmem_support(device, result, hsk, hsv, f32acc, k_type, v_type)) {
|
||||
result.block_rows /= 2;
|
||||
}
|
||||
|
||||
if (result.shmem_staging && !ggml_vk_flash_attn_scalar_shmem_support(device, result, hsk, hsv, f32acc, k_type, v_type)) {
|
||||
result.shmem_staging = 0;
|
||||
}
|
||||
|
||||
// With head sizes above 128, 16 rows per workgroup spill registers on Qualcomm and run about
|
||||
// 40x slower; at large KV a single dispatch can then exceed the Windows GPU timeout.
|
||||
if (device->vendor_id == VK_VENDOR_ID_QUALCOMM && (hsk > 128 || hsv > 128)) {
|
||||
@@ -3886,8 +3890,16 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
// variant (spec constant 1) additionally gathers the qwen4 indexer input on the fly.
|
||||
{
|
||||
const uint32_t BLOCK_SIZE = 1u << std::min(10u, device->max_workgroup_size_log2);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_f32, "topk_radix_f32", topk_radix_select_f32_len, topk_radix_select_f32_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 0}, 1, true);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_qsa, "topk_radix_qsa", topk_radix_select_f32_len, topk_radix_select_f32_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 1}, 1, true);
|
||||
// the bucket scan needs one invocation per radix bucket
|
||||
if (BLOCK_SIZE >= 256) {
|
||||
if (device->subgroup_arithmetic && device->subgroup_require_full_support) {
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_f32, "topk_radix_f32", topk_radix_select_f32_subgroup_len, topk_radix_select_f32_subgroup_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 0, device->subgroup_size}, 1, true, true, device->subgroup_size);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_qsa, "topk_radix_qsa", topk_radix_select_f32_subgroup_len, topk_radix_select_f32_subgroup_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 1, device->subgroup_size}, 1, true, true, device->subgroup_size);
|
||||
} else {
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_f32, "topk_radix_f32", topk_radix_select_f32_len, topk_radix_select_f32_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 0}, 1, true);
|
||||
ggml_vk_create_pipeline2(device, device->pipeline_topk_radix_qsa, "topk_radix_qsa", topk_radix_select_f32_len, topk_radix_select_f32_data, "main", 5, sizeof(vk_op_topk_radix_push_constants), {BLOCK_SIZE, 1, 1}, {BLOCK_SIZE, 1}, 1, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ggml_vk_create_pipeline(device, device->pipeline_argmax_f32, "argmax_f32", argmax_f32_len, argmax_f32_data, "main", 2, sizeof(vk_op_push_constants), {1, 1, 1}, { device->subgroup_size }, 1);
|
||||
@@ -4282,6 +4294,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
spec_constants_cpy.push_back(conv2d_csh_store); \
|
||||
spec_constants_cpy.push_back(conv2d_WM); \
|
||||
spec_constants_cpy.push_back(conv2d_WN); \
|
||||
spec_constants_cpy.push_back(state.split_k); \
|
||||
ggml_vk_create_pipeline( \
|
||||
device, c.second, "conv3d" #type_suffix, \
|
||||
conv3d##type_suffix##spv_suffix##_len, conv3d##type_suffix##spv_suffix##_data, "main", 3, \
|
||||
@@ -8520,7 +8533,9 @@ bool ggml_vk_flash_attn_scalar_shmem_support(const vk_device& device, const vk_f
|
||||
kblocksh_size = 0;
|
||||
}
|
||||
|
||||
const uint32_t total_size = tmpsh + tmpshv4 + masksh + iq_shmem + Qf + kvsh + kblocksh_size;
|
||||
const uint32_t occupancy_limiter = std::max(params.limit_occupancy_shmem, 1u) * 4 * sizeof(float);
|
||||
|
||||
const uint32_t total_size = tmpsh + tmpshv4 + masksh + iq_shmem + Qf + kvsh + kblocksh_size + occupancy_limiter;
|
||||
const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize;
|
||||
|
||||
VK_LOG_DEBUG("ggml_vk_flash_attn_scalar_shmem_support(HSK=" << hsk << ", HSV=" << hsv << ", mmq=" << mmq << ", total_size=" << total_size << ", supported=" << supported);
|
||||
@@ -9271,24 +9286,58 @@ static vk_conv_shapes ggml_vk_conv_select_shape(ggml_backend_vk_context * ctx, u
|
||||
}
|
||||
}
|
||||
|
||||
static vk_conv_shapes ggml_vk_conv3d_select_shape(ggml_backend_vk_context * ctx, uint32_t K, uint32_t NPQ) {
|
||||
const auto & block = vk_conv_block_sizes[CONV_SHAPE_256x128];
|
||||
static vk_conv_shapes ggml_vk_conv3d_select_shape(ggml_backend_vk_context * ctx, const ggml_tensor * kernel, const ggml_tensor * dst, uint32_t & split_k) {
|
||||
split_k = 1;
|
||||
const uint32_t K = ggml_get_op_params_i32(dst, 11);
|
||||
const uint32_t NPQ = ggml_get_op_params_i32(dst, 10) * dst->ne[0] * dst->ne[1] * dst->ne[2];
|
||||
const vk_conv_shapes fallback = ggml_vk_conv_select_shape(ctx, K, NPQ);
|
||||
const uint32_t shader_core_count = ctx->device->shader_core_count;
|
||||
// Slowdowns were observed on DGX Spark with drivers older than R615.
|
||||
const bool driver_supported = ctx->device->driver_id != vk::DriverId::eNvidiaProprietary ||
|
||||
(ctx->device->properties.driverVersion >> 22) >= 615;
|
||||
if (ctx->device->coopmat2 && shader_core_count > 0 && K % block.K == 0 && driver_supported) {
|
||||
const uint64_t n_tiles = (uint64_t)(K / block.K) * CEIL_DIV((uint64_t)NPQ, block.NPQ);
|
||||
// Use the largest cm2 padding and include the dummy Csh[8].
|
||||
const uint32_t pad = 8, csh_elems = 8;
|
||||
const uint32_t shmem = (block.K * (block.CRS + pad) + block.CRS * (block.NPQ + pad) + csh_elems) * sizeof(uint16_t);
|
||||
// Reuse each loaded input across 256 output channels instead of 128.
|
||||
// Require full channel tiles and at least as many workgroups as GPU cores.
|
||||
if (n_tiles >= shader_core_count && shmem <= ctx->device->properties.limits.maxComputeSharedMemorySize) {
|
||||
return CONV_SHAPE_256x128;
|
||||
}
|
||||
if (!ctx->device->coopmat2 || shader_core_count == 0 || K == 0 || NPQ == 0 || !driver_supported) {
|
||||
return fallback;
|
||||
}
|
||||
return ggml_vk_conv_select_shape(ctx, K, NPQ);
|
||||
const auto & block = vk_conv_block_sizes[CONV_SHAPE_256x128];
|
||||
const auto & limits = ctx->device->properties.limits;
|
||||
// Use the largest cm2 padding and include the dummy Csh[8].
|
||||
const uint32_t pad = 8, csh_elems = 8;
|
||||
const uint32_t shmem = (block.K * (block.CRS + pad) + block.CRS * (block.NPQ + pad) + csh_elems) * sizeof(uint16_t);
|
||||
if (shmem > limits.maxComputeSharedMemorySize) {
|
||||
return fallback;
|
||||
}
|
||||
const uint32_t channel_tiles = CEIL_DIV(K, block.K);
|
||||
const uint64_t n_tiles = (uint64_t)channel_tiles * CEIL_DIV(NPQ, block.NPQ);
|
||||
// Keep full channel tiles when the grid already fills the GPU.
|
||||
if (n_tiles >= shader_core_count) {
|
||||
return K % block.K == 0 ? CONV_SHAPE_256x128 : fallback;
|
||||
}
|
||||
static const bool disable_split_k = getenv("GGML_VK_DISABLE_CONV3D_SPLIT_K") != nullptr;
|
||||
// Do not use a tile with more than half its output channels padded.
|
||||
if (disable_split_k || K < block.K / 2 || !ggml_is_contiguous(dst) || get_misalign_bytes(ctx, dst) != 0) {
|
||||
return fallback;
|
||||
}
|
||||
// Use power-of-two splits to give each core at least one workgroup.
|
||||
uint64_t splits = 1;
|
||||
while (n_tiles * splits < shader_core_count) {
|
||||
splits *= 2;
|
||||
}
|
||||
const uint64_t crs = (uint64_t)ggml_get_op_params_i32(dst, 9) * kernel->ne[0] * kernel->ne[1] * kernel->ne[2];
|
||||
const uint64_t operand_bytes = block.K * ggml_type_size(kernel->type) + block.NPQ * sizeof(float);
|
||||
const uint64_t scratch_bytes = 2 * sizeof(float) * block.K * block.NPQ;
|
||||
// Do not add more scratch bytes than the estimated operand load bytes.
|
||||
if (crs * operand_bytes < splits * scratch_bytes) {
|
||||
return fallback;
|
||||
}
|
||||
const uint64_t ne = ggml_nelements(dst);
|
||||
const uint64_t bytes = ggml_nbytes(dst);
|
||||
if (ne == 0 || CEIL_DIV(ne, ctx->device->pipeline_matmul_split_k_reduce->wg_denoms[0]) > limits.maxComputeWorkGroupCount[0] ||
|
||||
splits > limits.maxStorageBufferRange / bytes ||
|
||||
splits > limits.maxComputeWorkGroupCount[0] / channel_tiles) {
|
||||
return fallback;
|
||||
}
|
||||
split_k = (uint32_t)splits;
|
||||
return CONV_SHAPE_256x128;
|
||||
}
|
||||
|
||||
static vk_pipeline ggml_vk_rms_norm_pipeline(const std::array<vk_pipeline, 3> & pipelines, int64_t ncols) {
|
||||
@@ -9997,7 +10046,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
const uint32_t IC = (uint32_t)ggml_get_op_params_i32(dst, 9);
|
||||
const uint32_t N = (uint32_t)ggml_get_op_params_i32(dst, 10);
|
||||
const uint32_t NPQ = N * dst->ne[2] * dst->ne[1] * dst->ne[0];
|
||||
const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, OC, NPQ);
|
||||
uint32_t split_k;
|
||||
const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, src0, dst, split_k);
|
||||
|
||||
const uint32_t KW = (uint32_t)src0->ne[0];
|
||||
const uint32_t KH = (uint32_t)src0->ne[1];
|
||||
@@ -10020,7 +10070,7 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
(CRS % BS_CRS == 0) &&
|
||||
(NPQ % BS_NPQ == 0)) ? 1u : 0u;
|
||||
|
||||
vk_conv3d_pipeline_state conv3d_pipeline_state(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned);
|
||||
vk_conv3d_pipeline_state conv3d_pipeline_state(s0, s1, s2, p0, p1, p2, d0, d1, d2, KW, KH, KD, aligned, split_k);
|
||||
|
||||
std::map<vk_conv3d_pipeline_state, vk_pipeline> *pipelines = nullptr;
|
||||
if (src0->type == GGML_TYPE_F32) {
|
||||
@@ -10126,6 +10176,7 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
|
||||
init_pushconst_tensor_offsets(ctx, pc, src0, src1, src2, src3, dst);
|
||||
|
||||
std::array<uint32_t, 3> elements;
|
||||
uint32_t split_k = 1;
|
||||
|
||||
switch (op) {
|
||||
case GGML_OP_NORM:
|
||||
@@ -10301,10 +10352,12 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
|
||||
case GGML_OP_CONV_3D:
|
||||
if constexpr (std::is_same_v<PC, vk_op_conv3d_push_constants>) {
|
||||
const uint32_t NPQ = pc.N * pc.OD * pc.OH * pc.OW;
|
||||
const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, pc.OC, NPQ);
|
||||
const uint32_t NPQ_blocks = CEIL_DIV(NPQ, vk_conv_block_sizes[shape].NPQ);
|
||||
const vk_conv_shapes shape = ggml_vk_conv3d_select_shape(ctx, src0, dst, split_k);
|
||||
const auto & block = vk_conv_block_sizes[shape];
|
||||
const uint32_t NPQ_blocks = CEIL_DIV(NPQ, block.NPQ);
|
||||
const uint32_t channel_groups = split_k > 1 ? CEIL_DIV(pc.OC, block.K) * block.K * split_k : pc.OC;
|
||||
|
||||
elements = { pc.OC, NPQ_blocks, 1 };
|
||||
elements = { channel_groups, NPQ_blocks, 1 };
|
||||
if (elements[1] > 512) {
|
||||
elements[2] = CEIL_DIV(elements[1], 512);
|
||||
elements[1] = 512;
|
||||
@@ -10435,6 +10488,29 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co
|
||||
break;
|
||||
}
|
||||
|
||||
if constexpr (std::is_same_v<PC, vk_op_conv3d_push_constants>) {
|
||||
if (split_k > 1) {
|
||||
const uint32_t ne = (uint32_t)ggml_nelements(dst);
|
||||
const size_t scratch_size = ggml_nbytes(dst) * split_k;
|
||||
if (ctx->prealloc_size_split_k < scratch_size) {
|
||||
ctx->prealloc_size_split_k = scratch_size;
|
||||
ggml_vk_preallocate_buffers(ctx, subctx);
|
||||
}
|
||||
if (ctx->prealloc_split_k_need_sync) {
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
}
|
||||
vk_subbuffer scratch = { ctx->prealloc_split_k, 0, scratch_size };
|
||||
pc.dst_offset = 0;
|
||||
ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, 1);
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { src0_buf, src1_buf, scratch }, pc, elements);
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
const std::array<uint32_t, 2> reduce_pc = { ne, split_k };
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_matmul_split_k_reduce, { scratch, dst_buf }, reduce_pc, { ne, 1, 1 });
|
||||
ctx->prealloc_split_k_need_sync = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (op == GGML_OP_ADD || op == GGML_OP_RMS_NORM) {
|
||||
vk_subbuffer a_buf = src0_buf;
|
||||
if (ctx->do_add_rms_partials) {
|
||||
@@ -17228,7 +17304,7 @@ void vk_queue_handle_unsynchronized::submit(vk::ArrayProxy<const vk::SubmitInfo>
|
||||
|
||||
vk_device_struct::~vk_device_struct() {
|
||||
// the build failed before the device was created, nothing to destroy
|
||||
if (device == VK_NULL_HANDLE) {
|
||||
if (!device) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -112,6 +112,8 @@ const uint warps_M = BS_K / WM;
|
||||
const uint warps_N = BS_NPQ / WN;
|
||||
#endif
|
||||
|
||||
layout(constant_id = 22) const uint split_k = 1;
|
||||
|
||||
// without padding, ID_idx/IH_idx/IW_idx are in bounds by construction
|
||||
const bool dhw_in_bounds = (p0 == 0) && (p1 == 0) && (p2 == 0);
|
||||
|
||||
@@ -170,8 +172,11 @@ D,R,S=KD,KH,KW
|
||||
Z,P,Q=OD,OH,OW
|
||||
*/
|
||||
|
||||
uint32_t B_idx_K = gl_WorkGroupID.x;
|
||||
// Packing splits into the low bits would avoid runtime division, but adjacent channel tiles give better input cache reuse.
|
||||
uint32_t B_idx_K = split_k > 1 ? gl_WorkGroupID.x % splitWork(K, BS_K) : gl_WorkGroupID.x;
|
||||
uint32_t B_idx_split = split_k > 1 ? gl_WorkGroupID.x / splitWork(K, BS_K) : 0;
|
||||
uint32_t B_idx_NPQ = gl_WorkGroupID.y + gl_WorkGroupID.z * 512;
|
||||
uint32_t split_offset = B_idx_split * K * NPQ;
|
||||
|
||||
uint32_t T_y = tid / NT_NPQ;
|
||||
uint32_t T_x = tid % NT_NPQ;
|
||||
@@ -219,7 +224,7 @@ ACC_TYPE perElemOpStore(const in uint32_t r, const in uint32_t c, const in ACC_T
|
||||
split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx);
|
||||
uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3;
|
||||
if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
|
||||
dst_data[dst_idx + p.dst_offset] = D_TYPE(elem);
|
||||
dst_data[dst_idx + p.dst_offset + split_offset] = D_TYPE(elem);
|
||||
}
|
||||
return elem;
|
||||
}
|
||||
@@ -249,7 +254,11 @@ void main() {
|
||||
}
|
||||
#endif
|
||||
/* Advance block in CRS dim */
|
||||
[[dont_unroll]] for (uint32_t B_idx_CRS = 0; B_idx_CRS < NB_CRS; B_idx_CRS++) {
|
||||
const uint32_t crs_count = NB_CRS / split_k;
|
||||
const uint32_t crs_extra = NB_CRS % split_k;
|
||||
const uint32_t crs_begin = B_idx_split * crs_count + min(B_idx_split, crs_extra);
|
||||
const uint32_t crs_end = crs_begin + crs_count + uint32_t(B_idx_split < crs_extra);
|
||||
[[dont_unroll]] for (uint32_t B_idx_CRS = crs_begin; B_idx_CRS < crs_end; B_idx_CRS++) {
|
||||
uint32_t CRS_idx_a = B_idx_CRS * BS_CRS + Ac;
|
||||
uint32_t IC_idx_a;
|
||||
uint32_t KD_idx_a;
|
||||
@@ -398,7 +407,7 @@ void main() {
|
||||
split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx);
|
||||
uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3;
|
||||
if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
|
||||
dst_data[dst_idx + p.dst_offset] = D_TYPE(Csh[k_local * Csh_stride + npq_thread]);
|
||||
dst_data[dst_idx + p.dst_offset + split_offset] = D_TYPE(Csh[k_local * Csh_stride + npq_thread]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -420,7 +429,7 @@ void main() {
|
||||
split_npq(NPQ_idx, N_idx, OD_idx, OH_idx, OW_idx);
|
||||
uint32_t dst_idx = OW_idx + OH_idx * p.nb1 + OD_idx * p.nb2 + (N_idx * p.OC + K_idx) * p.nb3;
|
||||
if (aligned != 0 || (K_idx < K && NPQ_idx < NPQ)) {
|
||||
dst_data[dst_idx + p.dst_offset] = D_TYPE(regC[T_ly][T_lx]);
|
||||
dst_data[dst_idx + p.dst_offset + split_offset] = D_TYPE(regC[T_ly][T_lx]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,11 +2,20 @@
|
||||
|
||||
#extension GL_EXT_control_flow_attributes : enable
|
||||
#extension GL_EXT_shader_16bit_storage : require
|
||||
#if USE_SUBGROUPS
|
||||
#extension GL_KHR_shader_subgroup_basic : enable
|
||||
#extension GL_KHR_shader_subgroup_arithmetic : enable
|
||||
#endif
|
||||
|
||||
#include "types.glsl"
|
||||
|
||||
layout(constant_id = 0) const int BLOCK_SIZE = 1024;
|
||||
layout(constant_id = 1) const int QSA = 0; // 1: fuse the qwen4 QSA indexer gather + f16 mask
|
||||
#if USE_SUBGROUPS
|
||||
layout(constant_id = 2) const int SUBGROUP_SIZE = 32;
|
||||
|
||||
const uint NUM_SUBGROUPS = BLOCK_SIZE / SUBGROUP_SIZE;
|
||||
#endif
|
||||
|
||||
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
@@ -26,12 +35,19 @@ layout (push_constant) uniform parameter {
|
||||
} p;
|
||||
|
||||
#define RADIX_BITS 8
|
||||
#define UNROLL 4
|
||||
#define RADIX_SIZE (1 << RADIX_BITS)
|
||||
|
||||
shared uint histo[RADIX_SIZE];
|
||||
shared uint sh_bucket;
|
||||
shared uint sh_above;
|
||||
shared uint sh_tied;
|
||||
shared uint out_count;
|
||||
#if USE_SUBGROUPS
|
||||
shared uint sg_tied[NUM_SUBGROUPS];
|
||||
#else
|
||||
shared uint tie_scan[2 * BLOCK_SIZE];
|
||||
#endif
|
||||
|
||||
// order-preserving float -> uint mapping
|
||||
uint f2ui(float x) {
|
||||
@@ -87,52 +103,155 @@ void topk(const uint row) {
|
||||
const bool first = (shift == 32 - RADIX_BITS);
|
||||
const uint hi_mask = (shift + RADIX_BITS >= 32) ? 0u : (0xFFFFFFFFu << uint(shift + RADIX_BITS));
|
||||
const uint prefix_hi = prefix & hi_mask;
|
||||
for (uint i = tid; i < ncols; i += BLOCK_SIZE) {
|
||||
const uint key = f2ui(load(row, i, first));
|
||||
if ((key & hi_mask) == prefix_hi) {
|
||||
atomicAdd(histo[(key >> uint(shift)) & (RADIX_SIZE - 1)], 1u);
|
||||
uint run_b = RADIX_SIZE;
|
||||
uint run_n = 0;
|
||||
for (uint i0 = tid; i0 < ncols; i0 += UNROLL * BLOCK_SIZE) {
|
||||
uint keys[UNROLL];
|
||||
[[unroll]] for (uint u = 0; u < UNROLL; ++u) {
|
||||
const uint i = i0 + u * BLOCK_SIZE;
|
||||
keys[u] = f2ui(load(row, min(i, ncols - 1), first));
|
||||
}
|
||||
[[unroll]] for (uint u = 0; u < UNROLL; ++u) {
|
||||
const uint key = keys[u];
|
||||
if (i0 + u * BLOCK_SIZE < ncols && (key & hi_mask) == prefix_hi) {
|
||||
const uint b = (key >> uint(shift)) & (RADIX_SIZE - 1);
|
||||
if (b != run_b) {
|
||||
if (run_n != 0) {
|
||||
atomicAdd(histo[run_b], run_n);
|
||||
}
|
||||
run_b = b;
|
||||
run_n = 0;
|
||||
}
|
||||
run_n++;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (run_n != 0) {
|
||||
atomicAdd(histo[run_b], run_n);
|
||||
}
|
||||
barrier();
|
||||
|
||||
// top-down scan for the bucket holding the K-th value
|
||||
if (tid == 0) {
|
||||
uint acc = 0;
|
||||
uint b = 0;
|
||||
for (int bb = RADIX_SIZE - 1; bb >= 0; --bb) {
|
||||
const uint c = histo[bb];
|
||||
if (acc + c >= desired) { b = uint(bb); break; }
|
||||
acc += c;
|
||||
#if USE_SUBGROUPS
|
||||
uint c = 0, incl = 0;
|
||||
if (tid < RADIX_SIZE) {
|
||||
c = histo[RADIX_SIZE - 1 - tid];
|
||||
incl = subgroupInclusiveAdd(c);
|
||||
if (gl_SubgroupInvocationID == gl_SubgroupSize - 1) {
|
||||
sg_tied[gl_SubgroupID] = incl;
|
||||
}
|
||||
sh_bucket = b;
|
||||
sh_above = acc;
|
||||
}
|
||||
barrier();
|
||||
if (tid < RADIX_SIZE) {
|
||||
for (uint s = 0; s < gl_SubgroupID; ++s) {
|
||||
incl += sg_tied[s];
|
||||
}
|
||||
if (incl - c < desired && incl >= desired) {
|
||||
sh_bucket = RADIX_SIZE - 1 - tid;
|
||||
sh_above = incl - c;
|
||||
sh_tied = c;
|
||||
}
|
||||
}
|
||||
barrier();
|
||||
#else
|
||||
uint c = 0;
|
||||
if (tid < RADIX_SIZE) {
|
||||
c = histo[RADIX_SIZE - 1 - tid];
|
||||
tie_scan[tid] = c;
|
||||
}
|
||||
barrier();
|
||||
uint src = 0;
|
||||
[[unroll]] for (uint off = 1; off < RADIX_SIZE; off <<= 1) {
|
||||
if (tid < RADIX_SIZE) {
|
||||
const uint v = tie_scan[src + tid];
|
||||
tie_scan[(RADIX_SIZE - src) + tid] = tid >= off ? v + tie_scan[src + tid - off] : v;
|
||||
}
|
||||
src = RADIX_SIZE - src;
|
||||
barrier();
|
||||
}
|
||||
if (tid < RADIX_SIZE) {
|
||||
const uint incl = tie_scan[src + tid];
|
||||
if (incl - c < desired && incl >= desired) {
|
||||
sh_bucket = RADIX_SIZE - 1 - tid;
|
||||
sh_above = incl - c;
|
||||
sh_tied = c;
|
||||
}
|
||||
}
|
||||
barrier();
|
||||
#endif
|
||||
|
||||
prefix |= sh_bucket << uint(shift);
|
||||
desired -= sh_above;
|
||||
barrier();
|
||||
}
|
||||
|
||||
const uint threshold = prefix;
|
||||
const bool all_ties = sh_tied == desired;
|
||||
|
||||
if (tid == 0) {
|
||||
out_count = 0;
|
||||
}
|
||||
barrier();
|
||||
|
||||
// emit everything above the threshold, then fill the rest from ties
|
||||
const uint threshold = prefix;
|
||||
if (all_ties) {
|
||||
for (uint i = tid; i < ncols; i += BLOCK_SIZE) {
|
||||
if (f2ui(load(row, i, false)) >= threshold) {
|
||||
data_d[row_out + atomicAdd(out_count, 1u)] = int(i);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// more ties than slots: rank them in invocation order, which does not depend on the scheduling
|
||||
uint ct = 0;
|
||||
for (uint i = tid; i < ncols; i += BLOCK_SIZE) {
|
||||
if (f2ui(load(row, i, false)) > threshold) {
|
||||
const uint key = f2ui(load(row, i, false));
|
||||
if (key > threshold) {
|
||||
data_d[row_out + atomicAdd(out_count, 1u)] = int(i);
|
||||
}
|
||||
ct += key == threshold ? 1u : 0u;
|
||||
}
|
||||
#if USE_SUBGROUPS
|
||||
const uint sg = gl_SubgroupID;
|
||||
const uint st = subgroupAdd(ct);
|
||||
if (subgroupElect()) {
|
||||
sg_tied[sg] = st;
|
||||
}
|
||||
barrier();
|
||||
|
||||
if (sg == 0) {
|
||||
const uint lane = gl_SubgroupInvocationID;
|
||||
uint acc = 0;
|
||||
for (uint c0 = 0; c0 < NUM_SUBGROUPS; c0 += SUBGROUP_SIZE) {
|
||||
const uint c = c0 + lane;
|
||||
const uint t = c < NUM_SUBGROUPS ? sg_tied[c] : 0u;
|
||||
const uint e = subgroupExclusiveAdd(t);
|
||||
if (c < NUM_SUBGROUPS) {
|
||||
sg_tied[c] = acc + e;
|
||||
}
|
||||
acc += subgroupAdd(t);
|
||||
}
|
||||
}
|
||||
barrier();
|
||||
for (uint i = tid; i < ncols; i += BLOCK_SIZE) {
|
||||
|
||||
uint pos = p.k - desired + sg_tied[sg] + subgroupExclusiveAdd(ct);
|
||||
#else
|
||||
// inclusive Hillis-Steele scan, ping-ponging between the two halves
|
||||
uint src = 0;
|
||||
tie_scan[tid] = ct;
|
||||
barrier();
|
||||
for (uint off = 1; off < BLOCK_SIZE; off <<= 1) {
|
||||
const uint v = tie_scan[src + tid];
|
||||
tie_scan[(BLOCK_SIZE - src) + tid] = tid >= off ? v + tie_scan[src + tid - off] : v;
|
||||
src = BLOCK_SIZE - src;
|
||||
barrier();
|
||||
}
|
||||
uint pos = p.k - desired + tie_scan[src + tid] - ct;
|
||||
#endif
|
||||
const uint pos_end = min(p.k, pos + ct);
|
||||
for (uint i = tid; i < ncols && pos < pos_end; i += BLOCK_SIZE) {
|
||||
if (f2ui(load(row, i, false)) == threshold) {
|
||||
const uint pos = atomicAdd(out_count, 1u);
|
||||
if (pos < p.k) {
|
||||
data_d[row_out + pos] = int(i);
|
||||
}
|
||||
data_d[row_out + pos++] = int(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1115,6 +1115,7 @@ void process_shaders() {
|
||||
string_to_spv("topk_argsort_f32", "topk_argsort.comp", {{"A_TYPE", "float"}});
|
||||
string_to_spv("topk_nary_search_f32", "topk_nary_search.comp", {{"A_TYPE", "float"}});
|
||||
string_to_spv("topk_radix_select_f32", "topk_radix_select.comp", {{"A_TYPE", "float"}});
|
||||
string_to_spv("topk_radix_select_f32_subgroup", "topk_radix_select.comp", {{"A_TYPE", "float"}, {"USE_SUBGROUPS", "1"}});
|
||||
|
||||
string_to_spv("argmax_f32", "argmax.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "int"}}));
|
||||
string_to_spv("sum_rows_f32", "sum_rows.comp", merge_maps(base_dict, {{"A_TYPE", "float"}, {"D_TYPE", "float"}}));
|
||||
|
||||
@@ -962,13 +962,16 @@ inline uint32_t ggml_webgpu_flash_attn_get_vec_kv_tile(size_t wg_mem_limit_byt
|
||||
uint32_t head_dim_qk,
|
||||
uint32_t head_dim_v,
|
||||
bool has_mask,
|
||||
bool kv_direct) {
|
||||
const uint32_t max_kv_tile =
|
||||
ggml_webgpu_flash_attn_max_kv_tile(wg_mem_limit_bytes, 1u, 1u, head_dim_qk, head_dim_v, has_mask, kv_direct);
|
||||
bool k_direct,
|
||||
bool v_direct) {
|
||||
// kv_shmem is dropped only when both K and V are read directly
|
||||
const uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(wg_mem_limit_bytes, 1u, 1u, head_dim_qk, head_dim_v,
|
||||
has_mask, k_direct && v_direct);
|
||||
GGML_ASSERT(max_kv_tile > 0);
|
||||
|
||||
uint32_t kv_tile = std::min(GGML_WEBGPU_FLASH_ATTN_VEC_MAX_KV_TILE, max_kv_tile);
|
||||
if (kv_direct) {
|
||||
// a direct read has no bounds check, so the tile must divide the KV padding
|
||||
if (k_direct || v_direct) {
|
||||
kv_tile = std::min(kv_tile, GGML_WEBGPU_KV_SEQ_PAD);
|
||||
while (GGML_WEBGPU_KV_SEQ_PAD % kv_tile != 0) {
|
||||
kv_tile -= 1u;
|
||||
@@ -3050,7 +3053,7 @@ class ggml_webgpu_shader_lib {
|
||||
uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
|
||||
context.wg_mem_limit_bytes, decisions.q_tile, decisions.use_sg_matrix ? context.sg_mat_n : 1u,
|
||||
key.common.head_dim_qk, key.common.head_dim_v, key.common.has_mask,
|
||||
key.common.k_direct || key.common.v_direct);
|
||||
key.common.k_direct && key.common.v_direct);
|
||||
|
||||
// WorkGroup storage size isn't enough for some params with subgroup matrices path (ref. https://github.com/ggml-org/llama.cpp/pull/26566)
|
||||
if (max_kv_tile == 0) {
|
||||
@@ -3063,7 +3066,7 @@ class ggml_webgpu_shader_lib {
|
||||
key.use_sg_matrix = false;
|
||||
max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
|
||||
context.wg_mem_limit_bytes, decisions.q_tile, 1u, key.common.head_dim_qk, key.common.head_dim_v,
|
||||
key.common.has_mask, key.common.k_direct || key.common.v_direct);
|
||||
key.common.has_mask, key.common.k_direct && key.common.v_direct);
|
||||
GGML_ASSERT(max_kv_tile > 0);
|
||||
}
|
||||
|
||||
@@ -3125,7 +3128,7 @@ class ggml_webgpu_shader_lib {
|
||||
ggml_webgpu_flash_attn_vec_decisions decisions = {};
|
||||
decisions.kv_tile = ggml_webgpu_flash_attn_get_vec_kv_tile(context.wg_mem_limit_bytes, key.common.head_dim_qk,
|
||||
key.common.head_dim_v, key.common.has_mask,
|
||||
key.common.k_direct || key.common.v_direct);
|
||||
key.common.k_direct, key.common.v_direct);
|
||||
decisions.wg_size = context.max_subgroup_size;
|
||||
|
||||
std::string variant = "flash_attn_vec";
|
||||
|
||||
@@ -90,7 +90,7 @@ static inline uint32_t ggml_webgpu_u32_from_f32(float value) {
|
||||
#define WEBGPU_NUM_PARAM_SLOT_SAFETY_MARGIN 10u
|
||||
#define WEBGPU_RUNTIME_WAIT_TIMEOUT_MS 30000u
|
||||
#define WEBGPU_RUNTIME_WAIT_TIMEOUT_NS (WEBGPU_RUNTIME_WAIT_TIMEOUT_MS * 1e6)
|
||||
#define WEBGPU_PARAMS_BUF_SIZE_BYTES 128 // enough for 32 parameters
|
||||
#define WEBGPU_PARAMS_BUF_SIZE_BYTES 256 // enough for 64 parameters
|
||||
#define WEBGPU_SET_ROWS_ERROR_BUF_SIZE_BYTES 4
|
||||
#define WEBGPU_STORAGE_BUF_BINDING_MULT 4 // a storage buffer binding size must be a multiple of 4
|
||||
|
||||
@@ -2104,8 +2104,10 @@ static ggml_webgpu_flash_attn_op ggml_webgpu_flash_attn_prepare(webgpu_context &
|
||||
(uint32_t) (V->nb[2] / ggml_type_size(V->type)), // stride (elements/blocks) of V in dimension 2
|
||||
(uint32_t) (V->nb[3] / ggml_type_size(V->type)), // stride (elements/blocks) of V in dimension 3
|
||||
op.has_mask ? (uint32_t) (mask->nb[3] / ggml_type_size(mask->type)) : 0, // stride of mask dim 3
|
||||
(uint32_t) (Q->ne[2] / K->ne[2]), // repeat factor for K/V in dim 2 (MHA/MQA/GQA)
|
||||
ggml_webgpu_u32_from_f32(scale), // scale (possibly adjusted for logit softcap)
|
||||
op.has_mask ? (uint32_t) (mask->nb[2] / ggml_type_size(mask->type)) : 0, // stride of mask dim 2
|
||||
op.has_mask ? (uint32_t) mask->ne[2] : 1u, // mask heads, broadcast over the Q heads
|
||||
(uint32_t) (Q->ne[2] / K->ne[2]), // repeat factor for K/V in dim 2 (MHA/MQA/GQA)
|
||||
ggml_webgpu_u32_from_f32(scale), // scale (possibly adjusted for logit softcap)
|
||||
ggml_webgpu_u32_from_f32(max_bias),
|
||||
ggml_webgpu_u32_from_f32(logit_softcap),
|
||||
ggml_webgpu_u32_from_f32(n_head_log2),
|
||||
@@ -2211,7 +2213,9 @@ static webgpu_encoded_op ggml_webgpu_flash_attn_vec(webgpu_context & ct
|
||||
blk_nblk1 = (uint32_t) Q->ne[1];
|
||||
blk_buf = ggml_webgpu_tensor_buf(dst);
|
||||
const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
|
||||
blk_batch_count = stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u;
|
||||
const uint32_t stride_mask2 = (uint32_t) (mask->nb[2] / ggml_type_size(mask->type));
|
||||
const uint32_t mask_ne2 = (uint32_t) mask->ne[2];
|
||||
blk_batch_count = (stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u) * mask_ne2;
|
||||
const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
|
||||
blk_size_bytes = ROUNDUP_POW2(blk_elems * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT);
|
||||
const ggml_webgpu_shader_lib_context blk_shader_ctx = op.shader_lib_ctx;
|
||||
@@ -2222,6 +2226,8 @@ static webgpu_encoded_op ggml_webgpu_flash_attn_vec(webgpu_context & ct
|
||||
(uint32_t) Q->ne[1], // seq_len_q
|
||||
(uint32_t) K->ne[1], // seq_len_kv
|
||||
stride_mask3, // stride_mask3
|
||||
stride_mask2, // stride_mask2
|
||||
mask_ne2, // mask_ne2
|
||||
blk_nblk0, // nblk0
|
||||
blk_nblk1, // nblk1
|
||||
};
|
||||
@@ -4381,12 +4387,11 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer
|
||||
// longest vec batch and the size stays monotonic in the batch length
|
||||
const int64_t n_q = std::min<int64_t>(Q->ne[1], GGML_WEBGPU_FLASH_ATTN_VEC_MAX_SEQ_LEN - 1);
|
||||
if (ggml_webgpu_flash_attn_use_vec_path(ctx->webgpu_global_ctx, Q, K, V, n_q)) {
|
||||
const bool kv_direct =
|
||||
ggml_webgpu_flash_attn_k_direct(Q, K, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH) ||
|
||||
ggml_webgpu_flash_attn_v_direct(Q, V, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH);
|
||||
const uint32_t kv_tile = ggml_webgpu_flash_attn_get_vec_kv_tile(
|
||||
capabilities.limits.maxComputeWorkgroupStorageSize, (uint32_t) Q->ne[0], (uint32_t) V->ne[0],
|
||||
mask != nullptr, kv_direct);
|
||||
mask != nullptr,
|
||||
ggml_webgpu_flash_attn_k_direct(Q, K, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH),
|
||||
ggml_webgpu_flash_attn_v_direct(Q, V, GGML_WEBGPU_FLASH_ATTN_TILE_KV_VEC_WIDTH));
|
||||
|
||||
const uint32_t vec_nwg_cap = capabilities.min_subgroup_size;
|
||||
uint32_t nwg = ggml_webgpu_flash_attn_vec_nwg(vec_nwg_cap, kv_tile, (uint32_t) K->ne[1]);
|
||||
@@ -4403,11 +4408,12 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer
|
||||
res += WEBGPU_STORAGE_BUF_BINDING_MULT + align;
|
||||
}
|
||||
if (mask != nullptr) {
|
||||
const uint32_t blk_nblk0 = CEIL_DIV((uint32_t) K->ne[1], kv_tile);
|
||||
const uint32_t blk_nblk1 = (uint32_t) n_q;
|
||||
const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
|
||||
const uint32_t blk_batch_count = stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u;
|
||||
const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
|
||||
const uint32_t blk_nblk0 = CEIL_DIV((uint32_t) K->ne[1], kv_tile);
|
||||
const uint32_t blk_nblk1 = (uint32_t) n_q;
|
||||
const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
|
||||
const uint32_t blk_batch_count =
|
||||
(stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u) * (uint32_t) mask->ne[2];
|
||||
const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
|
||||
const size_t blk_size_bytes =
|
||||
ROUNDUP_POW2(blk_elems * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT);
|
||||
res += blk_size_bytes + align;
|
||||
@@ -5014,14 +5020,22 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
|
||||
const uint32_t q_tile =
|
||||
use_subgroup_matrix ? capabilities.sg_mat_m : GGML_WEBGPU_FLASH_ATTN_TILE_Q_TILE;
|
||||
const uint32_t kv_granularity = use_subgroup_matrix ? capabilities.sg_mat_n : 1u;
|
||||
const bool kv_direct = use_subgroup_matrix ?
|
||||
ggml_webgpu_flash_attn_k_direct(src0, src1, capabilities.sg_mat_k) ||
|
||||
ggml_webgpu_flash_attn_v_direct(src0, src2, capabilities.sg_mat_k) :
|
||||
false;
|
||||
const uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
|
||||
// same direct read rule as get_flash_attn_pipeline: F16 only, on the subgroup matrix path
|
||||
const bool kv_direct = use_subgroup_matrix && src1->type == GGML_TYPE_F16 &&
|
||||
src2->type == GGML_TYPE_F16 &&
|
||||
ggml_webgpu_flash_attn_k_direct(src0, src1, capabilities.sg_mat_k) &&
|
||||
ggml_webgpu_flash_attn_v_direct(src0, src2, capabilities.sg_mat_k);
|
||||
const uint32_t max_kv_tile = ggml_webgpu_flash_attn_max_kv_tile(
|
||||
capabilities.limits.maxComputeWorkgroupStorageSize, q_tile, kv_granularity, (uint32_t) src0->ne[0],
|
||||
(uint32_t) src2->ne[0], op->src[3] != nullptr, kv_direct);
|
||||
supports_op = max_kv_tile > 0;
|
||||
// get_flash_attn_pipeline falls back to the register tile path when the subgroup matrix tile does not fit
|
||||
const uint32_t fallback_kv_tile =
|
||||
use_subgroup_matrix ?
|
||||
ggml_webgpu_flash_attn_max_kv_tile(
|
||||
capabilities.limits.maxComputeWorkgroupStorageSize, GGML_WEBGPU_FLASH_ATTN_TILE_Q_TILE, 1u,
|
||||
(uint32_t) src0->ne[0], (uint32_t) src2->ne[0], op->src[3] != nullptr, false) :
|
||||
0u;
|
||||
supports_op = max_kv_tile > 0 || fallback_kv_tile > 0;
|
||||
break;
|
||||
}
|
||||
case GGML_OP_GROUP_NORM:
|
||||
|
||||
@@ -116,7 +116,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
|
||||
#ifdef MASK
|
||||
// mask offset
|
||||
let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + q_row_start * params.seq_len_kv;
|
||||
let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
|
||||
#endif
|
||||
|
||||
// note that the output is permuted, the layout is [head_dim_v, n_heads, seq_len_q, batch_size]
|
||||
|
||||
@@ -65,6 +65,8 @@ struct Params {
|
||||
stride_v2: u32,
|
||||
stride_v3: u32,
|
||||
stride_mask3: u32,
|
||||
stride_mask2: u32,
|
||||
mask_ne2: u32,
|
||||
|
||||
// repeat factors for K/V, e.g., MHA vs. MQA vs. GQA
|
||||
q_per_kv: u32,
|
||||
|
||||
@@ -65,7 +65,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
let row_active = subgroup_id < Q_TILE && global_q_row < params.seq_len_q;
|
||||
|
||||
#ifdef MASK
|
||||
let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + q_row_start * params.seq_len_kv;
|
||||
let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
|
||||
#endif
|
||||
|
||||
let dst_global_offset = dst_batch_offset + q_row_start * dst2_stride + head_idx * HEAD_DIM_V;
|
||||
|
||||
@@ -9,6 +9,8 @@ struct Params {
|
||||
seq_len_q: u32,
|
||||
seq_len_kv: u32,
|
||||
stride_mask3: u32,
|
||||
stride_mask2: u32,
|
||||
mask_ne2: u32,
|
||||
// Number of KV blocks and Q blocks per batch.
|
||||
// nblk0 = ceil(seq_len_kv / KV_TILE), nblk1 = seq_len_q.
|
||||
nblk0: u32,
|
||||
@@ -30,11 +32,12 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
@builtin(local_invocation_id) local_id: vec3<u32>) {
|
||||
// Dispatch mapping:
|
||||
// - x indexes KV blocks
|
||||
// - y flattens (batch_idx, q_blk) as y = batch_idx * nblk1 + q_blk
|
||||
// - y flattens (batch_idx, mask_head, q_blk) as y = (batch_idx * mask_ne2 + mask_head) * nblk1 + q_blk
|
||||
let kv_blk = wg_id.x;
|
||||
let y = wg_id.y;
|
||||
let q_blk = y % params.nblk1;
|
||||
let batch_idx = y / params.nblk1;
|
||||
let mask_head = (y / params.nblk1) % params.mask_ne2;
|
||||
let batch_idx = y / (params.nblk1 * params.mask_ne2);
|
||||
if (kv_blk >= params.nblk0) {
|
||||
return;
|
||||
}
|
||||
@@ -43,7 +46,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
let k_start = kv_blk * KV_TILE;
|
||||
|
||||
let mask_batch = select(0u, batch_idx, params.stride_mask3 > 0u);
|
||||
let mask_batch_base = params.offset_mask + mask_batch * params.stride_mask3;
|
||||
let mask_batch_base = params.offset_mask + mask_batch * params.stride_mask3 + mask_head * params.stride_mask2;
|
||||
|
||||
// We keep min/max to classify:
|
||||
// - fully masked (max <= MASK_MIN)
|
||||
@@ -95,7 +98,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
}
|
||||
}
|
||||
|
||||
let blk_idx = (batch_idx * params.nblk1 + q_blk) * params.nblk0 + kv_blk;
|
||||
let blk_idx = ((batch_idx * params.mask_ne2 + mask_head) * params.nblk1 + q_blk) * params.nblk0 + kv_blk;
|
||||
blk[blk_idx] = state;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,7 +107,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
|
||||
#ifdef MASK
|
||||
// mask offset
|
||||
let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + q_row_start * params.seq_len_kv;
|
||||
let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
|
||||
#endif
|
||||
|
||||
let head = f32(head_idx);
|
||||
@@ -129,7 +129,8 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
let q_blk = q_row_start;
|
||||
let kv_blk = kv_tile / KV_TILE;
|
||||
let blk_batch = select(0u, batch_idx, params.stride_mask3 > 0u);
|
||||
let blk_idx = params.blk_base + (blk_batch * params.blk_nblk1 + q_blk) * params.blk_nblk0 + kv_blk;
|
||||
let blk_head = head_idx % params.mask_ne2;
|
||||
let blk_idx = params.blk_base + ((blk_batch * params.mask_ne2 + blk_head) * params.blk_nblk1 + q_blk) * params.blk_nblk0 + kv_blk;
|
||||
let blk_state_local = blk[blk_idx];
|
||||
#else
|
||||
let blk_state_local = 1u;
|
||||
|
||||
@@ -1335,6 +1335,28 @@ class MODEL_TENSOR(IntEnum):
|
||||
A_GEN_CODE_EH_PROJ = auto() # drama3: per-layer [code embd; hidden] -> hidden projection
|
||||
A_GEN_CODE_BLK_EMBD = auto() # drama3: per-layer code embedding table
|
||||
A_GEN_CODE_BLK_OUT_NORM = auto() # drama3: per-layer output norm
|
||||
A_GEN_CODE_TOK_EMBD = auto() # fish_s2: code predictor input embedding, shared by all codebooks
|
||||
# fish_s2 codec encoder (voice cloning): reference clip -> codes
|
||||
A_CODEC_ENC_ENTRY = auto() # entry conv
|
||||
A_CODEC_ENC_SNAKE = auto() # encoder block, Snake before the strided conv
|
||||
A_CODEC_ENC_CONV = auto() # encoder block, strided conv
|
||||
A_CODEC_ENC_RES_ACT1 = auto()
|
||||
A_CODEC_ENC_RES_CONV1 = auto()
|
||||
A_CODEC_ENC_RES_ACT2 = auto()
|
||||
A_CODEC_ENC_RES_CONV2 = auto()
|
||||
A_CODEC_ENC_POST_SNAKE = auto()
|
||||
A_CODEC_ENC_POST_CONV = auto()
|
||||
A_CODEC_DOWN_CONV = auto() # quantizer downsample, strided conv + ConvNeXt
|
||||
A_CODEC_DOWN_DWCONV = auto()
|
||||
A_CODEC_DOWN_NORM = auto()
|
||||
A_CODEC_DOWN_PW1 = auto()
|
||||
A_CODEC_DOWN_PW2 = auto()
|
||||
A_CODEC_DOWN_GAMMA = auto()
|
||||
A_CODEC_PRE_NORM = auto() # output norm of the quantizer pre_module
|
||||
A_CODEC_VQ_FIRST_IN = auto() # semantic VQ input projection
|
||||
A_CODEC_VQ_FIRST_CB = auto() # semantic VQ codebook, l2-normalized
|
||||
A_CODEC_VQ_REST_IN = auto() # residual VQ input projections, merged 3D
|
||||
A_CODEC_VQ_REST_CB = auto() # residual VQ codebooks, l2-normalized, merged 3D
|
||||
# qwen3tts code2wav: RVQ codes -> raw PCM
|
||||
A_GEN_WAV_QUANT_FIRST_IN = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D)
|
||||
A_GEN_WAV_QUANT_FIRST_OUT = auto() # semantic RVQ, out_proj
|
||||
@@ -2277,6 +2299,27 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
|
||||
MODEL_TENSOR.A_GEN_CODE_EH_PROJ: "a.gen.code.blk.{bid}.eh_proj",
|
||||
MODEL_TENSOR.A_GEN_CODE_BLK_EMBD: "a.gen.code.blk.{bid}.embd",
|
||||
MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM: "a.gen.code.blk.{bid}.out_norm",
|
||||
MODEL_TENSOR.A_GEN_CODE_TOK_EMBD: "a.gen.code.tok_embd",
|
||||
MODEL_TENSOR.A_CODEC_ENC_ENTRY: "a.codec.enc.entry",
|
||||
MODEL_TENSOR.A_CODEC_ENC_SNAKE: "a.codec.enc.blk.{bid}.snake",
|
||||
MODEL_TENSOR.A_CODEC_ENC_CONV: "a.codec.enc.blk.{bid}.conv",
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_ACT1: "a.codec.enc.blk.{bid}.res.{xid}.act1",
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_CONV1: "a.codec.enc.blk.{bid}.res.{xid}.conv1",
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_ACT2: "a.codec.enc.blk.{bid}.res.{xid}.act2",
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_CONV2: "a.codec.enc.blk.{bid}.res.{xid}.conv2",
|
||||
MODEL_TENSOR.A_CODEC_ENC_POST_SNAKE: "a.codec.enc.post_snake",
|
||||
MODEL_TENSOR.A_CODEC_ENC_POST_CONV: "a.codec.enc.post_conv",
|
||||
MODEL_TENSOR.A_CODEC_DOWN_CONV: "a.codec.down.blk.{bid}.conv",
|
||||
MODEL_TENSOR.A_CODEC_DOWN_DWCONV: "a.codec.down.blk.{bid}.dwconv",
|
||||
MODEL_TENSOR.A_CODEC_DOWN_NORM: "a.codec.down.blk.{bid}.norm",
|
||||
MODEL_TENSOR.A_CODEC_DOWN_PW1: "a.codec.down.blk.{bid}.pw1",
|
||||
MODEL_TENSOR.A_CODEC_DOWN_PW2: "a.codec.down.blk.{bid}.pw2",
|
||||
MODEL_TENSOR.A_CODEC_DOWN_GAMMA: "a.codec.down.blk.{bid}.gamma",
|
||||
MODEL_TENSOR.A_CODEC_PRE_NORM: "a.codec.pre_norm",
|
||||
MODEL_TENSOR.A_CODEC_VQ_FIRST_IN: "a.codec.vq.first.in_proj",
|
||||
MODEL_TENSOR.A_CODEC_VQ_FIRST_CB: "a.codec.vq.first.codebook",
|
||||
MODEL_TENSOR.A_CODEC_VQ_REST_IN: "a.codec.vq.rest.in_proj",
|
||||
MODEL_TENSOR.A_CODEC_VQ_REST_CB: "a.codec.vq.rest.codebook",
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN: "a.gen.wav.quant.first.in_proj",
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj",
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB: "a.gen.wav.quant.first.codebook",
|
||||
@@ -2711,6 +2754,27 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
|
||||
MODEL_TENSOR.A_GEN_CODE_EH_PROJ,
|
||||
MODEL_TENSOR.A_GEN_CODE_BLK_EMBD,
|
||||
MODEL_TENSOR.A_GEN_CODE_BLK_OUT_NORM,
|
||||
MODEL_TENSOR.A_GEN_CODE_TOK_EMBD,
|
||||
MODEL_TENSOR.A_CODEC_ENC_ENTRY,
|
||||
MODEL_TENSOR.A_CODEC_ENC_SNAKE,
|
||||
MODEL_TENSOR.A_CODEC_ENC_CONV,
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_ACT1,
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_CONV1,
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_ACT2,
|
||||
MODEL_TENSOR.A_CODEC_ENC_RES_CONV2,
|
||||
MODEL_TENSOR.A_CODEC_ENC_POST_SNAKE,
|
||||
MODEL_TENSOR.A_CODEC_ENC_POST_CONV,
|
||||
MODEL_TENSOR.A_CODEC_DOWN_CONV,
|
||||
MODEL_TENSOR.A_CODEC_DOWN_DWCONV,
|
||||
MODEL_TENSOR.A_CODEC_DOWN_NORM,
|
||||
MODEL_TENSOR.A_CODEC_DOWN_PW1,
|
||||
MODEL_TENSOR.A_CODEC_DOWN_PW2,
|
||||
MODEL_TENSOR.A_CODEC_DOWN_GAMMA,
|
||||
MODEL_TENSOR.A_CODEC_PRE_NORM,
|
||||
MODEL_TENSOR.A_CODEC_VQ_FIRST_IN,
|
||||
MODEL_TENSOR.A_CODEC_VQ_FIRST_CB,
|
||||
MODEL_TENSOR.A_CODEC_VQ_REST_IN,
|
||||
MODEL_TENSOR.A_CODEC_VQ_REST_CB,
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN,
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT,
|
||||
MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB,
|
||||
@@ -7223,6 +7287,9 @@ class VisionProjectorType:
|
||||
DRAMA3_GEN = "drama3_gen" # audio generation: codebook tower + Fish DAC decoder
|
||||
ZONOS3_GEN = "zonos3_gen" # audio generation: 10 codebook heads + ZAC decoder
|
||||
ZONOS3_SPKENC = "zonos3_spkenc" # audio: MECT-B2 speaker encoder
|
||||
YUE2_GEN = "yue2_gen" # audio generation: flow-matching NAR + Oobleck VAE decoder
|
||||
FISH_S2_GEN = "fish_s2_gen" # audio generation: fast AR code predictor + Fish DAC decoder
|
||||
FISH_S2_SPKENC = "fish_s2_spkenc" # audio: Fish DAC encoder + RVQ, reference clip -> prompt rows
|
||||
HUNYUANVL = "hunyuanvl"
|
||||
PARAKEET = "parakeet" # audio
|
||||
MINIMAXM3 = "minimax_m3"
|
||||
|
||||
+15
-9
@@ -271,6 +271,7 @@ extern "C" {
|
||||
// - if not: only the last token is output
|
||||
// )
|
||||
//
|
||||
// [DEPRECATED] use llama_batch_ext instead
|
||||
typedef struct llama_batch {
|
||||
int32_t n_tokens;
|
||||
|
||||
@@ -945,9 +946,10 @@ extern "C" {
|
||||
//
|
||||
// NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it
|
||||
//
|
||||
LLAMA_API struct llama_batch llama_batch_get_one(
|
||||
DEPRECATED(LLAMA_API struct llama_batch llama_batch_get_one(
|
||||
llama_token * tokens,
|
||||
int32_t n_tokens);
|
||||
int32_t n_tokens),
|
||||
"use llama_batch_ext_init and llama_batch_ext_add_token instead");
|
||||
|
||||
// Allocates a batch of tokens on the heap that can hold a maximum of n_tokens
|
||||
// Each token can be assigned up to n_seq_max sequence ids
|
||||
@@ -956,13 +958,15 @@ extern "C" {
|
||||
// Otherwise, llama_batch.token will be allocated to store n_tokens llama_token
|
||||
// The rest of the llama_batch members are allocated with size n_tokens
|
||||
// All members are left uninitialized
|
||||
LLAMA_API struct llama_batch llama_batch_init(
|
||||
DEPRECATED(LLAMA_API struct llama_batch llama_batch_init(
|
||||
int32_t n_tokens,
|
||||
int32_t embd,
|
||||
int32_t n_seq_max);
|
||||
int32_t n_seq_max),
|
||||
"use llama_batch_ext_init instead");
|
||||
|
||||
// Frees a batch of tokens allocated with llama_batch_init()
|
||||
LLAMA_API void llama_batch_free(struct llama_batch batch);
|
||||
DEPRECATED(LLAMA_API void llama_batch_free(struct llama_batch batch),
|
||||
"use llama_batch_ext_free instead");
|
||||
|
||||
// Process a batch of tokens.
|
||||
// In contrast to llama_decode() - this call does not use KV cache.
|
||||
@@ -970,9 +974,10 @@ extern "C" {
|
||||
// Can store the encoder output internally for later use by the decoder's cross-attention layers.
|
||||
// 0 - success
|
||||
// < 0 - error. the memory state is restored to the state before this call
|
||||
LLAMA_API int32_t llama_encode(
|
||||
DEPRECATED(LLAMA_API int32_t llama_encode(
|
||||
struct llama_context * ctx,
|
||||
struct llama_batch batch);
|
||||
struct llama_batch batch),
|
||||
"use llama_process with LLAMA_PROCESS_TYPE_ENCODE instead");
|
||||
|
||||
// Process a batch of tokens.
|
||||
// Requires the context to have a memory.
|
||||
@@ -986,9 +991,10 @@ extern "C" {
|
||||
// 2 - aborted (processed ubatches will remain in the context's memory)
|
||||
// -1 - invalid input batch
|
||||
// < -1 - fatal error (processed ubatches will remain in the context's memory)
|
||||
LLAMA_API int32_t llama_decode(
|
||||
DEPRECATED(LLAMA_API int32_t llama_decode(
|
||||
struct llama_context * ctx,
|
||||
struct llama_batch batch);
|
||||
struct llama_batch batch),
|
||||
"use llama_process with LLAMA_PROCESS_TYPE_DECODE instead");
|
||||
|
||||
//
|
||||
// Extended batch API
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
{%- set effort_map = {"none": 0.0, "minimal": 0.1, "low": 0.2, "medium": 0.7, "high": 0.9, "xhigh": 0.99, "max": 0.99} -%}
|
||||
{%- set role_token = {"user": "<|message_user|>", "assistant": "<|message_model|>", "system": "<|message_system|>", "tool": "<|message_tool|>"} -%}
|
||||
|
||||
{%- macro emit_thinking_effort() -%}
|
||||
{%- set eff = reasoning_effort if reasoning_effort is defined and reasoning_effort is not none else 0.9 -%}
|
||||
{%- if eff is string -%}
|
||||
{%- set key = eff | trim -%}
|
||||
{%- if key not in effort_map -%}
|
||||
{{- raise_exception("Unknown reasoning_effort: " ~ eff) -}}
|
||||
{%- endif -%}
|
||||
{%- set num = effort_map[key] -%}
|
||||
{%- else -%}
|
||||
{%- set num = eff | float -%}
|
||||
{%- endif -%}
|
||||
{%- if num < 0.0 or num > 0.99 -%}
|
||||
{{- raise_exception("reasoning_effort must be in [0.0, 0.99]") -}}
|
||||
{%- endif -%}
|
||||
{{- "<|message_system|><|content_text|>Thinking effort level: " -}}
|
||||
{%- if num == 0.0 -%}0{%- else -%}{{ num }}{%- endif -%}
|
||||
{{- "<|end_message|>" -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- if tools -%}
|
||||
{%- set tool_state = namespace(specs=[]) -%}
|
||||
{%- for tool in tools -%}
|
||||
{%- set fn = tool.function if tool.function is defined else tool -%}
|
||||
{%- set spec = {
|
||||
"description": (fn.description if fn.description is defined and fn.description else ""),
|
||||
"name": fn.name,
|
||||
"parameters": (fn.parameters if fn.parameters is defined and fn.parameters else {}),
|
||||
"type": (tool.type if tool.type is defined and tool.type else "function"),
|
||||
} -%}
|
||||
{%- set tool_state.specs = tool_state.specs + [spec] -%}
|
||||
{%- endfor -%}
|
||||
{{- "<|message_system|>tool_declare<|content_xml|>" -}}
|
||||
{{- tool_state.specs | tojson(sort_keys=true, separators=(",", ":")) -}}
|
||||
{{- "<|end_message|>" -}}
|
||||
{%- endif -%}
|
||||
|
||||
{%- set state = namespace(effort_emitted=false) -%}
|
||||
{%- for message in messages -%}
|
||||
{%- if message.role not in role_token -%}
|
||||
{{- raise_exception("Unknown message role: " ~ message.role) -}}
|
||||
{%- endif -%}
|
||||
{%- if not state.effort_emitted and message.role != "system" -%}
|
||||
{{- emit_thinking_effort() -}}
|
||||
{%- set state.effort_emitted = true -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- set header = namespace(text=role_token[message.role]) -%}
|
||||
{%- if message.role == "tool" -%}
|
||||
{%- if message.name is defined and message.name -%}
|
||||
{%- set header.text = header.text ~ message.name -%}
|
||||
{%- elif message.tool_call_id is defined and message.tool_call_id -%}
|
||||
{%- for prev in messages -%}
|
||||
{%- if prev.role == "assistant" and prev.tool_calls -%}
|
||||
{%- for tc in prev.tool_calls -%}
|
||||
{%- if tc.id is defined and tc.id == message.tool_call_id and tc.function.name is defined -%}
|
||||
{%- set header.text = header.text ~ tc.function.name -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
{%- elif message.role == "assistant" and message.name is defined and message.name -%}
|
||||
{%- set header.text = header.text ~ message.name -%}
|
||||
{%- endif -%}
|
||||
{%- set rtok = header.text -%}
|
||||
|
||||
{%- if message.role == "assistant" and message.reasoning_content is defined and message.reasoning_content -%}
|
||||
{{- rtok ~ "<|content_thinking|>" ~ message.reasoning_content ~ "<|end_message|>" -}}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message.content is string and (message.content or not (message.role == "assistant" and (message.tool_calls or message.reasoning_content))) -%}
|
||||
{{- rtok ~ "<|content_text|>" ~ message.content ~ "<|end_message|>" -}}
|
||||
{%- elif message.content -%}
|
||||
{%- for part in message.content -%}
|
||||
{%- if part is string -%}
|
||||
{{- rtok ~ "<|content_text|>" ~ part ~ "<|end_message|>" -}}
|
||||
{%- elif part.type is not defined or part.type in ("text", "input_text") -%}
|
||||
{%- set text_part = (part.text if part.text is defined and part.text is string else "") -%}
|
||||
{{- rtok ~ "<|content_text|>" ~ text_part ~ "<|end_message|>" -}}
|
||||
{%- elif part.type in ("image", "input_image", "image_url") -%}
|
||||
{%- if message.role not in ("user", "tool") -%}
|
||||
{{- raise_exception("Image content is only supported for user and tool messages") -}}
|
||||
{%- endif -%}
|
||||
{{- rtok ~ "<|content_image|><|unused_200054|><|end_message|>" -}}
|
||||
{%- elif part.type in ("audio", "input_audio", "audio_url") -%}
|
||||
{{- raise_exception("Inkling 2 does not accept audio input") -}}
|
||||
{%- else -%}
|
||||
{{- raise_exception("Unsupported content part type: " ~ part.type) -}}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message.role == "assistant" and message.tool_calls -%}
|
||||
{%- for tc in message.tool_calls -%}
|
||||
{%- set fn = tc.function -%}
|
||||
{%- if fn.name is not defined or fn.name is not string -%}
|
||||
{{- raise_exception("tool call function name must be a string") -}}
|
||||
{%- endif -%}
|
||||
{%- if message.name is defined and message.name and message.name != fn.name -%}
|
||||
{{- raise_exception("assistant name " ~ message.name ~ " does not match its tool call " ~ fn.name) -}}
|
||||
{%- endif -%}
|
||||
{%- set args = fn.arguments if fn.arguments is defined and fn.arguments else {} -%}
|
||||
{%- if args is string -%}
|
||||
{{- raise_exception("tool call arguments must be a parsed object, not a JSON string; canonicalize upstream") -}}
|
||||
{%- endif -%}
|
||||
{%- if args is not mapping -%}
|
||||
{{- raise_exception("tool call arguments must be an object") -}}
|
||||
{%- endif -%}
|
||||
{{- "<|message_model|>" ~ fn.name ~ "<|content_invoke_tool_json|>" -}}
|
||||
{{- '{"name":' ~ (fn.name | tojson(sort_keys=true, separators=(",", ":"))) ~ ',"args":' -}}
|
||||
{{- "{" -}}
|
||||
{%- for key, value in args | dictsort(case_sensitive=true) -%}
|
||||
{{- (key | tojson) ~ ":" ~ (value | tojson(separators=(",", ":"))) -}}
|
||||
{%- if not loop.last -%},{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{{- "}}<|end_message|>" -}}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message.role == "assistant" -%}
|
||||
{{- "<|content_model_end_sampling|>" -}}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
{%- if not state.effort_emitted -%}
|
||||
{{- emit_thinking_effort() -}}
|
||||
{%- endif -%}
|
||||
@@ -0,0 +1,129 @@
|
||||
{%- set effort_map = {"none": 0.0, "minimal": 0.1, "low": 0.2, "medium": 0.7, "high": 0.9, "max": 0.99} -%}
|
||||
{%- set role_token = {"user": "<|message_user|>", "assistant": "<|message_model|>", "system": "<|message_system|>", "tool": "<|message_tool|>"} -%}
|
||||
|
||||
{%- macro emit_thinking_effort() -%}
|
||||
{%- set eff = reasoning_effort if reasoning_effort is defined and reasoning_effort is not none else 0.9 -%}
|
||||
{%- if eff is string -%}
|
||||
{%- set key = eff | trim -%}
|
||||
{%- if key not in effort_map -%}
|
||||
{{- raise_exception("Unknown reasoning_effort: " ~ eff) -}}
|
||||
{%- endif -%}
|
||||
{%- set num = effort_map[key] -%}
|
||||
{%- else -%}
|
||||
{%- set num = eff | float -%}
|
||||
{%- endif -%}
|
||||
{%- if num < 0.0 or num > 0.99 -%}
|
||||
{{- raise_exception("reasoning_effort must be in [0.0, 0.99]") -}}
|
||||
{%- endif -%}
|
||||
{{- "<|message_system|><|content_text|>Thinking effort level: " -}}
|
||||
{%- if num == 0.0 -%}0{%- else -%}{{ num }}{%- endif -%}
|
||||
{{- "<|end_message|>" -}}
|
||||
{%- endmacro -%}
|
||||
|
||||
{%- if tools -%}
|
||||
{%- set tool_state = namespace(specs=[]) -%}
|
||||
{%- for tool in tools -%}
|
||||
{%- set fn = tool.function if tool.function is defined else tool -%}
|
||||
{%- set spec = {
|
||||
"description": (fn.description if fn.description is defined and fn.description else ""),
|
||||
"name": fn.name,
|
||||
"parameters": (fn.parameters if fn.parameters is defined and fn.parameters else {}),
|
||||
"type": (tool.type if tool.type is defined and tool.type else "function"),
|
||||
} -%}
|
||||
{%- set tool_state.specs = tool_state.specs + [spec] -%}
|
||||
{%- endfor -%}
|
||||
{{- "<|message_system|>tool_declare<|content_xml|>" -}}
|
||||
{{- tool_state.specs | tojson(sort_keys=true, separators=(",", ":")) -}}
|
||||
{{- "<|end_message|>" -}}
|
||||
{%- endif -%}
|
||||
|
||||
{%- set state = namespace(effort_emitted=false) -%}
|
||||
{%- for message in messages -%}
|
||||
{%- if message.role not in role_token -%}
|
||||
{{- raise_exception("Unknown message role: " ~ message.role) -}}
|
||||
{%- endif -%}
|
||||
{%- if not state.effort_emitted and message.role != "system" -%}
|
||||
{{- emit_thinking_effort() -}}
|
||||
{%- set state.effort_emitted = true -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- set rtok = role_token[message.role] -%}
|
||||
|
||||
{%- if message.role == "tool" -%}
|
||||
{%- set tool_name_state = namespace(name="") -%}
|
||||
{%- if message.name is defined and message.name -%}
|
||||
{%- set tool_name_state.name = message.name -%}
|
||||
{%- elif message.tool_call_id is defined and message.tool_call_id -%}
|
||||
{%- for prev in messages -%}
|
||||
{%- if prev.role == "assistant" and prev.tool_calls -%}
|
||||
{%- for tc in prev.tool_calls -%}
|
||||
{%- if tc.id is defined and tc.id == message.tool_call_id and tc.function.name is defined -%}
|
||||
{%- set tool_name_state.name = tc.function.name -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
{{- rtok -}}
|
||||
{%- if tool_name_state.name -%}{{- tool_name_state.name -}}{%- endif -%}
|
||||
{{- "<|content_text|>" -}}
|
||||
{%- if message.content is string -%}{{- message.content -}}{%- endif -%}
|
||||
{{- "<|end_message|>" -}}
|
||||
|
||||
{%- else -%}
|
||||
{%- if message.role == "assistant" and message.reasoning_content is defined and message.reasoning_content -%}
|
||||
{{- "<|message_model|><|content_thinking|>" ~ message.reasoning_content ~ "<|end_message|>" -}}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message.content is string -%}
|
||||
{{- rtok ~ "<|content_text|>" ~ message.content ~ "<|end_message|>" -}}
|
||||
{%- elif message.content -%}
|
||||
{%- for part in message.content -%}
|
||||
{%- if part is string -%}
|
||||
{{- rtok ~ "<|content_text|>" ~ part ~ "<|end_message|>" -}}
|
||||
{%- elif part.type is not defined or part.type in ("text", "input_text") -%}
|
||||
{%- set text_part = (part.text if part.text is defined and part.text is string else "") -%}
|
||||
{{- rtok ~ "<|content_text|>" ~ text_part ~ "<|end_message|>" -}}
|
||||
{%- elif part.type in ("image", "input_image", "image_url") -%}
|
||||
{{- rtok ~ "<|content_image|><|unused_200054|><|end_message|>" -}}
|
||||
{%- elif part.type in ("audio", "input_audio", "audio_url") -%}
|
||||
{{- rtok ~ "<|content_audio_input|><|unused_200053|><|audio_end|><|end_message|>" -}}
|
||||
{%- else -%}
|
||||
{{- raise_exception("Unsupported content part type: " ~ part.type) -}}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message.role == "assistant" and message.tool_calls -%}
|
||||
{%- for tc in message.tool_calls -%}
|
||||
{%- set fn = tc.function -%}
|
||||
{%- if fn.name is not defined or fn.name is not string -%}
|
||||
{{- raise_exception("tool call function name must be a string") -}}
|
||||
{%- endif -%}
|
||||
{%- set args = fn.arguments if fn.arguments is defined and fn.arguments else {} -%}
|
||||
{%- if args is string -%}
|
||||
{{- raise_exception("tool call arguments must be a parsed object, not a JSON string; canonicalize upstream") -}}
|
||||
{%- endif -%}
|
||||
{%- if args is not mapping -%}
|
||||
{{- raise_exception("tool call arguments must be an object") -}}
|
||||
{%- endif -%}
|
||||
{{- "<|message_model|>" ~ fn.name ~ "<|content_invoke_tool_json|>" -}}
|
||||
{{- '{"name":' ~ (fn.name | tojson(sort_keys=true, separators=(",", ":"))) ~ ',"args":' -}}
|
||||
{{- (args | tojson(sort_keys=true, separators=(",", ":"))) -}}
|
||||
{{- "}<|end_message|>" -}}
|
||||
{%- endfor -%}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if message.role == "assistant" -%}
|
||||
{{- "<|content_model_end_sampling|>" -}}
|
||||
{%- endif -%}
|
||||
{%- endif -%}
|
||||
{%- endfor -%}
|
||||
|
||||
{%- if not state.effort_emitted -%}
|
||||
{{- emit_thinking_effort() -}}
|
||||
{%- endif -%}
|
||||
|
||||
{%- if add_generation_prompt -%}
|
||||
{{- "<|message_model|>" -}}
|
||||
{%- endif -%}
|
||||
@@ -33,7 +33,7 @@ Write a summary of the change log in a few sections:
|
||||
|
||||
### UI changes (if applicable)
|
||||
|
||||
[summarize the changes to `/tools/ui/`]
|
||||
[summarize the changes to `/ui/apps/chat/`]
|
||||
|
||||
### ggml changes (if applicable)
|
||||
|
||||
|
||||
@@ -144,7 +144,7 @@ endfunction()
|
||||
# Generate ui.cpp/ui.h embedding every file of ${dist_dir} (empty table when
|
||||
# it has no index.html), gzip-compressed when LLAMA_UI_GZIP is enabled.
|
||||
function(emit_files dist_dir)
|
||||
set(UI_TEMPLATE_DIR "${LLAMA_SOURCE_DIR}/tools/ui")
|
||||
set(UI_TEMPLATE_DIR "${LLAMA_SOURCE_DIR}/ui/apps/chat")
|
||||
|
||||
# Collect the asset list once and reuse it for the fingerprint,
|
||||
# validation, compression and embedding.
|
||||
@@ -518,7 +518,7 @@ function(hf_download version out_var out_resolved)
|
||||
endfunction()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. Priority 1: pre-built assets supplied in tools/ui/dist
|
||||
# 1. Priority 1: pre-built assets supplied in ui/apps/chat/dist
|
||||
# ---------------------------------------------------------------------------
|
||||
if(EXISTS "${SRC_DIST_DIR}/index.html")
|
||||
message(STATUS "UI: using pre-built assets from ${SRC_DIST_DIR}")
|
||||
@@ -585,7 +585,7 @@ if(NOT provisioned)
|
||||
"In a disconnected environment, download the pre-built UI "
|
||||
"from a llama.cpp release at "
|
||||
"https://github.com/ggml-org/llama.cpp/releases and "
|
||||
"extract to tools/ui/dist.")
|
||||
"extract to ui/apps/chat/dist.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
@@ -22,11 +22,24 @@ Identify what actually changed and which area checklists below apply. Run `git d
|
||||
- `ggml/` (any backend, op, or `ggml.h`) -> **ggml / backend**
|
||||
- `include/llama.h` and other public headers -> **Public API**
|
||||
- `tools/server/` -> **Server**
|
||||
- `tools/ui/` (or the legacy `tools/server/webui/`) -> **WebUI**
|
||||
- `ui/apps/chat/` (or the legacy `tools/ui/`, `tools/server/webui/`) -> **Llama Chat**
|
||||
- `tests/`, `tools/server/tests/`, or any other added/changed test -> **Tests**
|
||||
- anything else, plus all of the above -> **General** (always runs)
|
||||
|
||||
Always run the **Scope and quick-reject gate**, the **Security review**, and the **General** checklist. Run each area checklist whose paths were touched. Additionally, if the diff introduces a new component, subsystem, or piece of infrastructure (a new file/class/module, a new abstraction, or hand-rolled machinery), run the **Approach and design** review. Tell the user which checklists you're running and why.
|
||||
Some areas have a developer README that documents internals and conventions the review must enforce. If the diff touches one of these areas, read the matching file into context before reviewing (required, not optional):
|
||||
|
||||
- `tools/server/` -> [tools/server/README-dev.md](../../tools/server/README-dev.md)
|
||||
- `tools/mtmd/` -> [tools/mtmd/README-dev.md](../../tools/mtmd/README-dev.md)
|
||||
|
||||
Always run the **Prior review context** step, the **Scope and quick-reject gate**, the **Security review**, and the **General** checklist. Run each area checklist whose paths were touched. Additionally, if the diff introduces a new component, subsystem, or piece of infrastructure (a new file/class/module, a new abstraction, or hand-rolled machinery), run the **Approach and design** review. Tell the user which checklists you're running and why.
|
||||
|
||||
## Prior review context (always)
|
||||
|
||||
Maintainers tend to repeat the same comments on the same parts of the code. Before reviewing, find out what they said last time so this diff doesn't repeat a mistake that was already flagged.
|
||||
|
||||
- For each touched file (or the hot spots in a large diff), find recent PRs that changed the same code: `git log --oneline -n 20 -- <file>` (merged PR numbers appear as `(#NNNNN)` in the subject), and `git blame -L <start>,<end> master -- <file>` for the exact lines being changed. For in-flight work, `gh pr list --state all --search "<file or symbol>"`.
|
||||
- Read the reviewer feedback on the most relevant few: `gh pr view <n> --comments` for the conversation and `gh api repos/ggml-org/llama.cpp/pulls/<n>/comments` for inline review comments.
|
||||
- Note recurring reviewer requests and frequently-occurring mistakes for this area (e.g. a helper that should be reused, a naming or layering rule, a backend that keeps getting broken, a test that is always asked for), then check the current diff against each one. Report a match as a finding and cite the earlier PR so the user can read the original context.
|
||||
|
||||
## Scope and quick-reject gate (always)
|
||||
|
||||
@@ -119,9 +132,9 @@ Public API changes carry a higher bar than internal ones (`CONTRIBUTING.md`). Re
|
||||
- Security: don't trust client-supplied headers (e.g. `X-Forwarded-For`) or add footguns; things like IP allowlisting belong at a reverse proxy unless there's a trusted-proxy design.
|
||||
- Wire new behavior into the existing request/response and checkpoint paths correctly; watch for resource leaks across requests.
|
||||
|
||||
## WebUI (`tools/ui/`)
|
||||
## Llama Chat (`ui/apps/chat/`)
|
||||
|
||||
These apply only to changes under `tools/ui/` (formerly `tools/server/webui/`) - the SvelteKit + Svelte 5 frontend. They are frontend conventions, not repo-wide ones, so do not apply them to C++ or other tool code.
|
||||
These apply only to changes under `ui/apps/chat/` (formerly `tools/ui/`, `tools/server/webui/`) - the SvelteKit + Svelte 5 frontend. They are frontend conventions, not repo-wide ones, so do not apply them to C++ or other tool code.
|
||||
|
||||
The dominant review theme in this area is placement: most accepted nits are about putting a value, function, or piece of logic in the layer where it belongs.
|
||||
|
||||
@@ -192,7 +205,7 @@ Adjacent data-shape rule: empty string is not a valid "unset" initial state - us
|
||||
- Sanitize anything rendered from model or remote data: the rehype/`{@html}` pipeline, SVG content, and URLs (use the `$lib/utils` URL sanitizer rather than ad-hoc checks or a third-party lib for trivial cases).
|
||||
- Proxy headers and agent/tool filesystem access are security boundaries - scope enforcement (e.g. browse-root sandboxing) must be validated end-to-end, not just in the UI; loop in server reviewers for the C++ side.
|
||||
|
||||
### PR hygiene specific to `tools/ui`
|
||||
### PR hygiene specific to `ui/apps/chat`
|
||||
|
||||
- Build artifacts are not committed - the UI is built by CI (`.github/workflows/ui-build.yml`) and downloaded/bundled at release time. `npm run build` must still succeed locally; a red UI build job is blocking.
|
||||
- `npm run lint && npm run check` (and `npm run format`) must pass; a failing Storybook test is blocking.
|
||||
|
||||
+13
-16
@@ -4254,7 +4254,7 @@ void llama_context::opt_epoch_iter(
|
||||
ggml_opt_result_t result,
|
||||
const std::vector<llama_token> & tokens,
|
||||
const std::vector<llama_token> & labels_sparse,
|
||||
llama_batch & batch,
|
||||
llama_batch_ext & batch,
|
||||
ggml_opt_epoch_callback callback,
|
||||
bool train,
|
||||
int64_t idata_in_loop,
|
||||
@@ -4268,22 +4268,21 @@ void llama_context::opt_epoch_iter(
|
||||
memory->clear(true);
|
||||
|
||||
for (uint32_t pos_ctx = 0; pos_ctx < n_ctx; pos_ctx += n_batch) {
|
||||
batch.n_tokens = n_batch;
|
||||
batch.clear();
|
||||
for (uint32_t pos_batch = 0; pos_batch < n_batch; ++pos_batch) {
|
||||
batch.token [pos_batch] = tokens[pos_ctx + pos_batch];
|
||||
batch.pos [pos_batch] = pos_ctx + pos_batch;
|
||||
batch.n_seq_id[pos_batch] = 1;
|
||||
batch.seq_id [pos_batch][0] = 0;
|
||||
batch.logits [pos_batch] = true;
|
||||
}
|
||||
|
||||
// TODO: use llama_batch_ext here
|
||||
{
|
||||
llama_batch_compat compat(this, batch);
|
||||
if (!balloc->init(*compat.batch_ext, model.vocab, true)) {
|
||||
const int32_t idx = batch.add_token(0);
|
||||
const llama_pos pos = pos_ctx + pos_batch;
|
||||
if (idx < 0 || !batch.set_token_id(idx, tokens[pos_ctx + pos_batch])) {
|
||||
LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__);
|
||||
return;
|
||||
}
|
||||
batch.set_token_pos(idx, &pos);
|
||||
batch.set_output(idx, true);
|
||||
}
|
||||
|
||||
if (!balloc->init(batch, model.vocab, true)) {
|
||||
LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__);
|
||||
return;
|
||||
}
|
||||
|
||||
const uint32_t n_tokens_all = balloc->get_n_tokens();
|
||||
@@ -4382,7 +4381,7 @@ void llama_context::opt_epoch(
|
||||
|
||||
const uint32_t ubatch_per_ctx = n_ctx / n_ubatch;
|
||||
|
||||
struct llama_batch batch = llama_batch_init(n_batch, 0, 1);
|
||||
llama_batch_ext batch(this);
|
||||
std::vector<llama_token> tokens(n_ctx);
|
||||
std::vector<llama_token> labels_sparse(n_ctx);
|
||||
|
||||
@@ -4409,8 +4408,6 @@ void llama_context::opt_epoch(
|
||||
opt_epoch_iter(dataset, result_eval, tokens, labels_sparse, batch,
|
||||
callback_eval, train, idata_in_loop, ndata_in_loop, t_loop_start);
|
||||
}
|
||||
|
||||
llama_batch_free(batch);
|
||||
}
|
||||
|
||||
//
|
||||
|
||||
+1
-1
@@ -233,7 +233,7 @@ struct llama_context {
|
||||
ggml_opt_result_t result,
|
||||
const std::vector<llama_token> & tokens,
|
||||
const std::vector<llama_token> & labels_sparse,
|
||||
llama_batch & batch,
|
||||
llama_batch_ext & batch,
|
||||
ggml_opt_epoch_callback callback,
|
||||
bool train,
|
||||
int64_t idata_in_loop,
|
||||
|
||||
@@ -403,6 +403,8 @@ size_t llama_grammar_trigger_pattern::find(const std::string & input) const {
|
||||
if (std::regex_match(input, match, regex)) {
|
||||
return find_start_pos(match);
|
||||
}
|
||||
// some regex implementations match ^/$ at line breaks
|
||||
return std::string::npos;
|
||||
}
|
||||
|
||||
// search anywhere
|
||||
|
||||
+1
-13
@@ -458,19 +458,7 @@ bool llama_kv_cache::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
|
||||
auto & cells = v_cells[seq_to_stream[seq_id]];
|
||||
auto & head = v_heads[seq_to_stream[seq_id]];
|
||||
|
||||
uint32_t new_head = cells.size();
|
||||
|
||||
for (uint32_t i = 0; i < cells.size(); ++i) {
|
||||
if (!cells.pos_in(i, p0, p1)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (cells.seq_has(i, seq_id) && cells.seq_rm(i, seq_id)) {
|
||||
if (new_head == cells.size()) {
|
||||
new_head = i;
|
||||
}
|
||||
}
|
||||
}
|
||||
const uint32_t new_head = cells.seq_rm(seq_id, p0, p1);
|
||||
|
||||
// If we freed up a slot, set head to it so searching can start there.
|
||||
if (new_head != cells.size() && new_head < head) {
|
||||
|
||||
@@ -308,6 +308,27 @@ public:
|
||||
return false;
|
||||
}
|
||||
|
||||
// remove seq_id from its cells at positions in [p0, p1)
|
||||
// return the lowest index of a cell that becomes empty, or size() if none does
|
||||
uint32_t seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) {
|
||||
assert(seq_id >= 0);
|
||||
assert(seq_id < LLAMA_MAX_SEQ);
|
||||
|
||||
uint32_t res = size();
|
||||
|
||||
const auto & sp = seq_pos[seq_id];
|
||||
for (auto it = sp.lower_bound({ p0, 0 }); it != sp.end() && it->first < p1; ) {
|
||||
const uint32_t i = it->second;
|
||||
++it; // advance first: seq_rm(i, seq_id) erases this entry from sp
|
||||
|
||||
if (seq_rm(i, seq_id)) {
|
||||
res = std::min(res, i);
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
// return true if the cell becomes empty (i.e. it did not contain seq_id before the call)
|
||||
bool seq_keep(uint32_t i, llama_seq_id seq_id) {
|
||||
assert(i < pos.size());
|
||||
|
||||
@@ -557,7 +557,8 @@ struct case_state_rotation : ctx_test {
|
||||
}
|
||||
|
||||
llama_token token = 0;
|
||||
if (llama_decode(src.get(), llama_batch_get_one(&token, 1))) {
|
||||
common_batch batch = common_batch_get_one(src.get(), &token, 1);
|
||||
if (llama_process(src.get(), LLAMA_PROCESS_TYPE_DECODE, batch.get())) {
|
||||
TLOG_ERR("%s: failed to decode a token\n", __func__);
|
||||
success = false;
|
||||
break;
|
||||
|
||||
@@ -89,6 +89,9 @@ std::vector<case_run> run_cases(model_run & mr, const std::vector<const ctx_test
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (const std::exception & e) {
|
||||
run.status = test_status::FAIL;
|
||||
run.reason = std::string("exception: ") + e.what();
|
||||
} catch (...) {
|
||||
run.status = test_status::FAIL;
|
||||
run.reason = "unhandled exception";
|
||||
|
||||
@@ -333,6 +333,33 @@ static void test(void) {
|
||||
assert(params.lora_adapters[2].path == "file3\"3\".gguf");
|
||||
assert(params.lora_adapters[3].path == "file4\".gguf");
|
||||
|
||||
{
|
||||
std::string ts_exact;
|
||||
for (size_t i = 0; i < llama_max_devices(); ++i) {
|
||||
ts_exact += (i > 0 ? "," : "") + std::to_string(i + 1);
|
||||
}
|
||||
common_params p_ts;
|
||||
argv = {"binary_name", "-m", "model.gguf", "-ts", ts_exact};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), p_ts, LLAMA_EXAMPLE_COMMON));
|
||||
for (size_t i = 0; i < llama_max_devices(); ++i) {
|
||||
assert(p_ts.tensor_split[i] == static_cast<float>(i + 1));
|
||||
}
|
||||
|
||||
common_params p_fitt;
|
||||
argv = {"binary_name", "-m", "model.gguf", "-fitt", ts_exact};
|
||||
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), p_fitt, LLAMA_EXAMPLE_COMMON));
|
||||
for (size_t i = 0; i < llama_max_devices(); ++i) {
|
||||
assert(p_fitt.fit_params_target[i] == static_cast<size_t>(i + 1) * 1024 * 1024);
|
||||
}
|
||||
|
||||
std::string ts_too_many = ts_exact + ",99";
|
||||
argv = {"binary_name", "-m", "model.gguf", "-ts", ts_too_many};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), p_ts, LLAMA_EXAMPLE_COMMON));
|
||||
|
||||
argv = {"binary_name", "-m", "model.gguf", "-fitt", ts_too_many};
|
||||
assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), p_fitt, LLAMA_EXAMPLE_COMMON));
|
||||
}
|
||||
|
||||
// skip this part on windows, because setenv is not supported
|
||||
#ifdef _WIN32
|
||||
printf("test-arg-parser: skip on windows build\n");
|
||||
|
||||
@@ -7496,10 +7496,10 @@ struct test_top_k : public test_case {
|
||||
|
||||
// the backends may break ties in different ways, but each backend must break them in the same way in each run
|
||||
bool deterministic(ggml_backend_t backend) override {
|
||||
// TODO: Vulkan is not deterministic yet, SYCL gathers the ties with an atomic counter as well
|
||||
// TODO: SYCL is not deterministic yet, it gathers the ties with an atomic counter
|
||||
const char * name = ggml_backend_reg_name(ggml_backend_dev_backend_reg(ggml_backend_get_device(backend)));
|
||||
// ROCm has no cub::DeviceTopK, every row goes through the radix or bitonic path
|
||||
if (strcmp(name, "ROCm") == 0) {
|
||||
// ROCm and Vulkan have no cub::DeviceTopK, every row goes through their own deterministic paths
|
||||
if (strcmp(name, "ROCm") == 0 || strcmp(name, "Vulkan") == 0) {
|
||||
return ties;
|
||||
}
|
||||
if (strcmp(name, "CUDA") != 0 && strcmp(name, "MTL") != 0) {
|
||||
@@ -7673,11 +7673,12 @@ struct test_topk_qsa : public test_case {
|
||||
}
|
||||
|
||||
std::string vars() override {
|
||||
return VARS_TO_STR5(n_blocks, n_kv, n_tps, n_stream, width);
|
||||
return VARS_TO_STR6(n_blocks, n_kv, n_tps, n_stream, width, tied);
|
||||
}
|
||||
|
||||
test_topk_qsa(int64_t n_blocks = 512, int64_t n_kv = 2048, int64_t n_tps = 2, int64_t n_stream = 1, int width = 1500)
|
||||
: n_blocks(n_blocks), n_kv(n_kv), n_tps(n_tps), n_stream(n_stream), width(width) {}
|
||||
const bool tied;
|
||||
test_topk_qsa(int64_t n_blocks = 512, int64_t n_kv = 2048, int64_t n_tps = 2, int64_t n_stream = 1, int width = 1500, bool tied = false)
|
||||
: n_blocks(n_blocks), n_kv(n_kv), n_tps(n_tps), n_stream(n_stream), width(width), tied(tied) {}
|
||||
|
||||
double max_err() override { return 0.0; }
|
||||
bool run_whole_graph() override { return true; }
|
||||
@@ -7710,13 +7711,13 @@ struct test_topk_qsa : public test_case {
|
||||
}
|
||||
if (t->type == GGML_TYPE_I32) {
|
||||
std::vector<int32_t> data(ggml_nelements(t));
|
||||
for (auto & v : data) { v = rand() % n_blocks; }
|
||||
for (size_t i = 0; i < data.size(); i++) { data[i] = tied ? (int32_t) ((i % n_kv) * n_blocks / n_kv) : rand() % n_blocks; }
|
||||
ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(int32_t));
|
||||
} else if (t->type == GGML_TYPE_F16) {
|
||||
std::vector<ggml_fp16_t> data(ggml_nelements(t));
|
||||
for (int64_t r = 0; r < ggml_nrows(t); r++) {
|
||||
for (int64_t i = 0; i < n_kv; i++) {
|
||||
data[r * n_kv + i] = ggml_fp32_to_fp16((float) i);
|
||||
data[r * n_kv + i] = ggml_fp32_to_fp16(tied ? 0.0f : (float) i);
|
||||
}
|
||||
}
|
||||
ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(ggml_fp16_t));
|
||||
@@ -11178,6 +11179,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
}
|
||||
}
|
||||
}
|
||||
test_cases.emplace_back(new test_conv_3d(2, 4097, 2, 32, 16, 128, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type));
|
||||
// Case with kernel size 1
|
||||
test_cases.emplace_back(new test_conv_3d(1, 4, 8, 8, 8, 8, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, kernel_type));
|
||||
test_cases.emplace_back(new test_conv_3d(2, 8, 5, 11, 9, 65, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, kernel_type));
|
||||
@@ -12279,6 +12281,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
for (int k : {1, 63, 65}) {
|
||||
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F16, GGML_TYPE_F32, 1, 1, false, 8, 16, k));
|
||||
}
|
||||
// odd k with few tokens, e.g. lora_b of a rank 1 LoRA on expert weights
|
||||
for (int n : {1, 2}) {
|
||||
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_F32, GGML_TYPE_F32, 4, 2, false, 32, n, 1));
|
||||
}
|
||||
test_cases.emplace_back(new test_mul_mat_id_fusion(GGML_TYPE_F16, GGML_TYPE_F32, 16, 16, false, 32, 32, 32, 3));
|
||||
for (ggml_type type : { GGML_TYPE_Q4_K, GGML_TYPE_Q8_0 }) {
|
||||
for (int64_t n : { 1, 2, 3, 4, 8, 9 }) {
|
||||
@@ -14028,8 +14034,13 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
|
||||
for (auto cols : {8192, 32768, 131072}) {
|
||||
for (auto nrows : {1, 2, 4, 8, 16, 32}) {
|
||||
test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {cols, nrows, 1, 1}, 2048));
|
||||
test_cases.emplace_back(new test_top_k(GGML_TYPE_F32, {cols, nrows, 1, 1}, 2048, true));
|
||||
}
|
||||
}
|
||||
for (int n_tps : {1, 16, 64}) {
|
||||
test_cases.emplace_back(new test_topk_qsa(516, 33024, n_tps, 1, 2051, true));
|
||||
test_cases.emplace_back(new test_topk_qsa(2064, 132096, n_tps, 1, 2051, true));
|
||||
}
|
||||
// bitonic vs radix crossover
|
||||
for (auto cols : {520, 1000, 2048, 3000, 4096}) {
|
||||
for (auto nrows : {2, 16, 32, 48, 64, 128, 256, 512}) {
|
||||
|
||||
@@ -978,7 +978,9 @@ static void test_compat(testing & t) {
|
||||
mem.ranges[0] = {0, 9};
|
||||
|
||||
llama_token token[3] = { 5, 6, 7 };
|
||||
llama_batch lb = llama_batch_get_one(token, 3);
|
||||
llama_batch lb = {};
|
||||
lb.n_tokens = 3;
|
||||
lb.token = token;
|
||||
|
||||
batch_builder bb(2, &mem, 4, 1, /*n_vocab*/ 100);
|
||||
llama_batch_compat::init(bb.b, lb);
|
||||
@@ -999,7 +1001,9 @@ static void test_compat(testing & t) {
|
||||
|
||||
t.test("auto_pos_starts_at_zero_without_memory", [&](testing & t) {
|
||||
llama_token token[2] = { 5, 6 };
|
||||
llama_batch lb = llama_batch_get_one(token, 2);
|
||||
llama_batch lb = {};
|
||||
lb.n_tokens = 2;
|
||||
lb.token = token;
|
||||
|
||||
batch_builder bb(2, nullptr, 4, 1, /*n_vocab*/ 100);
|
||||
llama_batch_compat::init(bb.b, lb);
|
||||
|
||||
@@ -5516,6 +5516,102 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
|
||||
.run();
|
||||
}
|
||||
|
||||
// Inkling 1 and 2
|
||||
for (const int version : { 1, 2 }) {
|
||||
auto tst = peg_tester(version == 1 ? "models/templates/thinkingmachines-Inkling.jinja" :
|
||||
"models/templates/thinkingmachines-Inkling-2-Small.jinja", detailed_debug);
|
||||
|
||||
// Inkling 1 has the first header in the generation prompt, Inkling 2 writes it
|
||||
const std::string H = "<|message_model|>";
|
||||
const std::string P = version == 1 ? "" : H;
|
||||
const std::string E = "<|end_message|>";
|
||||
const std::string CALL = "special_function<|content_invoke_tool_json|>{\"name\":\"special_function\",\"args\":{\"arg1\":1}}" + E;
|
||||
|
||||
// Content only
|
||||
tst.test(P + "<|content_text|>Hello, world!\nWhat's up?" + E)
|
||||
.expect(message_assist)
|
||||
.run();
|
||||
|
||||
// Thinking block, then content block
|
||||
tst.test(P + "<|content_thinking|>I'm thinking" + E + H + "<|content_text|>Hello, world!\nWhat's up?" + E)
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.expect(simple_assist_msg("Hello, world!\nWhat's up?", "I'm thinking"))
|
||||
.run();
|
||||
|
||||
// Without reasoning extraction, the thinking block stays in the content
|
||||
tst.test(P + "<|content_thinking|>I'm thinking" + E + H + "<|content_text|>Hello, world!\nWhat's up?" + E)
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_NONE)
|
||||
.expect_content("<|content_thinking|>I'm thinking" + E + "Hello, world!\nWhat's up?")
|
||||
.run();
|
||||
|
||||
// Cut by the token limit, in thinking and in content
|
||||
tst.test(P + "<|content_thinking|>I was still thinking")
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.expect_reasoning("I was still thinking")
|
||||
.run();
|
||||
tst.test(P + "<|content_thinking|>I'm thinking" + E + H + "<|content_text|>Hello, wor")
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.expect(simple_assist_msg("Hello, wor", "I'm thinking"))
|
||||
.run();
|
||||
|
||||
// Single tool call
|
||||
tst.test(P + CALL)
|
||||
.tools({ special_function_tool })
|
||||
.expect_tool_calls({
|
||||
{ "special_function", R"({"arg1":1})", "" },
|
||||
})
|
||||
.expect_rules({ "tool-0" })
|
||||
.run();
|
||||
|
||||
// Thinking, content, then a tool call
|
||||
tst.test(P + "<|content_thinking|>I should call it" + E + H + "<|content_text|>On it." + E + H + CALL)
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.tools({ special_function_tool })
|
||||
.expect(simple_assist_msg("On it.", "I should call it", "special_function", R"({"arg1":1})", ""))
|
||||
.run();
|
||||
|
||||
// Tools given, but the model answers with content
|
||||
tst.test(P + "<|content_thinking|>No tool needed" + E + H + "<|content_text|>Hello, world!\nWhat's up?" + E)
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.tools({ special_function_tool })
|
||||
.expect(simple_assist_msg("Hello, world!\nWhat's up?", "No tool needed"))
|
||||
.run();
|
||||
|
||||
// Parallel tool calls, one block each
|
||||
tst.test(P + CALL + H +
|
||||
"special_function_with_opt<|content_invoke_tool_json|>{\"name\":\"special_function_with_opt\",\"args\":{\"arg1\":1,\"arg2\":2}}" + E)
|
||||
.parallel_tool_calls(true)
|
||||
.tools({ special_function_tool, special_function_tool_with_optional_param })
|
||||
.expect_tool_calls({
|
||||
{ "special_function", R"({"arg1":1})", "" },
|
||||
{ "special_function_with_opt", R"({"arg1":1,"arg2":2})", "" },
|
||||
})
|
||||
.run();
|
||||
|
||||
// The block closer inside a string argument does not end the block
|
||||
tst.test(P + "python<|content_invoke_tool_json|>{\"name\":\"python\",\"args\":{\"code\":\"print('<|end_message|>')\"}}" + E)
|
||||
.tools({ python_tool })
|
||||
.expect_tool_calls({
|
||||
{ "python", R"JSON({"code":"print('<|end_message|>')"})JSON", "" },
|
||||
})
|
||||
.run();
|
||||
|
||||
// Forced tool call
|
||||
tst.test(P + "<|content_thinking|>I should call it" + E + H + CALL)
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.tool_choice(COMMON_CHAT_TOOL_CHOICE_REQUIRED)
|
||||
.tools({ special_function_tool })
|
||||
.expect(simple_assist_msg("", "I should call it", "special_function", R"({"arg1":1})", ""))
|
||||
.run();
|
||||
|
||||
// Response format
|
||||
tst.test(P + "<|content_thinking|>I'm thinking" + E + H + "<|content_text|>{\"amount\": 123.45}" + E)
|
||||
.reasoning_format(COMMON_REASONING_FORMAT_AUTO)
|
||||
.json_schema(invoice_schema)
|
||||
.expect(simple_assist_msg("{\"amount\": 123.45}", "I'm thinking"))
|
||||
.run();
|
||||
}
|
||||
|
||||
// Kimi-K2-Thinking tests - custom parser
|
||||
// Unique feature: tool call ID embeds function name as functions.<name>:<counter>
|
||||
{
|
||||
@@ -8614,6 +8710,17 @@ static void test_reasoning_effort_caps() {
|
||||
assert_effort_levels("models/templates/openai-gpt-oss-120b.jinja", all_levels);
|
||||
assert_effort_levels("models/templates/Cohere2MoE.jinja", {});
|
||||
assert_effort_levels("models/templates/Qwen-Qwen3-0.6B.jinja", {});
|
||||
|
||||
auto assert_effort_rejected = [](const std::string & path, const std::vector<std::string> & expected) {
|
||||
auto tmpls = read_templates(path);
|
||||
assert_equals(json(expected).dump(), common_chat_templates_get_caps(tmpls.get()).at("reasoning_effort_rejected").dump());
|
||||
};
|
||||
|
||||
assert_effort_rejected("models/templates/IFM-K2-Horizon.jinja", { "minimal", "xhigh", "max" });
|
||||
assert_effort_rejected("models/templates/tencent-Hy3.jinja", { "minimal", "medium", "xhigh", "max" });
|
||||
assert_effort_rejected("models/templates/deepseek-ai-DeepSeek-V4.jinja", {});
|
||||
assert_effort_rejected("models/templates/openai-gpt-oss-120b.jinja", {});
|
||||
assert_effort_rejected("models/templates/Cohere2MoE.jinja", {});
|
||||
}
|
||||
|
||||
static void test_msg_diffs_compute() {
|
||||
|
||||
@@ -1132,6 +1132,12 @@ static void test_filters(testing & t) {
|
||||
"{\"a\": 1, \"b\": 2}"
|
||||
);
|
||||
|
||||
test_template(t, "tojson sort_keys=true nested with separators",
|
||||
"{{ data|tojson(sort_keys=true, separators=(',',':')) }}",
|
||||
{{"data", {{"name", "f"}, {"args", {{"z", 1}, {"a", json::array({json{{"y", 2}, {"b", 3}}})}}}}}},
|
||||
"{\"args\":{\"a\":[{\"b\":3,\"y\":2}],\"z\":1},\"name\":\"f\"}"
|
||||
);
|
||||
|
||||
test_template(t, "tojson",
|
||||
"{{ data|tojson }}",
|
||||
{{"data", {{"a", 1}, {"b", json::array({1, 2})}}}},
|
||||
@@ -1469,6 +1475,15 @@ static void test_macros(testing & t) {
|
||||
"a+b"
|
||||
);
|
||||
|
||||
test_template(t, "caller saved and called after its scope ended",
|
||||
"{% set ns = namespace(f=none) %}"
|
||||
"{% macro keep() %}{% set ns.f = caller %}{% endmacro %}"
|
||||
"{% macro outer(p) %}{% call keep() %}[{{ p }}]{% endcall %}{% endmacro %}"
|
||||
"{{ outer('a') }}{{ outer('b') }}{{ ns.f() }}{{ ns.f() }}",
|
||||
json::object(),
|
||||
"[b][b]"
|
||||
);
|
||||
|
||||
test_template(t, "macro with caller",
|
||||
"\
|
||||
{%- macro nest_dict(o, i, ff='') %}\n\
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
#include <numeric>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
static void init_model(llama_model & model, uint32_t n_layer) {
|
||||
model.hparams = {};
|
||||
@@ -105,6 +106,75 @@ int main() {
|
||||
}
|
||||
}
|
||||
|
||||
// seq_rm removes [p0, p1) of one sequence, a negative bound is open
|
||||
{
|
||||
std::unique_ptr<llama_model> model(llama_model_create(LLM_ARCH_LLAMA, llama_model_default_params()));
|
||||
init_model(*model, 1);
|
||||
|
||||
llama_kv_cache kv(*model, false, model->hparams, GGML_TYPE_F16, GGML_TYPE_F16,
|
||||
false, false, true, 32, 2, 0, LLAMA_SWA_TYPE_NONE, nullptr, nullptr, nullptr, nullptr);
|
||||
|
||||
// returns the first cell that takes the tokens
|
||||
auto add = [&](llama_seq_id seq_id, llama_pos p0, uint32_t n) {
|
||||
std::vector<llama_token> token(n, 1);
|
||||
std::vector<llama_pos> pos(n);
|
||||
std::vector<int32_t> n_seq_id(n, 1);
|
||||
std::vector<llama_seq_id *> seq_ids(n, &seq_id);
|
||||
std::iota(pos.begin(), pos.end(), p0);
|
||||
|
||||
llama_ubatch ubatch = {};
|
||||
ubatch.n_tokens = n;
|
||||
ubatch.n_seq_tokens = n;
|
||||
ubatch.n_seqs = 1;
|
||||
ubatch.n_seqs_unq = 1;
|
||||
ubatch.n_pos = 1;
|
||||
ubatch.token = token.data();
|
||||
ubatch.pos = pos.data();
|
||||
ubatch.n_seq_id = n_seq_id.data();
|
||||
ubatch.seq_id = seq_ids.data();
|
||||
ubatch.seq_id_unq = &seq_id;
|
||||
|
||||
const auto sinfo = kv.find_slot(ubatch, false);
|
||||
GGML_ASSERT(!sinfo.empty());
|
||||
kv.apply_ubatch(sinfo, ubatch);
|
||||
|
||||
return sinfo.head();
|
||||
};
|
||||
|
||||
GGML_ASSERT(add(0, 0, 8) == 0);
|
||||
GGML_ASSERT(add(1, 0, 4) == 8);
|
||||
|
||||
// the tail of sequence 0, the other sequence keeps its cells
|
||||
GGML_ASSERT(kv.seq_rm(0, 5, -1));
|
||||
GGML_ASSERT(kv.seq_pos_min(0) == 0 && kv.seq_pos_max(0) == 4);
|
||||
GGML_ASSERT(kv.seq_pos_min(1) == 0 && kv.seq_pos_max(1) == 3);
|
||||
|
||||
// the next tokens go to the lowest freed cell
|
||||
GGML_ASSERT(add(0, 5, 2) == 5);
|
||||
|
||||
// p1 is not part of the range
|
||||
GGML_ASSERT(kv.seq_rm(0, -1, 2));
|
||||
GGML_ASSERT(kv.seq_pos_min(0) == 2 && kv.seq_pos_max(0) == 6);
|
||||
GGML_ASSERT(kv.seq_rm(0, 6, 6));
|
||||
GGML_ASSERT(kv.seq_pos_max(0) == 6);
|
||||
|
||||
// freed cells are taken in cell order, so a higher position can sit in a lower cell
|
||||
GGML_ASSERT(add(0, 7, 3) == 0);
|
||||
GGML_ASSERT(kv.seq_rm(0, 6, -1));
|
||||
GGML_ASSERT(add(0, 6, 1) == 0);
|
||||
|
||||
// a cell shared with another sequence stays for that sequence
|
||||
kv.seq_cp(0, 1, 5, -1);
|
||||
GGML_ASSERT(kv.seq_rm(0, 5, -1));
|
||||
GGML_ASSERT(kv.seq_pos_max(0) == 4 && kv.seq_pos_max(1) == 6);
|
||||
|
||||
// every cell of a repeated position is removed
|
||||
add(0, 5, 1);
|
||||
add(0, 5, 1);
|
||||
GGML_ASSERT(kv.seq_rm(0, 5, 6));
|
||||
GGML_ASSERT(kv.seq_pos_max(0) == 4);
|
||||
}
|
||||
|
||||
llama_backend_free();
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -22,7 +22,8 @@ else()
|
||||
add_subdirectory(perplexity)
|
||||
add_subdirectory(quantize)
|
||||
if (LLAMA_BUILD_SERVER)
|
||||
add_subdirectory(ui)
|
||||
# the chat app lives in ui/apps/chat, outside tools/
|
||||
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../ui/apps/chat ${CMAKE_BINARY_DIR}/ui/apps/chat)
|
||||
add_subdirectory(cli)
|
||||
add_subdirectory(server)
|
||||
endif()
|
||||
|
||||
@@ -84,7 +84,10 @@ add_library(mtmd
|
||||
models/qwen3tts-spkenc.cpp
|
||||
models/qwen3tts-gen.cpp
|
||||
models/drama3-gen.cpp
|
||||
models/fish-s2-gen.cpp
|
||||
models/fish-s2-spkenc.cpp
|
||||
models/zonos3-gen.cpp
|
||||
models/yue2-gen.cpp
|
||||
models/zonos3-spkenc.cpp
|
||||
models/pockettts-seanet.cpp
|
||||
models/pockettts-spkenc.cpp
|
||||
|
||||
@@ -39,6 +39,7 @@ Rules of thumb:
|
||||
- Models and backends must outlive the subcontexts, `mtmd_context` declares them first for that reason
|
||||
- Models having both audio and vision encoders must have different arch names (projector types), postfixed by `a` and `v`. See Gemma 4 as an example (`gemma4a` / `gemma4v`)
|
||||
- Locking happens only at the public API boundary (`mtmd.cpp`): a new public function that runs a graph must take `compute_mutex`. The internals (subcontexts, preprocessors, `mtmd_audio_log_mel()`) assume it is held and never lock
|
||||
- No magic numbers for a model's fixed shapes (layer counts, codebook sizes, window lengths...): define them once as `static constexpr` in the graph struct in `models/models.h`, and refer to them from the loader or the subcontext (e.g. `mtmd_graph_zonos3_gen::n_codebooks`, `mtmd_graph_fish_s2_spkenc::n_tfm_layer`)
|
||||
|
||||
## Pipeline
|
||||
|
||||
@@ -83,6 +84,18 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i
|
||||
- Core API `mtmd.h`: handles main inference. Important: `mtmd_gen_state` holds only the per-sequence data that the mmproj itself writes and reads back (RNG, decoder KV / conv state, output buffers). Data that goes to or comes from the backbone (hidden states, codes, PCM) must stay explicit in `mtmd_gen_inp` / `mtmd_gen_out`; the caller drives the backbone and handles audio frame accumulation.
|
||||
- Helper API `mtmd-helper.h`: provides a model-agnostic stateful API. Usage example can be found in the `tools/tts` directory.
|
||||
|
||||
### Running a transformer stage on libllama (mtmd-owned `llama_context`)
|
||||
|
||||
Some stages are a full LLM-sized transformer that attends to the keys/values of the backbone, for example the flow-matching NAR of YuE2 (same layer shapes as the backbone, its own weights, bidirectional over its rows, and it reads the backbone KV). Do not re-implement it as an mtmd graph. Instead, mtmd can own a second `llama_model` + `llama_context` (see `mtmd_yue2_nar_load()` and `mtmd_gen_flow_yue2()` in `mtmd.cpp`):
|
||||
|
||||
- Conversion: store the layers in the mmproj under `a.gen.<stage>.` with the libllama tensor names of an existing arch (e.g. `a.gen.nar.blk.0.attn_q.weight`)
|
||||
- Loading: build a synthetic `gguf_context` with the arch hparams (`general.architecture`, `tokenizer.ggml.model = none`, `vocab_size = 1`, ...) and one tensor info per mmproj tensor (only the type is used, the shapes come from the hparams), then call `llama_model_init_from_user()`. The `set_tensor_data` callback reads the data from the mmproj file, so no copy is kept in the mtmd model
|
||||
- libllama also creates every optional tensor of the arch (biases, `*.scale`, `*.input_scale`, `dense_2`, ...) for a model built from metadata. The callback must fill them with values that leave the graph unchanged: 0 for biases, 1 for scales, identity for `dense_2`. Zeros everywhere silently make every matmul output 0
|
||||
- Backbone keys/values: the helper passes the backbone sequence state (`llama_state_seq_get_data()`) through `mtmd_gen_inp`, mtmd loads it into its context with `llama_state_seq_set_data()`. This only checks the layer count and the K/V types and row sizes, so it works across models with the same KV layout. The two contexts must agree on the KV type and on flash attention (`v_trans`)
|
||||
- Inputs and outputs are embedding rows (`llama_batch::embd`, `llama_get_embeddings()` with `LLAMA_POOLING_TYPE_NONE`), the projections around them stay small mtmd graphs
|
||||
- Non-causal rows: `LLAMA_ATTENTION_TYPE_NON_CAUSAL`, all rows in one ubatch (`n_ubatch` >= rows), they see every cached position of the sequence. Remove them with `llama_memory_seq_rm()` after each pass
|
||||
- Size the context per call (`n_ctx`, `n_ubatch`) instead of keeping one for the whole session
|
||||
|
||||
### Checklist for porting new audio generation models to mtmd
|
||||
|
||||
1. Make sure to consult merged PRs about adding new TTS models, especially reviewer comments
|
||||
@@ -109,7 +122,9 @@ Due to wide variety of audio generation pipelines, the `mtmd_gen_audio` system i
|
||||
- 60% changes inside `mtmd-helper-gen.cpp`
|
||||
- 10% changes inside `libmtmd` core (`mtmd-model`, `mtmd-subcontext`, `models/`)
|
||||
- The rest downstream code (CLI, server) should have no changes at all
|
||||
5. Update usage documentation in `docs/multimodal/tts.md`
|
||||
5. Validate with several concurrent requests on `llama-server` (e.g. `-np 2`, one plain and one streaming request at the same time), not only with `llama-tts`
|
||||
- The server interleaves the slots on one `llama_context`, so a pipeline must copy what it needs (hidden state, logits) right after its own decode, not read it at the next step
|
||||
6. Update usage documentation in `docs/multimodal/tts.md`
|
||||
|
||||
IMPORTANT: If your model needs changes that don't fit the existing infrastructure, **open an issue first for discussion**.
|
||||
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
#include "models.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <vector>
|
||||
|
||||
// one frame: backbone hidden + semantic code -> the codes of this frame + next backbone input
|
||||
// fast AR row 0 is the backbone hidden, row p > 0 is the embedding of code p - 1 and picks code p (greedy, as the reference)
|
||||
ggml_cgraph * mtmd_graph_fish_s2_gen::build_fast_ar() {
|
||||
const int64_t n_cb = model.gen_code_embd_w->ne[2];
|
||||
const int64_t n_embd_cb = model.gen_code_embd_w->ne[0];
|
||||
const int64_t n_embd = model.gen_code_tok_embd_w->ne[0];
|
||||
const int64_t n_kv = model.layers[0].k_w->ne[1];
|
||||
const int n_layer = (int) model.layers.size();
|
||||
|
||||
ggml_tensor * h = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd);
|
||||
ggml_set_name(h, "inp_raw");
|
||||
ggml_set_input(h);
|
||||
|
||||
ggml_tensor * inp_code0 = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1);
|
||||
ggml_set_name(inp_code0, "inp_code0");
|
||||
ggml_set_input(inp_code0);
|
||||
|
||||
mtmd_graph_qwen3tts_gen::code_gen cg(*this, top_k, top_p);
|
||||
|
||||
std::vector<ggml_tensor *> k_cache(n_layer), v_cache(n_layer);
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
k_cache[il] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, n_cb), 0.0f);
|
||||
v_cache[il] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_kv, n_cb), 0.0f);
|
||||
}
|
||||
|
||||
ggml_tensor * out_codes = cg.cache_set(ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, 1, n_cb), 0, inp_code0);
|
||||
|
||||
for (int p = 0; p < n_cb; p++) {
|
||||
ggml_tensor * cur = h;
|
||||
if (p > 0) {
|
||||
ggml_tensor * code = ggml_view_1d(ctx0, out_codes, 1, (size_t) (p - 1) * out_codes->nb[1]);
|
||||
cur = ggml_reshape_1d(ctx0, ggml_get_rows(ctx0, model.gen_code_tok_embd_w, code), n_embd);
|
||||
}
|
||||
ggml_tensor * kq_mask = cg.causal_mask_row(n_cb, p);
|
||||
ggml_tensor * inp_pos = cg.const_i32(k_cache[0], (float) p);
|
||||
for (int il = 0; il < n_layer; il++) {
|
||||
cur = cg.layer_forward(cur, model.layers[il], inp_pos, kq_mask, k_cache[il], v_cache[il], n_cb, p, il);
|
||||
}
|
||||
if (p == 0) {
|
||||
continue; // only fills the KV
|
||||
}
|
||||
cur = ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, hparams.eps), model.gen_code_norm_w);
|
||||
ggml_tensor * logits = ggml_mul_mat(ctx0, model.gen_code_head_w, cur);
|
||||
out_codes = cg.cache_set(out_codes, p, ggml_argmax(ctx0, logits));
|
||||
}
|
||||
|
||||
out_codes = ggml_cont(ctx0, out_codes);
|
||||
ggml_set_name(out_codes, "out_codes");
|
||||
ggml_set_output(out_codes);
|
||||
ggml_build_forward_expand(gf, out_codes);
|
||||
|
||||
// backbone input of the next row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1)
|
||||
ggml_tensor * out_embd = ggml_get_rows(ctx0, model.gen_code_out_embd_w, inp_code0);
|
||||
for (int64_t g = 0; g < n_cb; g++) {
|
||||
ggml_tensor * table = ggml_view_2d(ctx0, model.gen_code_embd_w, n_embd_cb, model.gen_code_embd_w->ne[1],
|
||||
model.gen_code_embd_w->nb[1], (size_t) g * model.gen_code_embd_w->nb[2]);
|
||||
ggml_tensor * code = ggml_view_1d(ctx0, out_codes, 1, (size_t) g * out_codes->nb[1]);
|
||||
out_embd = ggml_add(ctx0, out_embd, ggml_get_rows(ctx0, table, code));
|
||||
}
|
||||
out_embd = ggml_scale(ctx0, out_embd, 1.0f / sqrtf((float) (n_cb + 1)));
|
||||
cb(out_embd, "gen_audio_out", -1);
|
||||
ggml_build_forward_expand(gf, out_embd);
|
||||
|
||||
return gf;
|
||||
}
|
||||
|
||||
ggml_cgraph * mtmd_graph_fish_s2_gen::build() {
|
||||
return gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV ? build_codec() : build_fast_ar();
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
#include "models.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <string>
|
||||
|
||||
// reference clip -> codec frames -> backbone input rows, see DAC.encode() and DownsampleResidualVectorQuantize.encode()
|
||||
// activations are [C, T]; a conv is a sum of matmuls over tap-shifted views: exact in F32, no im2col buffer
|
||||
|
||||
// causal conv, w is taps-major [stride * IC, OC, K / stride], see conversion/fish_s2.py
|
||||
ggml_tensor * mtmd_graph_fish_s2_spkenc::conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation) const {
|
||||
const int64_t n_tap = w->ne[2];
|
||||
const int pad = (int) (n_tap - 1) * stride * dilation;
|
||||
if (pad > 0) {
|
||||
x = ggml_pad_ext(ctx0, x, 0, 0, pad, 0, 0, 0, 0, 0);
|
||||
}
|
||||
if (stride > 1) {
|
||||
// stride frames per column, then each tap group is one matmul
|
||||
x = ggml_reshape_2d(ctx0, x, x->ne[0] * stride, x->ne[1] / stride);
|
||||
}
|
||||
const int64_t n_out = x->ne[1] - (n_tap - 1) * dilation;
|
||||
ggml_tensor * y = nullptr;
|
||||
for (int64_t k = 0; k < n_tap; k++) {
|
||||
ggml_tensor * wk = ggml_view_2d(ctx0, w, w->ne[0], w->ne[1], w->nb[1], (size_t) k * w->nb[2]);
|
||||
ggml_tensor * xk = ggml_view_2d(ctx0, x, x->ne[0], n_out, x->nb[1], (size_t) k * dilation * x->nb[1]);
|
||||
ggml_tensor * yk = ggml_mul_mat(ctx0, wk, xk);
|
||||
y = y ? ggml_add(ctx0, y, yk) : yk;
|
||||
}
|
||||
return ggml_add(ctx0, y, b);
|
||||
}
|
||||
|
||||
// x + sin(alpha * x)^2 / alpha, beta = 1 / alpha is folded at conversion
|
||||
// TODO @ngxson: on Metal, residual codes match the reference less often than on CPU (89-98% vs 99%), maybe the fast-math sin
|
||||
ggml_tensor * mtmd_graph_fish_s2_spkenc::snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const {
|
||||
ggml_tensor * s = ggml_sqr(ctx0, ggml_sin(ctx0, ggml_mul(ctx0, x, alpha)));
|
||||
return ggml_add(ctx0, x, ggml_mul(ctx0, s, beta));
|
||||
}
|
||||
|
||||
// causal depthwise conv (w [C, K]) -> LayerNorm -> pwconv1 -> GELU -> pwconv2 -> layer scale -> residual
|
||||
ggml_tensor * mtmd_graph_fish_s2_spkenc::convnext(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk) const {
|
||||
const int64_t n_tap = blk.dwconv_w->ne[1];
|
||||
const int64_t n_pos = x->ne[1];
|
||||
ggml_tensor * xp = ggml_pad_ext(ctx0, x, 0, 0, (int) n_tap - 1, 0, 0, 0, 0, 0);
|
||||
ggml_tensor * h = nullptr;
|
||||
for (int64_t k = 0; k < n_tap; k++) {
|
||||
ggml_tensor * wk = ggml_view_1d(ctx0, blk.dwconv_w, blk.dwconv_w->ne[0], (size_t) k * blk.dwconv_w->nb[1]);
|
||||
ggml_tensor * xk = ggml_view_2d(ctx0, xp, xp->ne[0], n_pos, xp->nb[1], (size_t) k * xp->nb[1]);
|
||||
ggml_tensor * hk = ggml_mul(ctx0, xk, wk);
|
||||
h = h ? ggml_add(ctx0, h, hk) : hk;
|
||||
}
|
||||
h = ggml_add(ctx0, h, blk.dwconv_b);
|
||||
h = ggml_add(ctx0, ggml_mul(ctx0, ggml_norm(ctx0, h, 1e-6f), blk.norm_w), blk.norm_b);
|
||||
h = ggml_gelu_erf(ctx0, ggml_add(ctx0, ggml_mul_mat(ctx0, blk.pw1_w, h), blk.pw1_b));
|
||||
h = ggml_add(ctx0, ggml_mul_mat(ctx0, blk.pw2_w, h), blk.pw2_b);
|
||||
return ggml_add(ctx0, x, ggml_mul(ctx0, h, blk.gamma));
|
||||
}
|
||||
|
||||
ggml_cgraph * mtmd_graph_fish_s2_spkenc::build() {
|
||||
const auto & c2w = model.c2w;
|
||||
|
||||
// the preprocessor hands over the waveform as a single-row "mel", [n_samples, 1] is also [1, n_samples]
|
||||
ggml_tensor * inp_raw = build_inp_raw(1);
|
||||
ggml_tensor * x = ggml_reshape_2d(ctx0, inp_raw, 1, inp_raw->ne[0]);
|
||||
|
||||
// windowed causal transformer on a cold start: zero state, positions from 0
|
||||
mtmd_graph_qwen3tts_gen::code2wav dec(*this);
|
||||
auto zero_state = [&](const std::string & name, int64_t n0, int64_t n1) {
|
||||
dec.state_in[name] = ggml_fill(ctx0, ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n0, n1), 0.0f);
|
||||
};
|
||||
auto transformer = [&](ggml_tensor * cur, int il0, int il1, int64_t window, ggml_tensor * norm_w) {
|
||||
const int64_t n_kv = model.layers[il0].k_w->ne[1];
|
||||
zero_state("tfm_pos", 1, 1);
|
||||
for (int il = il0; il < il1; il++) {
|
||||
zero_state("tfm_k_" + std::to_string(il), n_kv, window - 1);
|
||||
zero_state("tfm_v_" + std::to_string(il), n_kv, window - 1);
|
||||
cur = dec.tfm_layer_forward(cur, model.layers[il], il, window);
|
||||
}
|
||||
return ggml_mul(ctx0, ggml_rms_norm(ctx0, cur, hparams.wav_tfm_eps), norm_w);
|
||||
};
|
||||
|
||||
// encoder: entry conv, 4x (3 residual units, Snake, strided conv), transformer, Snake, post conv
|
||||
x = conv(x, c2w.dac_entry_w, c2w.dac_entry_b, 1, 1);
|
||||
static constexpr int DAC_DILATIONS[3] = { 1, 3, 9 };
|
||||
for (const auto & blk : c2w.dac) {
|
||||
for (size_t ir = 0; ir < blk.res.size(); ir++) {
|
||||
const auto & res = blk.res[ir];
|
||||
ggml_tensor * h = snake(x, res.act1_alpha, res.act1_beta);
|
||||
h = conv(h, res.conv1_w, res.conv1_b, 1, DAC_DILATIONS[ir]);
|
||||
h = snake(h, res.act2_alpha, res.act2_beta);
|
||||
h = conv(h, res.conv2_w, res.conv2_b, 1, 1);
|
||||
x = ggml_add(ctx0, x, h);
|
||||
}
|
||||
x = snake(x, blk.snake_alpha, blk.snake_beta);
|
||||
x = conv(x, blk.conv_w, blk.conv_b, (int) (blk.conv_w->ne[0] / x->ne[0]), 1);
|
||||
}
|
||||
x = transformer(x, 0, n_enc_tfm_layer, 512, model.post_ln_w);
|
||||
x = snake(x, c2w.dac_post_snake_alpha, c2w.dac_post_snake_beta);
|
||||
x = conv(x, c2w.dac_post_conv_w, c2w.dac_post_conv_b, 1, 1);
|
||||
cb(x, "codec_enc", -1);
|
||||
|
||||
// quantizer: 2x (strided conv + ConvNeXt), transformer
|
||||
for (const auto & down : c2w.upsample) {
|
||||
x = conv(x, down.conv_w, down.conv_b, (int) (down.conv_w->ne[0] / x->ne[0]), 1);
|
||||
x = convnext(x, down);
|
||||
}
|
||||
ggml_tensor * z = transformer(x, n_enc_tfm_layer, (int) model.layers.size(), 128, c2w.tfm_output_norm_w);
|
||||
cb(z, "codec_latent", -1);
|
||||
|
||||
// nearest entry by cosine similarity, then the entry through out_proj is removed from the residual
|
||||
auto vq = [&](ggml_tensor * in_w, ggml_tensor * in_b, ggml_tensor * cb_norm, ggml_tensor * table) {
|
||||
ggml_tensor * e = ggml_add(ctx0, ggml_mul_mat(ctx0, in_w, z), in_b);
|
||||
e = ggml_l2_norm(ctx0, e, 1e-12f);
|
||||
ggml_tensor * idx = ggml_argmax(ctx0, ggml_mul_mat(ctx0, cb_norm, e));
|
||||
z = ggml_sub(ctx0, z, ggml_get_rows(ctx0, table, idx));
|
||||
return idx;
|
||||
};
|
||||
auto page = [&](ggml_tensor * t, int64_t g) {
|
||||
return ggml_view_2d(ctx0, t, t->ne[0], t->ne[1], t->nb[1], (size_t) g * t->nb[2]);
|
||||
};
|
||||
|
||||
// backbone row: (E_lm[semantic] + sum of E_cb[g][code g]) / sqrt(n_cb + 1), same as fish_s2_gen
|
||||
const int64_t n_cb = model.gen_code_embd_w->ne[2];
|
||||
ggml_tensor * idx = vq(c2w.quant_first_in_w, c2w.quant_first_in_b, c2w.quant_first_cb_norm_w, c2w.quant_first_cb_w);
|
||||
ggml_tensor * out = ggml_add(ctx0, ggml_get_rows(ctx0, model.gen_code_out_embd_w, idx),
|
||||
ggml_get_rows(ctx0, page(model.gen_code_embd_w, 0), idx));
|
||||
for (int64_t g = 1; g < n_cb; g++) {
|
||||
ggml_tensor * in_b = ggml_view_1d(ctx0, c2w.quant_rest_in_b, c2w.quant_rest_in_b->ne[0], (size_t) (g - 1) * c2w.quant_rest_in_b->nb[1]);
|
||||
idx = vq(page(c2w.quant_rest_in_w, g - 1), in_b, page(c2w.quant_rest_cb_norm_w, g - 1), page(c2w.quant_rest_cb_w, g - 1));
|
||||
out = ggml_add(ctx0, out, ggml_get_rows(ctx0, page(model.gen_code_embd_w, g), idx));
|
||||
}
|
||||
out = ggml_scale(ctx0, out, 1.0f / sqrtf((float) (n_cb + 1)));
|
||||
cb(out, "spk_rows", -1);
|
||||
ggml_build_forward_expand(gf, out);
|
||||
|
||||
return gf;
|
||||
}
|
||||
@@ -462,7 +462,8 @@ struct mtmd_graph_qwen3tts_gen : mtmd_graph {
|
||||
ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const;
|
||||
|
||||
ggml_tensor * quant_decode(ggml_tensor * inp_codes) const;
|
||||
ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il) const;
|
||||
// window <= 0 uses hparams.wav_tfm_swa
|
||||
ggml_tensor * tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il, int64_t window = 0) const;
|
||||
ggml_tensor * convnext_block(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk, const std::string & state_prefix) const;
|
||||
ggml_tensor * dac_res_unit(ggml_tensor * x, const mtmd_code2wav::dac_res & res, int dilation, const std::string & state_name) const;
|
||||
|
||||
@@ -569,6 +570,32 @@ struct mtmd_graph_drama3_gen : mtmd_graph {
|
||||
ggml_tensor * inp_pos, ggml_tensor * k_slot, ggml_tensor * v_slot, int pos);
|
||||
};
|
||||
|
||||
//
|
||||
// fish_s2 generation:
|
||||
// GEN_CODE = fast AR, backbone hidden + semantic code -> other codes of the frame + next backbone input
|
||||
// GEN_WAV = Fish DAC decoder, same as drama3
|
||||
//
|
||||
struct mtmd_graph_fish_s2_gen : mtmd_graph_drama3_gen {
|
||||
using mtmd_graph_drama3_gen::mtmd_graph_drama3_gen;
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
ggml_cgraph * build_fast_ar();
|
||||
};
|
||||
|
||||
// fish_s2 voice cloning: raw 44.1 kHz clip -> codec encoder + RVQ -> one backbone input row per frame
|
||||
struct mtmd_graph_fish_s2_spkenc : mtmd_graph {
|
||||
mtmd_graph_fish_s2_spkenc(mtmd_subcontext * ctx, const mtmd_image_f32 & img) : mtmd_graph(ctx, img) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
// codec encoder transformer layers, the rest of model.layers is the quantizer pre_module
|
||||
static constexpr int n_enc_tfm_layer = 4;
|
||||
static constexpr int n_tfm_layer = 12;
|
||||
|
||||
ggml_tensor * conv(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int stride, int dilation) const;
|
||||
ggml_tensor * snake(ggml_tensor * x, ggml_tensor * alpha, ggml_tensor * beta) const;
|
||||
ggml_tensor * convnext(ggml_tensor * x, const mtmd_code2wav::upsample_block & blk) const;
|
||||
};
|
||||
|
||||
//
|
||||
// zonos3 generation:
|
||||
// GEN_CODE = 10 codebook heads, or the backbone row of a sampled frame
|
||||
@@ -610,6 +637,33 @@ struct mtmd_graph_zonos3_gen : mtmd_graph {
|
||||
|
||||
std::vector<c2w_state_slot> list_zonos3_gen_state_slots(const mtmd_hparams & hparams, const mtmd_model & model);
|
||||
|
||||
//
|
||||
// YuE2 generation, the NAR transformer itself runs on a llama_context, see mtmd_gen_flow_yue2():
|
||||
// LATENT_IN = latent rows -> backbone embd rows
|
||||
// LATENT_OUT = backbone hidden rows -> velocity rows
|
||||
// GEN_WAV = Oobleck VAE decoder, latent rows -> interleaved stereo PCM
|
||||
//
|
||||
struct mtmd_graph_yue2_gen : mtmd_graph {
|
||||
mtmd_graph_yue2_gen(mtmd_subcontext * ctx, const mtmd_image_f32 & img, mtmd_subctx_gen_process gen_process, int n_rows)
|
||||
: mtmd_graph(ctx, img), gen_process(gen_process), n_rows(n_rows) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
static constexpr int n_latent = 64;
|
||||
static constexpr int n_freq = 256; // time embedder input
|
||||
static constexpr int n_channels = 2;
|
||||
static constexpr int vae_ratio = 1920; // output samples per latent frame
|
||||
static constexpr int vae_trim = 64; // a decode of n frames has vae_ratio * n - vae_trim samples
|
||||
static constexpr float rope_theta = 1000000.0f;
|
||||
static constexpr int n_ctx_max = 24576; // the AR prefix and the NAR rows of one chunk
|
||||
static constexpr int ode_steps = 32; // midpoint solver
|
||||
|
||||
mtmd_subctx_gen_process gen_process;
|
||||
int n_rows;
|
||||
|
||||
ggml_tensor * snake(ggml_tensor * x, const mtmd_yue2::snake & s) const;
|
||||
ggml_tensor * conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const;
|
||||
};
|
||||
|
||||
struct mtmd_graph_soprano : mtmd_graph {
|
||||
mtmd_graph_soprano(mtmd_subcontext * ctx, const mtmd_image_f32 & img, int n_frames) : mtmd_graph(ctx, img), n_frames(n_frames) {}
|
||||
ggml_cgraph * build() override;
|
||||
|
||||
@@ -156,10 +156,13 @@ ggml_tensor * mtmd_graph_qwen3tts_gen::code_gen::layer_forward(
|
||||
q = ggml_reshape_3d(ctx0, q, d_head, n_head, 1);
|
||||
k = ggml_reshape_3d(ctx0, k, d_head, n_head_kv, 1);
|
||||
|
||||
q = ggml_rms_norm(ctx0, q, hparams.eps);
|
||||
q = ggml_mul(ctx0, q, layer.q_norm);
|
||||
k = ggml_rms_norm(ctx0, k, hparams.eps);
|
||||
k = ggml_mul(ctx0, k, layer.k_norm);
|
||||
// fish_s2 fast AR has no qk norm
|
||||
if (layer.q_norm) {
|
||||
q = ggml_rms_norm(ctx0, q, hparams.eps);
|
||||
q = ggml_mul(ctx0, q, layer.q_norm);
|
||||
k = ggml_rms_norm(ctx0, k, hparams.eps);
|
||||
k = ggml_mul(ctx0, k, layer.k_norm);
|
||||
}
|
||||
|
||||
q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, (int) d_head, GGML_ROPE_TYPE_NEOX, 0,
|
||||
hparams.rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
|
||||
@@ -418,12 +421,12 @@ ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::quant_decode(ggml_tensor * inp_
|
||||
// one pre_transformer layer over a batch of N = sliding_window new frames
|
||||
// attention runs over [(W-1)-frame prefix from the last batch] + [N new frames]
|
||||
// RoPE positions come from a persisted counter, so phases line up across batches
|
||||
ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il) const {
|
||||
ggml_tensor * mtmd_graph_qwen3tts_gen::code2wav::tfm_layer_forward(ggml_tensor * cur, const mtmd_layer & layer, int il, int64_t window) const {
|
||||
const int n_head = hparams.wav_tfm_n_head;
|
||||
const int n_head_kv = hparams.wav_tfm_n_head_kv;
|
||||
const int64_t d_head = layer.q_w->ne[1] / n_head;
|
||||
const float kq_scale = 1.0f / sqrtf((float) d_head);
|
||||
const int64_t W = hparams.wav_tfm_swa; // == N, frames per batch
|
||||
const int64_t W = window > 0 ? window : hparams.wav_tfm_swa; // code2wav: == N, frames per batch
|
||||
const int64_t N = cur->ne[1];
|
||||
const int64_t prefix = W - 1;
|
||||
const int64_t total_kv = prefix + N;
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
#include "models.h"
|
||||
|
||||
// SnakeBeta, alpha and beta folded at conversion: x + beta * sin(alpha * x)^2
|
||||
// x: [T, C], alpha/beta: [C]
|
||||
ggml_tensor * mtmd_graph_yue2_gen::snake(ggml_tensor * x, const mtmd_yue2::snake & s) const {
|
||||
ggml_tensor * a = ggml_reshape_2d(ctx0, s.alpha, 1, s.alpha->ne[0]);
|
||||
ggml_tensor * b = ggml_reshape_2d(ctx0, s.beta, 1, s.beta->ne[0]);
|
||||
|
||||
// expand reshapes first so mul/sin/sqr/mul/add lands as consecutive nodes, letting backends fuse them
|
||||
ggml_build_forward_expand(gf, a);
|
||||
ggml_build_forward_expand(gf, b);
|
||||
|
||||
ggml_tensor * y = ggml_sqr(ctx0, ggml_sin(ctx0, ggml_mul(ctx0, x, a)));
|
||||
return ggml_add(ctx0, x, ggml_mul(ctx0, y, b));
|
||||
}
|
||||
|
||||
// same-length conv, stride 1; x: [T, IC], w: [K, IC, OC], returns [T, OC]
|
||||
// im2col in F32: the reference decoder runs in FP32
|
||||
ggml_tensor * mtmd_graph_yue2_gen::conv1d(ggml_tensor * x, ggml_tensor * w, ggml_tensor * b, int dilation) const {
|
||||
const int pad = (int) (w->ne[0] - 1) / 2 * dilation;
|
||||
ggml_tensor * col = ggml_im2col(ctx0, w, x, 1, 0, pad, 0, dilation, 0, false, GGML_TYPE_F32); // [1, T, IC * K]
|
||||
ggml_tensor * y = ggml_mul_mat(ctx0,
|
||||
ggml_reshape_2d(ctx0, col, col->ne[0], col->ne[1] * col->ne[2]),
|
||||
ggml_reshape_2d(ctx0, w, w->ne[0] * w->ne[1], w->ne[2]));
|
||||
if (b) {
|
||||
y = ggml_add(ctx0, y, ggml_reshape_2d(ctx0, b, 1, b->ne[0]));
|
||||
}
|
||||
return y;
|
||||
}
|
||||
|
||||
ggml_cgraph * mtmd_graph_yue2_gen::build() {
|
||||
const auto & y = model.yue2;
|
||||
|
||||
if (gen_process == MTMD_SUBCTX_GEN_PROCESS_LATENT_IN || gen_process == MTMD_SUBCTX_GEN_PROCESS_LATENT_OUT) {
|
||||
const bool is_in = gen_process == MTMD_SUBCTX_GEN_PROCESS_LATENT_IN;
|
||||
ggml_tensor * w = is_in ? y.inp_w : y.out_w;
|
||||
ggml_tensor * b = is_in ? y.inp_b : y.out_b;
|
||||
ggml_tensor * cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, w->ne[0], n_rows);
|
||||
ggml_set_name(cur, "inp_feats");
|
||||
ggml_set_input(cur);
|
||||
cur = ggml_add(ctx0, build_mm(w, cur), b);
|
||||
ggml_set_name(cur, "out_feats");
|
||||
ggml_set_output(cur);
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
return gf;
|
||||
}
|
||||
|
||||
GGML_ASSERT(gen_process == MTMD_SUBCTX_GEN_PROCESS_GEN_WAV);
|
||||
static constexpr int dilations[3] = { 1, 3, 9 };
|
||||
|
||||
ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_latent, n_rows);
|
||||
ggml_set_name(inp, "inp_feats");
|
||||
ggml_set_input(inp);
|
||||
|
||||
ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [T, n_latent]
|
||||
x = conv1d(x, y.vae_in_w, y.vae_in_b, 1);
|
||||
for (size_t ib = 0; ib < y.vae.size(); ib++) {
|
||||
const auto & blk = y.vae[ib];
|
||||
const int stride = (int) (blk.up_w->ne[0] / 2);
|
||||
const int pad = (stride + 1) / 2; // ceil(stride / 2) of the reference
|
||||
x = snake(x, blk.act);
|
||||
{
|
||||
// ConvTranspose1d as GEMM + col2im, same as qwen3tts; up_w: [K, OC, IC]
|
||||
const int64_t K = blk.up_w->ne[0];
|
||||
const int64_t OC = blk.up_w->ne[1];
|
||||
ggml_tensor * w2 = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_reshape_2d(ctx0, blk.up_w, K * OC, blk.up_w->ne[2])));
|
||||
ggml_tensor * col = ggml_mul_mat(ctx0, w2, ggml_cont(ctx0, ggml_transpose(ctx0, x))); // [K * OC, T]
|
||||
x = ggml_col2im_1d(ctx0, col, stride, (int) OC, pad);
|
||||
}
|
||||
x = ggml_add(ctx0, x, ggml_reshape_2d(ctx0, blk.up_b, 1, blk.up_b->ne[0]));
|
||||
for (size_t ir = 0; ir < blk.res.size(); ir++) {
|
||||
const auto & ru = blk.res[ir];
|
||||
ggml_tensor * h = snake(x, ru.act1);
|
||||
h = conv1d(h, ru.conv1_w, ru.conv1_b, dilations[ir]);
|
||||
h = snake(h, ru.act2);
|
||||
h = conv1d(h, ru.conv2_w, ru.conv2_b, 1);
|
||||
x = ggml_add(ctx0, x, h);
|
||||
}
|
||||
cb(x, "vae_block", (int) ib);
|
||||
}
|
||||
x = snake(x, y.vae_post);
|
||||
x = conv1d(x, y.vae_out_w, nullptr, 1); // [T, n_channels]
|
||||
|
||||
x = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // interleaved
|
||||
ggml_set_name(x, "out_audio");
|
||||
ggml_set_output(x);
|
||||
ggml_build_forward_expand(gf, x);
|
||||
return gf;
|
||||
}
|
||||
@@ -2242,6 +2242,16 @@ bool mtmd_audio_preprocessor_pockettts::process(const float * sa
|
||||
return true;
|
||||
}
|
||||
|
||||
bool mtmd_audio_preprocessor_fish_s2::process(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output, bool shape_only) const {
|
||||
const size_t max_samples = (size_t) mtmd_hparams::fish_s2_max_spk_seconds * hparams.audio_sample_rate;
|
||||
if (n_samples > max_samples) {
|
||||
LOG_WRN("%s: speaker reference is %.1f s, truncating to the first %d s\n", __func__,
|
||||
(double) n_samples / hparams.audio_sample_rate, mtmd_hparams::fish_s2_max_spk_seconds);
|
||||
n_samples = max_samples;
|
||||
}
|
||||
return mtmd_audio_preprocessor_muse_spark_audio::process(samples, n_samples, output, shape_only);
|
||||
}
|
||||
|
||||
//
|
||||
// mtmd_audio_preprocessor_vibevoice_asr
|
||||
//
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user