mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-19 17:27:41 -06:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b94959a64d | ||
|
|
50518b64c6 | ||
|
|
c406f4367e | ||
|
|
7fce5439fe | ||
|
|
53ecea05a0 | ||
|
|
9a080f44b5 | ||
|
|
c7468107ef | ||
|
|
ac830a9cbb | ||
|
|
8af68d486a | ||
|
|
f8caa5ea30 | ||
|
|
4db8b7638f | ||
|
|
01592dc58b | ||
|
|
8bf2132c3f | ||
|
|
5812201a81 | ||
|
|
41d8297fe4 | ||
|
|
51a77c4f24 | ||
|
|
76f186df71 | ||
|
|
95b09b62be | ||
|
|
b5dbca1e44 | ||
|
|
80437488a4 | ||
|
|
39039aa27f | ||
|
|
7a4117eef9 | ||
|
|
573aa89de6 | ||
|
|
ecafc67145 | ||
|
|
d4f5c26049 | ||
|
|
08c252e221 | ||
|
|
e4b620af19 | ||
|
|
b001d7bb22 | ||
|
|
0a76757469 | ||
|
|
b044890408 | ||
|
|
78e05298f0 | ||
|
|
785f6bac00 | ||
|
|
edec462cc0 | ||
|
|
4483b2b987 | ||
|
|
bafa0420c1 | ||
|
|
2cbfc3cf34 | ||
|
|
6f311d7ad6 | ||
|
|
ff6b8d1761 | ||
|
|
98d67c8d53 | ||
|
|
387395aa8d | ||
|
|
a693a5f66f | ||
|
|
b069e526de | ||
|
|
ffc87c3a79 | ||
|
|
10f173b253 | ||
|
|
97069ff6c4 | ||
|
|
bba8408821 | ||
|
|
48a32194ee | ||
|
|
6da36139e8 | ||
|
|
61f182fcde | ||
|
|
03ec5b4326 | ||
|
|
7144124a5b | ||
|
|
43bd7a2a42 | ||
|
|
65758d01f5 | ||
|
|
a300a9c334 | ||
|
+4 |
4577932fab | ||
|
|
398f00d657 | ||
|
|
9e93cd8001 | ||
|
|
aaa807c568 | ||
|
|
401563f10e | ||
|
|
52975342ff | ||
|
|
3574df84b7 | ||
|
|
2e53a0bab7 | ||
|
|
744c68cd14 | ||
|
|
307e3f1551 | ||
|
|
9048bad8d6 | ||
|
|
a9d43ecc30 | ||
|
|
8c16babba5 | ||
|
|
ad9d9e8f50 | ||
|
|
955c4e8ee5 | ||
|
|
10152bc4a0 | ||
|
|
8777c0c833 | ||
|
|
918076a635 | ||
|
|
ebc9fb933e | ||
|
|
8b8ca4c98e | ||
|
|
08a9b35893 | ||
|
|
acad5e405c |
@@ -3,11 +3,11 @@ ARG OPENVINO_VERSION_FULL=2026.4.1.22982.07f9c262b05
|
||||
ARG UBUNTU_VERSION=24.04
|
||||
|
||||
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
|
||||
ARG IGC_VERSION=v2.41.5
|
||||
ARG IGC_VERSION_FULL=2_2.41.5+22716
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.35.39758.10
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.35.39758.10-0
|
||||
ARG IGDGMM_VERSION=22.10.0
|
||||
ARG IGC_VERSION=v2.42.6
|
||||
ARG IGC_VERSION_FULL=2_2.42.6+23070
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.40.40196.8
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.40.40196.8-0
|
||||
ARG IGDGMM_VERSION=22.10.2
|
||||
|
||||
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
|
||||
ARG NPU_DRIVER_VERSION=v1.38.0
|
||||
|
||||
@@ -25,7 +25,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-11.7.91-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-11.7.101-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-11.7.91-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-11.7.91-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvml_dev-windows-x86_64-11.7.91-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_cudart-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvcc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvrtc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
@@ -54,7 +56,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-12.4.127-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvml_dev-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_cudart-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvcc-windows-x86_64-12.4.131-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvrtc-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
@@ -85,7 +89,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.1.80-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.1.68-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-13.1.78-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.1.68-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvml_dev-windows-x86_64-13.1.68-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_crt-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_cudart-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvcc-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
@@ -116,7 +122,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.5.37-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
@@ -143,7 +151,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.10.0.20-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-arm64/cuda_nvml_dev-windows-arm64-13.5.37-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-arm64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
|
||||
@@ -140,6 +140,11 @@ jobs:
|
||||
source .venv/Scripts/activate
|
||||
pip install -r requirements/requirements-convert_hf_to_gguf.txt
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
shell: cmd
|
||||
|
||||
@@ -187,6 +187,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
|
||||
@@ -61,6 +61,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Install OpenCL Headers and Libs
|
||||
id: install_opencl
|
||||
run: |
|
||||
|
||||
@@ -213,6 +213,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
|
||||
@@ -900,6 +900,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
@@ -1146,6 +1151,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
# TODO: these jobs need to use llvm toolchain in order to utilize the ccache
|
||||
#- name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
|
||||
@@ -175,6 +175,11 @@ jobs:
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
shell: cmd
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
name: UI Chat Publish
|
||||
|
||||
# Publishes Llama Chat to npm: @ggml-org/llama-chat (ui/apps/chat, the app and its CLI) and
|
||||
# llama-chat (ui/apps/chat/alias, the short name for npx), always at the same version.
|
||||
#
|
||||
# The version in ui/apps/chat/package.json is the release, set by hand, as for @ggml-org/llama-ui
|
||||
# in ui-lib-publish.yml. A commit that reaches master with a version npm does not have builds the
|
||||
# app and publishes both packages; any other commit publishes nothing. Bump both package.json
|
||||
# files in the PR that should ship.
|
||||
#
|
||||
# Auth: no npm token is stored here. The job exchanges its OIDC identity for one (npm trusted
|
||||
# publishing) and signs the tarballs with provenance. One-time setup, before this can publish: on
|
||||
# each of the two packages, Publishing settings, Trusted Publisher, GitHub Actions, with
|
||||
# organization ggml-org, repository llama.cpp, workflow file ui-chat-publish.yml and Environment
|
||||
# left blank. Until then the publish step fails on master; re-running the job publishes it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: 'Build and pack, but do not publish'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-chat-publish.yml',
|
||||
'ui/apps/chat/package.json',
|
||||
'ui/apps/chat/alias/package.json'
|
||||
]
|
||||
# a dry run for PRs that change what goes into the packages
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ui-chat-publish.yml',
|
||||
'ui/apps/chat/package.json',
|
||||
'ui/apps/chat/alias/**',
|
||||
'ui/apps/chat/bin/**',
|
||||
'ui/apps/chat/cli/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
# never cancel a run mid-publish: a version half-published cannot be taken back
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
APP_DIR: ui/apps/chat
|
||||
ALIAS_DIR: ui/apps/chat/alias
|
||||
VERSION_CHECK: ui/lib/ui/scripts/npm-version-check.mjs
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish @ggml-org/llama-chat and llama-chat
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: ubuntu-26.04-arm
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Check npm can publish with an OIDC token
|
||||
run: |
|
||||
node -e '
|
||||
const v = process.argv[1];
|
||||
const [major, minor] = v.split(".").map(Number);
|
||||
if (major < 11 || (major === 11 && minor < 5)) {
|
||||
console.error(`npm ${v} cannot publish with trusted publishing: 11.5.1 or newer is needed`);
|
||||
process.exit(1);
|
||||
}
|
||||
' "$(npm -v)"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Build the app
|
||||
# the same build.json version as the UI that llama-server embeds, for the app's update check.
|
||||
# This folder's .npmrc sets ignore-scripts, so npm never builds it: this step does.
|
||||
run: LLAMA_BUILD_NUMBER="b$(git rev-list --count HEAD)" npm run build
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Test the CLI and the alias version
|
||||
run: npx vitest --project=unit --run tests/unit/cli.test.ts
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Check what goes into the package
|
||||
run: |
|
||||
npm pack --dry-run --json > "$RUNNER_TEMP/pack.json"
|
||||
node -e '
|
||||
const [{ files }] = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
|
||||
const paths = files.map((file) => file.path);
|
||||
const maps = paths.filter((path) => path.endsWith(".map"));
|
||||
if (!paths.includes("dist/index.html")) {
|
||||
console.error("dist/index.html is not in the package: nothing would be served");
|
||||
process.exit(1);
|
||||
}
|
||||
if (maps.length > 0) {
|
||||
console.error(`source maps in the package: ${maps.join(", ")}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`${paths.length} files, no source maps`);
|
||||
' "$RUNNER_TEMP/pack.json"
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Compare @ggml-org/llama-chat with npm
|
||||
id: app
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node "$VERSION_CHECK" "$APP_DIR"
|
||||
|
||||
- name: Compare llama-chat with npm
|
||||
id: alias
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node "$VERSION_CHECK" "$ALIAS_DIR"
|
||||
|
||||
# master only: npm's trusted publisher matches the repository and the workflow file, not the
|
||||
# branch, so this condition is what keeps a manual run on a PR branch from publishing.
|
||||
# The app goes first, since the alias depends on it at the same version.
|
||||
- name: Publish @ggml-org/llama-chat
|
||||
if: github.ref == 'refs/heads/master' && steps.app.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
TAG: ${{ steps.app.outputs.tag }}
|
||||
run: npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Publish llama-chat
|
||||
if: github.ref == 'refs/heads/master' && steps.alias.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
TAG: ${{ steps.alias.outputs.tag }}
|
||||
run: npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.ALIAS_DIR }}
|
||||
|
||||
- name: Report
|
||||
if: always()
|
||||
# values reach the shell through env, so a name from a PR's package.json cannot inject here
|
||||
env:
|
||||
APP: ${{ steps.app.outputs.name }}@${{ steps.app.outputs.version }}
|
||||
APP_VERDICT: ${{ steps.app.outputs.verdict || 'not reached' }}
|
||||
APP_NOTE: ${{ steps.app.outputs.note }}
|
||||
ALIAS: ${{ steps.alias.outputs.name }}@${{ steps.alias.outputs.version }}
|
||||
ALIAS_VERDICT: ${{ steps.alias.outputs.verdict || 'not reached' }}
|
||||
ALIAS_NOTE: ${{ steps.alias.outputs.note }}
|
||||
DRY: ${{ inputs.dry_run }}
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
{
|
||||
echo "### Llama Chat on npm"
|
||||
echo
|
||||
echo "| package | verdict | note |"
|
||||
echo "| --- | --- | --- |"
|
||||
echo "| \`$APP\` | \`$APP_VERDICT\` | $APP_NOTE |"
|
||||
echo "| \`$ALIAS\` | \`$ALIAS_VERDICT\` | $ALIAS_NOTE |"
|
||||
echo
|
||||
if [ "$EVENT" = "pull_request" ] || [ "$DRY" = "true" ]; then echo "Dry run: nothing was published."; fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Mind the verdict
|
||||
# a version going backwards is a mistake; an unchanged version is a package.json edit that
|
||||
# is not a release, such as a dependency update
|
||||
if: always() && github.event_name != 'pull_request'
|
||||
env:
|
||||
VERDICTS: ${{ steps.app.outputs.verdict }} ${{ steps.alias.outputs.verdict }}
|
||||
run: |
|
||||
case " $VERDICTS " in
|
||||
*" stale "*)
|
||||
echo "::error::a version in package.json is older than the one on npm"
|
||||
exit 1 ;;
|
||||
esac
|
||||
@@ -0,0 +1,201 @@
|
||||
name: UI Lib Publish
|
||||
|
||||
# Publishes @ggml-org/llama-ui, the library in ui/lib/ui, to npm.
|
||||
#
|
||||
# The version in ui/lib/ui/package.json is the release. A commit that reaches master with a
|
||||
# version npm does not have builds the library and publishes it; a commit that changes the
|
||||
# library without bumping it publishes nothing and says so on the run, which is the drift that
|
||||
# let 0.3.0 sit a day behind master. Bump the version in the PR whose change should ship, and
|
||||
# this workflow does the rest.
|
||||
#
|
||||
# Auth: no npm token is stored here. The job exchanges its OIDC identity for one (npm trusted
|
||||
# publishing) and signs the tarball with a provenance attestation. release-publish.yml has done
|
||||
# the same exchange on every push to master since it landed, its actions/attest step being the
|
||||
# proof that this repository can be issued a token.
|
||||
#
|
||||
# One-time setup, before this can publish: on the package, Publishing settings, Trusted
|
||||
# Publisher, GitHub Actions, with organization ggml-org, repository llama.cpp, workflow file
|
||||
# ui-lib-publish.yml, and Environment left blank - this job names no environment, since creating
|
||||
# one needs repository admin. Merging this workflow before that exists leaves the publish step
|
||||
# red on master: the release is late rather than wrong, and re-running the job publishes it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: 'Build and pack, but do not publish'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-lib-publish.yml',
|
||||
'ui/lib/ui/**.*'
|
||||
]
|
||||
# the advisory half, so a library change without a bump is annotated on the PR rather than
|
||||
# discovered after the merge: the path filter keeps it off PRs that touch only Llama Chat
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ui-lib-publish.yml',
|
||||
'ui/lib/ui/**.*'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
# never cancel a run mid-publish: a version half-published cannot be taken back. GitHub holds
|
||||
# one pending run per group, so a newer push replaces an older queued one - no release is lost,
|
||||
# as each run reads the version from its own commit.
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
LIB_DIR: ui/lib/ui
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish @ggml-org/llama-ui
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: ubuntu-26.04-arm
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Check npm can publish with an OIDC token
|
||||
run: |
|
||||
npm -v
|
||||
node -e '
|
||||
const v = process.argv[1];
|
||||
const [major, minor] = v.split(".").map(Number);
|
||||
if (major < 11 || (major === 11 && minor < 5)) {
|
||||
console.error(`npm ${v} cannot publish with trusted publishing: 11.5.1 or newer is needed`);
|
||||
process.exit(1);
|
||||
}
|
||||
' "$(npm -v)"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Run type checking
|
||||
run: npm run check
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Build the package and lint it
|
||||
run: npm run prepack
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Compare the version with npm
|
||||
id: version
|
||||
# on a PR this is advisory and must not fail the job: a registry outage is not the
|
||||
# contributor's mistake. On master the publish decision rests on the answer, so it fails.
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node scripts/npm-version-check.mjs
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Note a library change that ships no new version
|
||||
if: github.event_name == 'pull_request' && always() && steps.version.outcome == 'success' && steps.version.outputs.should_publish != 'true'
|
||||
env:
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
run: echo "::warning::$NOTE"
|
||||
|
||||
- name: Publish
|
||||
# master only. npm's trusted publisher matches on the repository and the workflow file,
|
||||
# not the branch, so this condition is the only thing stopping a manual run from a PR
|
||||
# branch - one that anyone with write access can start - from publishing code that no
|
||||
# one reviewed. A dispatch on master remains the way to re-run a release.
|
||||
if: github.ref == 'refs/heads/master' && steps.version.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
# --ignore-scripts publishes the tarball prepack built and publint passed rather than
|
||||
# rebuilding it here; --provenance is implied by trusted publishing and named to be
|
||||
# explicit. A package published with nothing in it cannot be taken back, so refuse first.
|
||||
env:
|
||||
TAG: ${{ steps.version.outputs.tag }}
|
||||
run: |
|
||||
test -d dist || { echo "dist/ is missing: nothing was built, so nothing is published"; exit 1; }
|
||||
npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.LIB_DIR }}
|
||||
|
||||
- name: Verify it landed
|
||||
if: github.ref == 'refs/heads/master' && steps.version.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
NAME: ${{ steps.version.outputs.name }}
|
||||
VERSION: ${{ steps.version.outputs.version }}
|
||||
run: |
|
||||
# the registry can serve a new version some time after publish returns, so retry rather
|
||||
# than call a release that worked a failure
|
||||
for try in 1 2 3 4 5 6; do
|
||||
seen=$(npm view "$NAME@$VERSION" version 2>/dev/null || true)
|
||||
if [ "$seen" = "$VERSION" ]; then
|
||||
echo "$NAME@$VERSION is on npm"
|
||||
npm view "$NAME@$VERSION" dist-tags version dist.integrity
|
||||
exit 0
|
||||
fi
|
||||
echo "not visible yet (attempt $try of 6)"
|
||||
sleep 10
|
||||
done
|
||||
echo "::error::$NAME@$VERSION published without error but is not visible on the registry after 6 attempts. The publish did not fail: check the package on npm before publishing again."
|
||||
exit 1
|
||||
|
||||
- name: Report
|
||||
if: always()
|
||||
# every value reaches the shell through env, not through the script, so a package name
|
||||
# from a PR's package.json cannot inject into this step
|
||||
env:
|
||||
NAME: ${{ steps.version.outputs.name || '@ggml-org/llama-ui' }}
|
||||
VERSION: ${{ steps.version.outputs.version || 'unknown' }}
|
||||
LATEST: ${{ steps.version.outputs.latest }}
|
||||
VERDICT: ${{ steps.version.outputs.verdict || 'not reached' }}
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
SHOULD: ${{ steps.version.outputs.should_publish }}
|
||||
DRY: ${{ inputs.dry_run }}
|
||||
run: |
|
||||
if [ "$DRY" = "true" ]; then published="no (dry run)"
|
||||
elif [ "$SHOULD" = "true" ]; then published=yes
|
||||
else published=no
|
||||
fi
|
||||
{
|
||||
echo "### $NAME $VERSION"
|
||||
echo
|
||||
echo "| | |"
|
||||
echo "| --- | --- |"
|
||||
echo "| npm latest was | ${LATEST:-nothing published yet} |"
|
||||
echo "| verdict | \`$VERDICT\` |"
|
||||
echo "| published | $published |"
|
||||
echo
|
||||
echo "> $NOTE"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Mind the verdict
|
||||
if: always() && github.event_name != 'pull_request' && steps.version.outcome == 'success'
|
||||
# a version going backwards is a mistake; a version not bumped is usually one too, but it
|
||||
# is also what a stories-only commit looks like, so that only fails the run when the
|
||||
# repository variable UI_LIB_PUBLISH_FAIL_ON_UNBUMPED is set to true
|
||||
env:
|
||||
NOTE: ${{ steps.version.outputs.note }}
|
||||
VERDICT: ${{ steps.version.outputs.verdict }}
|
||||
FAIL_ON_UNBUMPED: ${{ vars.UI_LIB_PUBLISH_FAIL_ON_UNBUMPED }}
|
||||
run: |
|
||||
case "$VERDICT" in
|
||||
stale)
|
||||
echo "::error::$NOTE"
|
||||
exit 1 ;;
|
||||
published)
|
||||
if [ "$FAIL_ON_UNBUMPED" = "true" ]; then
|
||||
echo "::error::$NOTE"
|
||||
exit 1
|
||||
fi
|
||||
echo "::warning::$NOTE" ;;
|
||||
esac
|
||||
@@ -0,0 +1,84 @@
|
||||
name: UI Storybook
|
||||
|
||||
# Builds Llama UI and Llama Chat's Storybook (ui/apps/chat) and publishes it to its Hugging Face Space
|
||||
# whenever the UI changes on master, so the Space always shows the latest build.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-storybook.yml',
|
||||
'ui/apps/chat/**',
|
||||
'ui/lib/ui/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# only the newest build matters: a newer push replaces a deploy still in progress
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
name: Deploy to Hugging Face Spaces
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: hf-jobs-cpu-upgrade:ubuntu26_04
|
||||
|
||||
env:
|
||||
HF_SPACE: ${{ vars.HF_SPACE_UI_STORYBOOK || 'ggml-org/llama-ui-storybook' }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build Storybook
|
||||
run: npm run build-storybook
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Write the Space card
|
||||
run: |
|
||||
cat > ui/apps/chat/storybook-static/README.md << EOF
|
||||
---
|
||||
title: Llama UI Storybook
|
||||
emoji: 🦙
|
||||
colorFrom: gray
|
||||
colorTo: gray
|
||||
sdk: static
|
||||
pinned: false
|
||||
---
|
||||
|
||||
The Storybook of Llama UI and Llama Chat, built from [ggml-org/llama.cpp](https://github.com/ggml-org/llama.cpp) (\`ui/apps/chat\`) at ${GITHUB_SHA}.
|
||||
EOF
|
||||
|
||||
- name: Install Hugging Face Hub CLI
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y python3 python3-venv
|
||||
python3 -m venv .venv-hf
|
||||
.venv-hf/bin/pip install -U huggingface_hub==1.28.0
|
||||
|
||||
- name: Upload to the Space
|
||||
env:
|
||||
# a token with write access to the Space; HF_TOKEN_UI_STATIC_OUTPUT if that one has it
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_UI_STORYBOOK || secrets.HF_TOKEN_UI_STATIC_OUTPUT }}
|
||||
run: |
|
||||
# --delete '*' removes files the new build no longer has; the Space's .gitattributes is always kept
|
||||
.venv-hf/bin/hf upload "$HF_SPACE" ui/apps/chat/storybook-static . \
|
||||
--repo-type space \
|
||||
--delete '*' \
|
||||
--commit-message "Storybook from ggml-org/llama.cpp@${GITHUB_SHA::10}"
|
||||
@@ -93,6 +93,11 @@ jobs:
|
||||
run: npm run lint
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Check DESIGN.md is current with the tokens
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check:design
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
@@ -132,6 +137,15 @@ jobs:
|
||||
run: npm run check
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Install Playwright browsers
|
||||
run: npx playwright install --with-deps chromium
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
# the unit tests and the Storybook stories (ui/lib/ui/tests/stories), which run as tests in Chromium
|
||||
- name: Run tests
|
||||
run: npm test
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Build the package
|
||||
run: npm run prepack
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<b>LLM inference in C/C++</b>
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v1)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v1)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=b)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
|
||||
@@ -912,6 +912,10 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context
|
||||
throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n");
|
||||
}
|
||||
|
||||
if (params.sampling.ignore_eos && !common_grammar_value(params.sampling.grammar).empty()) {
|
||||
throw std::invalid_argument("error: --ignore-eos is not supported with --grammar or --json-schema\n");
|
||||
}
|
||||
|
||||
const bool skip_model_download =
|
||||
// server will call common_params_handle_models() later, so we skip it here
|
||||
ctx_arg.ex == LLAMA_EXAMPLE_SERVER ||
|
||||
|
||||
@@ -871,7 +871,7 @@ bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key,
|
||||
return true;
|
||||
}
|
||||
|
||||
void common_chat_tts_norm_add_builtins(context & ctx, std::shared_ptr<const common_tts_dicts> dicts) {
|
||||
void common_chat_tts_norm_add_builtins(context & ctx, std::shared_ptr<const common_tts_dicts> dicts, common_tts_g2p g2p) {
|
||||
ctx.set_val("tts_norm_scan", mk_val<value_func>("tts_norm_scan", tts_norm_builtin_scan));
|
||||
ctx.set_val("tts_norm_sub", mk_val<value_func>("tts_norm_sub", tts_norm_builtin_sub));
|
||||
ctx.set_val("tts_dict", mk_val<value_func>("tts_dict", [dicts](const func_args & args) -> value {
|
||||
@@ -888,5 +888,15 @@ void common_chat_tts_norm_add_builtins(context & ctx, std::shared_ptr<const comm
|
||||
}
|
||||
return value_from_json(common_json::parse(doc), false);
|
||||
}));
|
||||
// none when the caller gives no letter-to-sound model, or the model has no pronunciation
|
||||
ctx.set_val("tts_g2p", mk_val<value_func>("tts_g2p", [g2p](const func_args & args) -> value {
|
||||
args.ensure_count(1, 1);
|
||||
args.ensure_vals<value_string>();
|
||||
const std::string out = g2p ? g2p(args.get_pos(0)->as_string().str()) : std::string();
|
||||
if (out.empty()) {
|
||||
return mk_val<value_none>();
|
||||
}
|
||||
return mk_val<value_string>(out);
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
@@ -70,6 +71,11 @@ using common_tts_dicts = std::map<std::string, common_tts_dict>;
|
||||
// binary search of key in a dictionary; on success, value is set to the JSON document
|
||||
bool common_tts_dict_find(const common_tts_dict & dict, const std::string & key, std::string & value);
|
||||
|
||||
// add tts_norm_scan(), tts_norm_sub() and tts_dict() to the context of a task template
|
||||
// pronunciation of a word by a letter-to-sound model, as phonemes separated by spaces (e.g. CMUdict notation)
|
||||
// empty when there is none; the notation is the one of the model, its "tts" template knows it
|
||||
using common_tts_g2p = std::function<std::string(const std::string & word)>;
|
||||
|
||||
// add tts_norm_scan(), tts_norm_sub(), tts_dict() and tts_g2p() to the context of a task template
|
||||
// they are not jinja global builtins on purpose: chat templates are not affected
|
||||
void common_chat_tts_norm_add_builtins(jinja::context & ctx, std::shared_ptr<const common_tts_dicts> dicts = nullptr);
|
||||
void common_chat_tts_norm_add_builtins(jinja::context & ctx, std::shared_ptr<const common_tts_dicts> dicts = nullptr,
|
||||
common_tts_g2p g2p = nullptr);
|
||||
|
||||
+6
-5
@@ -807,10 +807,11 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
|
||||
}
|
||||
|
||||
// renders a task template (e.g. "tts") with the given variables
|
||||
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars, const std::shared_ptr<common_tts_dicts> & tts_dicts) {
|
||||
static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars, const std::shared_ptr<common_tts_dicts> & tts_dicts,
|
||||
const common_tts_g2p & g2p = nullptr) {
|
||||
jinja::context ctx(tmpl.source());
|
||||
// text normalization helpers: tts_norm_scan(), tts_norm_sub() and tts_dict(), see docs/tts-norm-patterns.md
|
||||
common_chat_tts_norm_add_builtins(ctx, tts_dicts);
|
||||
// text normalization helpers: tts_norm_scan(), tts_norm_sub(), tts_dict() and tts_g2p(), see docs/tts-norm-patterns.md
|
||||
common_chat_tts_norm_add_builtins(ctx, tts_dicts, g2p);
|
||||
jinja::global_from_json(ctx, vars, false);
|
||||
ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
|
||||
ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
|
||||
@@ -886,7 +887,7 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates
|
||||
}
|
||||
|
||||
std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
|
||||
const std::string & language, const std::string & voice) {
|
||||
const std::string & language, const std::string & voice, const common_tts_g2p & g2p) {
|
||||
if (!chat_templates || !chat_templates->template_tts) {
|
||||
return "";
|
||||
}
|
||||
@@ -894,7 +895,7 @@ std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templa
|
||||
{"messages", json::array({{{"role", "user"}, {"content", text}}})},
|
||||
{"language", language},
|
||||
{"voice", voice},
|
||||
}, chat_templates->tts_dicts);
|
||||
}, chat_templates->tts_dicts, g2p);
|
||||
}
|
||||
|
||||
std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
|
||||
|
||||
+3
-1
@@ -504,8 +504,10 @@ common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates
|
||||
|
||||
// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
|
||||
// language and voice are optional, the template defines their defaults
|
||||
// g2p (optional): the letter-to-sound model of the template's tts_g2p(), e.g. mtmd_gen_audio_g2p()
|
||||
std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
|
||||
const std::string & language = "", const std::string & voice = "");
|
||||
const std::string & language = "", const std::string & voice = "",
|
||||
const std::function<std::string(const std::string & word)> & g2p = nullptr);
|
||||
|
||||
// system prompt line for the speech language and voice (e.g. LFM3-Audio), empty if not used
|
||||
std::string common_chat_get_speech_system_prompt(const common_chat_templates * chat_templates,
|
||||
|
||||
@@ -1313,6 +1313,11 @@ common_init_result::common_init_result(common_params & params, bool model_only)
|
||||
LOG_INF("%s", "decision model reads the embeddings output, enabling embedding mode\n");
|
||||
}
|
||||
|
||||
// 0 is n_batch
|
||||
if ((decision_type == COMMON_DECISION_TYPE_LFM3_D1 || decision_type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) && cparams.n_outputs_max != 0) {
|
||||
cparams.n_outputs_max = std::max(cparams.n_outputs_max, (uint32_t) COMMON_DECISION_LFM3_D1_SET_SIZE);
|
||||
}
|
||||
|
||||
if (llama_model_classifier_free_guidance(model)) {
|
||||
params.kv_unified = true;
|
||||
cparams.kv_unified = true;
|
||||
@@ -2334,6 +2339,9 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
|
||||
if (t.decision_order != 0) {
|
||||
llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order);
|
||||
}
|
||||
if (t.decision_group != 0 && !llama_batch_ext_set_decision_group(res, idx, t.decision_group)) {
|
||||
GGML_ABORT("%s: failed to set the decision group of the entry at index %d\n", __func__, i);
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
|
||||
@@ -1002,6 +1002,9 @@ enum common_decision_type {
|
||||
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
|
||||
};
|
||||
|
||||
// lfm3-d1: max questions packed in one prompt
|
||||
#define COMMON_DECISION_LFM3_D1_SET_SIZE 16
|
||||
|
||||
common_decision_type common_get_decision_type(const struct llama_model * model);
|
||||
|
||||
// metadata of a GGUF file, read without loading the model
|
||||
@@ -1117,6 +1120,7 @@ struct common_batch {
|
||||
llama_embd state; // non-owning view of the data passed to set_embd_state(), data == NULL if none
|
||||
std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
|
||||
int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
|
||||
int32_t decision_group = 0; // see llama_batch_ext_set_decision_group()
|
||||
int32_t embd_row = -1; // see add_embd_row(), -1 if none
|
||||
int32_t state_row = -1; // see set_embd_state_row(), -1 if none
|
||||
};
|
||||
|
||||
@@ -203,6 +203,13 @@ struct common_sampler * common_sampler_init(
|
||||
throw std::invalid_argument("penalty_present must be finite");
|
||||
}
|
||||
const llama_vocab * vocab = llama_model_get_vocab(model);
|
||||
if (!common_grammar_value(params.grammar).empty()) {
|
||||
for (const auto & b : params.logit_bias) {
|
||||
if (b.bias == -INFINITY && llama_vocab_is_eog(vocab, b.token)) {
|
||||
throw std::invalid_argument("EOG tokens cannot be banned when a grammar is set");
|
||||
}
|
||||
}
|
||||
}
|
||||
llama_sampler_chain_params lparams = llama_sampler_chain_default_params();
|
||||
|
||||
lparams.no_perf = params.no_perf;
|
||||
|
||||
@@ -283,8 +283,11 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"Qwen3NextForCausalLM": "qwen",
|
||||
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
|
||||
"PocketTTSModel": "pockettts",
|
||||
"KokoroModel": "kokoro",
|
||||
"TinyVocaModel": "tiny_voca",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"VoxCPM2Model": "voxcpm",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
@@ -338,6 +341,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"XverseForCausalLM": "xverse",
|
||||
"YoutuForCausalLM": "deepseek",
|
||||
"YoutuVLForConditionalGeneration": "deepseek",
|
||||
"ZayaForCausalLM": "zaya1",
|
||||
"fish_qwen3_omni": "fish_s2",
|
||||
"modeling_grove_moe.GroveMoeForCausalLM": "grovemoe",
|
||||
"modeling_sarvam_moe.SarvamMoEForCausalLM": "bailingmoe",
|
||||
@@ -364,6 +368,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"FishOmniModel": "drama3",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"VoxCPM2Model": "voxcpm",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Gemma3ForConditionalGeneration": "gemma",
|
||||
"Gemma3nForConditionalGeneration": "gemma",
|
||||
@@ -423,6 +428,8 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"Qwen3ASRForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3OmniMoeForConditionalGeneration": "qwen3vl",
|
||||
"PocketTTSModel": "pockettts",
|
||||
"KokoroModel": "kokoro",
|
||||
"TinyVocaModel": "tiny_voca",
|
||||
"SopranoModel": "soprano",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
|
||||
@@ -0,0 +1,419 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable
|
||||
|
||||
import torch
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, MmprojModel, gguf, logger
|
||||
|
||||
# Kokoro (StyleTTS2 with an iSTFTNet decoder), https://huggingface.co/hexgrad/Kokoro-82M
|
||||
# text model: the PL-BERT phoneme encoder (ALBERT), one token per phoneme character
|
||||
# mmproj: bert_encoder, prosody predictor, text encoder, decoder and the voices
|
||||
# the text to phonemes step is the "tts" chat template, its dictionaries come from the misaki and nltk python packages
|
||||
|
||||
DEFAULT_VOICE = "af_heart"
|
||||
|
||||
MODULE_PREFIX = {
|
||||
"bert_encoder": "a.gen.bert_enc.",
|
||||
"predictor": "a.gen.pred.",
|
||||
"text_encoder": "a.gen.text.",
|
||||
"decoder": "a.gen.dec.",
|
||||
}
|
||||
|
||||
|
||||
def _checkpoint(dir_model: Path) -> Path | None:
|
||||
paths = sorted(dir_model.glob("kokoro-*.pth"))
|
||||
return paths[0] if paths else None
|
||||
|
||||
|
||||
def _is_kokoro_checkpoint(dir_model: Path) -> bool:
|
||||
if _checkpoint(dir_model) is None or not (dir_model / "config.json").is_file():
|
||||
return False
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
return "plbert" in config and "istftnet" in config and "vocab" in config
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_kokoro_checkpoint)
|
||||
def _load_kokoro_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected Kokoro checkpoint")
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
plbert = config["plbert"]
|
||||
return {
|
||||
"architectures": ["KokoroModel"],
|
||||
"model_type": "kokoro",
|
||||
"num_hidden_layers": plbert["num_hidden_layers"],
|
||||
"hidden_size": plbert["hidden_size"],
|
||||
"intermediate_size": plbert["intermediate_size"],
|
||||
"num_attention_heads": plbert["num_attention_heads"],
|
||||
"max_position_embeddings": plbert["max_position_embeddings"],
|
||||
"vocab_size": config["n_token"],
|
||||
"layer_norm_eps": 1e-12, # AlbertConfig default
|
||||
"kokoro": config,
|
||||
# the mmproj tensors are named explicitly, they do not go through the block tensor map
|
||||
"audio_config": {"num_hidden_layers": 0},
|
||||
}
|
||||
|
||||
|
||||
def _load_modules(dir_model: Path, modules: Iterable[str]) -> dict[str, dict[str, Tensor]]:
|
||||
path = _checkpoint(dir_model)
|
||||
assert path is not None
|
||||
state = torch.load(path, map_location="cpu", mmap=True, weights_only=True)
|
||||
# every module was saved from a DataParallel wrapper
|
||||
return {m: {k.removeprefix("module."): v for k, v in state[m].items()} for m in modules}
|
||||
|
||||
|
||||
def _g2p_fallback(lexicons: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
# model for the words missing from the lexicon: n-grams over units "letter:sounds", learned from the lexicon
|
||||
# key "<0 to 4 units before> <letter>" -> {sounds: probability}; "<s>" pads the start of a word, the letter "$" is its end
|
||||
# the "tts" template decodes it with a beam search
|
||||
order, min_count, stress = 5, 3, "\u02c8\u02cc"
|
||||
vowels = set("AIOQWYaiu\u00e6\u0251\u0252\u0254\u0259\u025b\u025c\u026a\u028a\u028c\u1d7b\u1d4a")
|
||||
|
||||
def sounds(ps: str) -> list[str]:
|
||||
# a stress mark goes with the sound after it, a length mark with the one before
|
||||
out: list[str] = []
|
||||
pre = ""
|
||||
for c in ps:
|
||||
if c in stress:
|
||||
pre += c
|
||||
elif c == "\u02d0" and out:
|
||||
out[-1] += c
|
||||
else:
|
||||
out.append(pre + c)
|
||||
pre = ""
|
||||
return out
|
||||
|
||||
def penalty(letter: str, chunk: str) -> float:
|
||||
# a consonant letter says consonants only; a vowel letter says a vowel, with at most a glide
|
||||
flat = [c for c in chunk if c not in stress and c != "\u02d0"]
|
||||
n_vowel = sum(c in vowels for c in flat)
|
||||
if letter in "bcdfgjkmnpqstvxz" and n_vowel > 0:
|
||||
return -8.0
|
||||
if letter in "aeiou" and flat and (n_vowel == 0 or any(c not in vowels and c not in "jw" for c in flat)):
|
||||
return -8.0
|
||||
return 0.0
|
||||
|
||||
words: dict[str, list[str]] = {}
|
||||
for lexicon in lexicons:
|
||||
for word, v in lexicon.items():
|
||||
ps = v["DEFAULT"] if isinstance(v, dict) else v
|
||||
# same spelling normalization as the template: doubled consonants and "ck" say one sound
|
||||
w = re.sub(r"([b-df-hj-np-tv-z])\1", r"\1", word.lower()).replace("ck", "k")
|
||||
if ps and " " not in ps and len(w) >= 2 and re.fullmatch(r"[a-z']+", w) and w not in words:
|
||||
t = sounds(ps)
|
||||
if len(t) <= 2 * len(w):
|
||||
words[w] = t
|
||||
|
||||
def align(w: str, t: list[str], cost: dict[str, dict[str, float]]) -> list[str] | None:
|
||||
# best split of the sounds over the letters, each letter takes 0, 1 or 2 of them
|
||||
n_l, n_t = len(w), len(t)
|
||||
two = [t[j] + t[j + 1] for j in range(n_t - 1)]
|
||||
best = [[-1e18] * (n_t + 1) for _ in range(n_l + 1)]
|
||||
back = [[0] * (n_t + 1) for _ in range(n_l + 1)]
|
||||
best[0][0] = 0.0
|
||||
for i in range(n_l):
|
||||
letter = w[i]
|
||||
row = cost.setdefault(letter, {})
|
||||
cur, nxt, bk = best[i], best[i + 1], back[i + 1]
|
||||
# the sounds left must fit in the letters left
|
||||
for j in range(max(0, n_t - 2 * (n_l - i)), n_t + 1):
|
||||
base = cur[j]
|
||||
if base < -1e17:
|
||||
continue
|
||||
for k, chunk in ((0, ""), (1, t[j] if j < n_t else None), (2, two[j] if j + 1 < n_t else None)):
|
||||
if chunk is None:
|
||||
break
|
||||
c = row.get(chunk)
|
||||
if c is None:
|
||||
# not seen yet for this letter
|
||||
c = row[chunk] = -12.0 + penalty(letter, chunk)
|
||||
if base + c > nxt[j + k]:
|
||||
nxt[j + k] = base + c
|
||||
bk[j + k] = k
|
||||
if best[n_l][n_t] < -1e17:
|
||||
return None
|
||||
out: list[str] = []
|
||||
j = n_t
|
||||
for i in range(n_l, 0, -1):
|
||||
k = back[i][j]
|
||||
out.append("".join(t[j - k:j]))
|
||||
j -= k
|
||||
return out[::-1]
|
||||
|
||||
# hard EM: align with the current letter -> sounds costs, count, repeat
|
||||
# the costs are learned on a sample of the words, then every word is aligned once
|
||||
def step(items: Iterable[tuple[str, list[str]]], cost: dict[str, dict[str, float]]) -> list[tuple[str, list[str]]]:
|
||||
return [(w, a) for w, a in ((w, align(w, t, cost)) for w, t in items) if a is not None]
|
||||
|
||||
sample = list(words.items())[::max(1, len(words) // 20000)]
|
||||
cost: dict[str, dict[str, float]] = {}
|
||||
for _ in range(5):
|
||||
counts: dict[str, Counter[str]] = defaultdict(Counter)
|
||||
for w, a in step(sample, cost):
|
||||
for c, o in zip(w, a):
|
||||
counts[c][o] += 1
|
||||
# small cost for a silent letter, so that a sound stays on its own letter
|
||||
cost = {c: {o: math.log(n / sum(cnt.values())) - (0.5 if o == "" else 0.0) + penalty(c, o) for o, n in cnt.items()}
|
||||
for c, cnt in counts.items()}
|
||||
aligned = step(words.items(), cost)
|
||||
|
||||
# counts of a unit after each of its contexts, from 4 units down to none
|
||||
ngram: dict[str, Counter[str]] = defaultdict(Counter)
|
||||
n_ctx: Counter[str] = Counter()
|
||||
for w, a in aligned:
|
||||
units = ["<s>"] * (order - 1) + [c + ":" + o for c, o in zip(w, a)] + ["$:"]
|
||||
for i in range(order - 1, len(units)):
|
||||
letter, _, o = units[i].partition(":")
|
||||
for n in range(order):
|
||||
ctx = " ".join(units[i - n:i])
|
||||
ngram[(ctx + " " if ctx else "") + letter][o] += 1
|
||||
n_ctx[ctx] += 1
|
||||
|
||||
# the probability is over all the units after the context, not only the ones of this letter
|
||||
# this tells the search how likely the letter is after the sounds it chose, so it can drop a bad choice
|
||||
model: dict[str, Any] = {}
|
||||
for key, cnt in ngram.items():
|
||||
ctx, _, letter = key.rpartition(" ")
|
||||
# the two shortest contexts keep all the readings that the letter has min_count times or more
|
||||
kept = {o: float(f"{n / n_ctx[ctx]:.4g}") for o, n in cnt.most_common()
|
||||
if n >= min_count or (key.count(" ") < 2 and ngram[letter][o] >= min_count)}
|
||||
if kept:
|
||||
model[key] = kept
|
||||
return model
|
||||
|
||||
|
||||
def _g2p_dicts() -> dict[str, dict[str, Any]] | None:
|
||||
# dictionaries of the "tts" template: the misaki lexicon and the nltk averaged perceptron tagger
|
||||
spec = importlib.util.find_spec("misaki")
|
||||
if spec is None or spec.origin is None:
|
||||
logger.warning("misaki is not installed, the model will take phonemes as prompt (pip install misaki nltk)")
|
||||
return None
|
||||
data = Path(spec.origin).parent / "data"
|
||||
try:
|
||||
nltk = importlib.import_module("nltk")
|
||||
base = Path(str(nltk.data.find("taggers/averaged_perceptron_tagger_eng"))) / "averaged_perceptron_tagger_eng"
|
||||
except (ImportError, LookupError):
|
||||
logger.warning("the nltk tagger is not available, the model will take phonemes as prompt "
|
||||
"(pip install nltk && python -m nltk.downloader averaged_perceptron_tagger_eng)")
|
||||
return None
|
||||
|
||||
def load(path: Path) -> Any:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
# tagger: classes sorted by label, feature -> {"c<class index>": weight}
|
||||
# (index keys: jinja2's map(attribute=...) splits on "." and some labels are punctuation)
|
||||
classes = sorted(load(Path(f"{base}.classes.json")))
|
||||
cid = {c: f"c{i}" for i, c in enumerate(classes)}
|
||||
weights = load(Path(f"{base}.weights.json"))
|
||||
lex = {
|
||||
"gold": load(data / "us_gold.json"),
|
||||
"silver": load(data / "us_silver.json"),
|
||||
"gold_gb": load(data / "gb_gold.json"),
|
||||
"silver_gb": load(data / "gb_silver.json"),
|
||||
}
|
||||
logger.info("gguf: learning the pronunciation of unknown words from the lexicon, this takes a few minutes")
|
||||
# one process per variant
|
||||
with ProcessPoolExecutor(max_workers=2) as pool:
|
||||
lts, lts_gb = pool.map(_g2p_fallback, [[lex["gold"], lex["silver"]], [lex["gold_gb"], lex["silver_gb"]]])
|
||||
return {
|
||||
**lex,
|
||||
"lts": lts,
|
||||
"lts_gb": lts_gb,
|
||||
"pos_weights": {f: {cid[t]: w for t, w in tw.items()} for f, tw in weights.items()},
|
||||
"pos_tagdict": load(Path(f"{base}.tagdict.json")),
|
||||
"pos_classes": {"": classes},
|
||||
}
|
||||
|
||||
|
||||
@ModelBase.register("KokoroModel")
|
||||
@ModelBase.example("hexgrad/Kokoro-82M")
|
||||
class KokoroTextModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.KOKORO
|
||||
|
||||
_LAYER = "encoder.albert_layer_groups.0.albert_layers.0."
|
||||
_TENSOR_MAP = {
|
||||
"embeddings.LayerNorm": gguf.MODEL_TENSOR.TOKEN_EMBD_NORM,
|
||||
"encoder.embedding_hidden_mapping_in": gguf.MODEL_TENSOR.TOKEN_EMBD_PROJ,
|
||||
}
|
||||
# the layers share their weights, stored as block 0
|
||||
_LAYER_TENSOR_MAP = {
|
||||
"attention.query": gguf.MODEL_TENSOR.ATTN_Q,
|
||||
"attention.key": gguf.MODEL_TENSOR.ATTN_K,
|
||||
"attention.value": gguf.MODEL_TENSOR.ATTN_V,
|
||||
"attention.dense": gguf.MODEL_TENSOR.ATTN_OUT,
|
||||
"attention.LayerNorm": gguf.MODEL_TENSOR.ATTN_OUT_NORM,
|
||||
"ffn": gguf.MODEL_TENSOR.FFN_UP,
|
||||
"ffn_output": gguf.MODEL_TENSOR.FFN_DOWN,
|
||||
"full_layer_layer_norm": gguf.MODEL_TENSOR.LAYER_OUT_NORM,
|
||||
}
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
del remote_hf_model_id
|
||||
bert = _load_modules(self.dir_model, ["bert"])["bert"]
|
||||
return {name: (lambda t=t: t) for name, t in bert.items()}
|
||||
|
||||
def set_vocab(self):
|
||||
config = self.hparams["kokoro"]
|
||||
n_vocab = config["n_token"]
|
||||
tokens: list[str] = [f"[UNUSED_{i}]" for i in range(n_vocab)]
|
||||
toktypes = [gguf.TokenType.UNUSED] * n_vocab
|
||||
for text, i in config["vocab"].items():
|
||||
tokens[i] = text
|
||||
toktypes[i] = gguf.TokenType.NORMAL
|
||||
# id 0 pads both ends of the phoneme sequence
|
||||
tokens[0] = "<pad>"
|
||||
toktypes[0] = gguf.TokenType.CONTROL
|
||||
|
||||
# the phonemes are mapped to tokens one character at a time by the audio pipeline
|
||||
self.gguf_writer.add_tokenizer_model("llama")
|
||||
self.gguf_writer.add_tokenizer_pre("default")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores([0.0] * n_vocab)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
self.gguf_writer.add_bos_token_id(0)
|
||||
self.gguf_writer.add_eos_token_id(0)
|
||||
self.gguf_writer.add_pad_token_id(0)
|
||||
self.gguf_writer.add_add_bos_token(False)
|
||||
self.gguf_writer.add_add_eos_token(False)
|
||||
|
||||
dicts = _g2p_dicts()
|
||||
if dicts is not None:
|
||||
template_path = Path(__file__).parent.parent / "models" / "templates" / "hexgrad-Kokoro-82M-tts.jinja"
|
||||
with open(template_path, encoding="utf-8") as f:
|
||||
self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}])
|
||||
self.gguf_writer.add_tts_dicts(dicts)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
self.gguf_writer.add_causal_attention(False)
|
||||
self.gguf_writer.add_pooling_type(gguf.PoolingType.NONE)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
del bid
|
||||
n_embd = self.hparams["hidden_size"]
|
||||
|
||||
if name == "embeddings.word_embeddings.weight":
|
||||
# zero-pad the rows to n_embd, llama.cpp wants token and embedding inputs of the same width
|
||||
data_torch = torch.nn.functional.pad(data_torch, (0, n_embd - data_torch.shape[1]))
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), data_torch)
|
||||
return
|
||||
if name == "embeddings.position_embeddings.weight":
|
||||
# every token has type 0, fold its embedding into the positions
|
||||
type0 = self.model_tensors["embeddings.token_type_embeddings.weight"]()[0]
|
||||
yield (self.format_tensor_name(gguf.MODEL_TENSOR.POS_EMBD), data_torch + type0)
|
||||
return
|
||||
if name.startswith(("embeddings.token_type_embeddings.", "pooler.")):
|
||||
return
|
||||
|
||||
key, suffix = name.rsplit(".", 1)
|
||||
if key in self._TENSOR_MAP:
|
||||
yield (self.format_tensor_name(self._TENSOR_MAP[key], suffix="." + suffix), data_torch)
|
||||
return
|
||||
if key.startswith(self._LAYER):
|
||||
tensor = self._LAYER_TENSOR_MAP[key.removeprefix(self._LAYER)]
|
||||
yield (self.format_tensor_name(tensor, 0, suffix="." + suffix), data_torch)
|
||||
return
|
||||
raise ValueError(f"unexpected tensor {name}")
|
||||
|
||||
|
||||
@ModelBase.register("KokoroModel")
|
||||
@ModelBase.example("hexgrad/Kokoro-82M")
|
||||
class KokoroMmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._wnorm_g: dict[str, Tensor] = {}
|
||||
self._wnorm_v: dict[str, Tensor] = {}
|
||||
self._lstm_bias: dict[str, Tensor] = {}
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
return self.global_config.get("audio_config")
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
del remote_hf_model_id
|
||||
tensors: dict[str, Callable[[], Tensor]] = {}
|
||||
for module, state in _load_modules(self.dir_model, MODULE_PREFIX.keys()).items():
|
||||
for name, t in state.items():
|
||||
tensors[MODULE_PREFIX[module] + name] = (lambda t=t: t)
|
||||
return tensors
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
# the names are final already
|
||||
return item
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
config = self.global_config["kokoro"]
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.KOKORO)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(config["hidden_dim"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(config["hidden_dim"])
|
||||
self.gguf_writer.add_gen_audio_block_count(config["n_layer"])
|
||||
self.gguf_writer.add_gen_audio_head_count(1)
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5)
|
||||
if (self.dir_model / "voices" / f"{DEFAULT_VOICE}.pt").is_file():
|
||||
self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
del bid
|
||||
# torch weight_norm: weight = g * v / |v| over dims 1..n
|
||||
if name.endswith((".weight_g", ".weight_v")):
|
||||
base = name[:-len(".weight_g")]
|
||||
(self._wnorm_g if name.endswith("_g") else self._wnorm_v)[base] = data_torch
|
||||
if base in self._wnorm_g and base in self._wnorm_v:
|
||||
g = self._wnorm_g.pop(base).float()
|
||||
v = self._wnorm_v.pop(base).float()
|
||||
w = g * v / v.norm(dim=tuple(range(1, v.dim())), keepdim=True)
|
||||
if base.endswith(".pool"):
|
||||
# depthwise transposed conv [C, 1, 3], stored as one row per tap
|
||||
w = w.squeeze(1).T.contiguous()
|
||||
yield (base + ".weight", w)
|
||||
return
|
||||
# the two LSTM biases are always added
|
||||
if ".bias_ih_l0" in name or ".bias_hh_l0" in name:
|
||||
base = name.replace(".bias_ih_l0", ".bias_l0").replace(".bias_hh_l0", ".bias_l0")
|
||||
if base in self._lstm_bias:
|
||||
yield (base, self._lstm_bias.pop(base).float() + data_torch.float())
|
||||
else:
|
||||
self._lstm_bias[base] = data_torch
|
||||
return
|
||||
# snake alphas are per channel
|
||||
if ".alpha1." in name or ".alpha2." in name:
|
||||
data_torch = data_torch.reshape(-1)
|
||||
yield (name, data_torch)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
del name, bid
|
||||
# tensors read on the host stay F32
|
||||
if "lstm" in new_name or ".shared." in new_name or new_name.startswith(("a.gen.bert_enc.", "voice.")) \
|
||||
or ".duration_proj." in new_name or ".m_source." in new_name or ".pool." in new_name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# CONV_TRANSPOSE_1D takes only F32 weights on some backends
|
||||
if ".ups." in new_name:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# the conv ops have no BF16 kernels
|
||||
if n_dims >= 2:
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# a voice is a table of style vectors, the row is picked by the number of phonemes
|
||||
for path in sorted((self.dir_model / "voices").glob("*.pt")):
|
||||
pack = torch.load(path, map_location="cpu", weights_only=True)
|
||||
yield (f"voice.{path.stem}", pack.squeeze(1).float())
|
||||
+72
-2
@@ -275,6 +275,14 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
|
||||
or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight")))
|
||||
):
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
# the reference encoder's convs run with an F32 im2col, see f32_im2col in pockettts-seanet.cpp
|
||||
if new_name.startswith("a.seanet.") or new_name == "a.downsample.conv.weight":
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# codes are picked by the nearest codebook entry: Q8_0 changes about 8% of them, F16 none
|
||||
if new_name.startswith("a.gen.enc.tfm.blk.") and new_name.endswith(".weight") and n_dims == 2:
|
||||
return gguf.GGMLQuantizationType.F16
|
||||
if new_name.startswith("a.gen.enc.quant."):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
# ConvTranspose1d kernels: only F16/F32 are implemented, no BF16
|
||||
if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
@@ -298,8 +306,8 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
|
||||
return super().filter_tensors((name, gen))
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# code2wav tensors are already named by generate_extra_tensors(), pass them through
|
||||
if name.startswith("a.gen.wav."):
|
||||
# code2wav and reference encoder tensors are already named by generate_extra_tensors(), pass them through
|
||||
if name.startswith(("a.gen.wav.", "a.gen.enc.", "a.seanet.", "a.downsample.")):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
@@ -356,6 +364,68 @@ class Qwen3TTSSpeakerEncoderModel(MmprojModel):
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
yield from self._generate_code2wav_tensors()
|
||||
yield from self._generate_ref_encoder_tensors()
|
||||
|
||||
# mimi encoder of the speech tokenizer: reference clip -> codes, for voice cloning with a transcript (ICL)
|
||||
def _generate_ref_encoder_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
from safetensors.torch import load_file
|
||||
|
||||
state_dict = load_file(self.dir_model / "speech_tokenizer" / "model.safetensors")
|
||||
if "encoder.downsample.conv.weight" not in state_dict:
|
||||
return
|
||||
|
||||
with open(self.dir_model / "speech_tokenizer" / "config.json", "r", encoding="utf-8") as f:
|
||||
enc_config = json.load(f)["encoder_config"]
|
||||
n_stage = len(enc_config["upsampling_ratios"])
|
||||
|
||||
T = gguf.MODEL_TENSOR
|
||||
|
||||
# encoder.encoder.layers: conv_in, then per stage a residual unit, an ELU and a strided conv, then an ELU and conv_out
|
||||
def seanet(idx: int, suffix: str) -> Tensor:
|
||||
return state_dict[f"encoder.encoder.layers.{idx}.{suffix}"]
|
||||
|
||||
for suffix in ("weight", "bias"):
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_CONV_IN, suffix=f".{suffix}"), seanet(0, f"conv.{suffix}"))
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_CONV_OUT, suffix=f".{suffix}"), seanet(3 * n_stage + 2, f"conv.{suffix}"))
|
||||
for stage in range(n_stage):
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_RES_CONV1, stage, suffix=f".{suffix}"), seanet(3 * stage + 1, f"block.1.conv.{suffix}"))
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_RES_CONV2, stage, suffix=f".{suffix}"), seanet(3 * stage + 1, f"block.3.conv.{suffix}"))
|
||||
yield (self.format_tensor_name(T.A_ENC_SEANET_SCALE_CONV, stage, suffix=f".{suffix}"), seanet(3 * stage + 3, f"conv.{suffix}"))
|
||||
|
||||
yield (self.format_tensor_name(T.A_ENC_DOWNSAMPLE_CONV), state_dict["encoder.downsample.conv.weight"])
|
||||
|
||||
tfm_layer_map = {
|
||||
"input_layernorm.weight": (T.A_GEN_ENC_TFM_ATTN_NORM, ".weight"),
|
||||
"input_layernorm.bias": (T.A_GEN_ENC_TFM_ATTN_NORM, ".bias"),
|
||||
"self_attn.q_proj.weight": (T.A_GEN_ENC_TFM_ATTN_Q, ".weight"),
|
||||
"self_attn.k_proj.weight": (T.A_GEN_ENC_TFM_ATTN_K, ".weight"),
|
||||
"self_attn.v_proj.weight": (T.A_GEN_ENC_TFM_ATTN_V, ".weight"),
|
||||
"self_attn.o_proj.weight": (T.A_GEN_ENC_TFM_ATTN_OUT, ".weight"),
|
||||
"self_attn_layer_scale.scale": (T.A_GEN_ENC_TFM_ATTN_SCALE, ".weight"),
|
||||
"post_attention_layernorm.weight": (T.A_GEN_ENC_TFM_FFN_NORM, ".weight"),
|
||||
"post_attention_layernorm.bias": (T.A_GEN_ENC_TFM_FFN_NORM, ".bias"),
|
||||
"mlp.fc1.weight": (T.A_GEN_ENC_TFM_FFN_UP, ".weight"),
|
||||
"mlp.fc2.weight": (T.A_GEN_ENC_TFM_FFN_DOWN, ".weight"),
|
||||
"mlp_layer_scale.scale": (T.A_GEN_ENC_TFM_FFN_SCALE, ".weight"),
|
||||
}
|
||||
for bid in range(enc_config["num_hidden_layers"]):
|
||||
for key, (tensor_id, suffix) in tfm_layer_map.items():
|
||||
yield (self.format_tensor_name(tensor_id, bid, suffix=suffix), state_dict[f"encoder.encoder_transformer.layers.{bid}.{key}"])
|
||||
|
||||
def rvq_codebook(prefix: str, idx: int) -> Tensor:
|
||||
# checkpoint has EMA accumulators, so codebook = embed_sum / cluster_usage
|
||||
embed_sum = state_dict[f"{prefix}.layers.{idx}.codebook.embed_sum"]
|
||||
cluster_usage = state_dict[f"{prefix}.layers.{idx}.codebook.cluster_usage"]
|
||||
return embed_sum / cluster_usage.clamp_min(1e-5).unsqueeze(-1)
|
||||
|
||||
# the tokenizer keeps the semantic codebook and the first acoustic ones, see encoder_valid_num_quantizers
|
||||
semantic = "encoder.quantizer.semantic_residual_vector_quantizer"
|
||||
acoustic = "encoder.quantizer.acoustic_residual_vector_quantizer"
|
||||
yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_FIRST_IN), state_dict[f"{semantic}.input_proj.weight"].squeeze(-1))
|
||||
yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_FIRST_CB), rvq_codebook(semantic, 0))
|
||||
yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_REST_IN), state_dict[f"{acoustic}.input_proj.weight"].squeeze(-1))
|
||||
yield (self.format_tensor_name(T.A_GEN_ENC_QUANT_REST_CB),
|
||||
torch.stack([rvq_codebook(acoustic, i) for i in range(self._CODE_GEN_N_CODEBOOKS)], dim=0))
|
||||
|
||||
def _generate_code2wav_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# code2wav weights live in speech_tokenizer/model.safetensors, not the main safetensors
|
||||
|
||||
@@ -0,0 +1,443 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, MmprojModel, gguf, logger
|
||||
|
||||
# tiny-voca, https://huggingface.co/n4ze3m/tiny-voca
|
||||
#
|
||||
# text model: phoneme encoder (4 blocks), the input rows are built by the audio pipeline (token + stress + voice)
|
||||
# mmproj: input tables, voices, duration predictor, prosody and acoustic stacks, source-filter heads
|
||||
#
|
||||
# the text to phonemes step (dizione) is the "tts" chat template, see models/templates/n4ze3m-tiny-voca-tts.jinja
|
||||
# its lexicon comes from the phonemizer/ directory of the model, the other tables from the dizione python package
|
||||
#
|
||||
# the prosody and acoustic stacks have the same blocks as the encoder, they are stored with the libllama names
|
||||
# of the tiny-voca arch under a.gen.prosody. and a.gen.acoustic., mtmd runs them on llama contexts of its own
|
||||
|
||||
CHECKPOINT = "tiny-voca.pt"
|
||||
DEFAULT_VOICE = "M2"
|
||||
|
||||
# phone classes of the reference (text.py): the phones get at least one frame, the vowels are always voiced
|
||||
VOWELS = ("ə", "ɐ", "ɚ", "ᵻ", "ɪ", "i", "iː", "ɛ", "æ", "ʌ", "ɑː", "ɔ", "ɔː", "ʊ", "uː", "ɜː",
|
||||
"eɪ", "aɪ", "ɔɪ", "oʊ", "aʊ", "iə", "oː",
|
||||
"ɑːɹ", "ɔːɹ", "oːɹ", "ɪɹ", "ɛɹ", "ʊɹ", "aɪɚ", "aɪə", "əl", "n̩")
|
||||
CONSONANTS = ("p", "b", "t", "d", "k", "ɡ", "f", "v", "θ", "ð", "s", "z", "ʃ", "ʒ", "h",
|
||||
"m", "n", "ŋ", "l", "ɹ", "j", "w", "tʃ", "dʒ", "ɾ", "ʔ")
|
||||
|
||||
# input letters and output phonemes of the unknown-word model, fixed in mtmd (see mtmd_graph_tiny_voca)
|
||||
G2P_LETTERS = "abcdefghijklmnopqrstuvwxyz'"
|
||||
G2P_SYMBOLS = [v + d for v in ("AA", "AE", "AH", "AO", "AW", "AY") for d in "012"] + ["B", "CH", "D", "DH"] + \
|
||||
[v + d for v in ("EH", "ER", "EY") for d in "012"] + ["F", "G", "HH"] + [v + d for v in ("IH", "IY") for d in "012"] + \
|
||||
["JH", "K", "L", "M", "N", "NG"] + [v + d for v in ("OW", "OY") for d in "012"] + ["P", "R", "S", "SH", "T", "TH"] + \
|
||||
[v + d for v in ("UH", "UW") for d in "012"] + ["V", "W", "Y", "Z", "ZH"]
|
||||
|
||||
# block tensors: reference name -> libllama name
|
||||
BLOCK_MAP = {
|
||||
"norm1": "attn_norm",
|
||||
"attention.qkv": "attn_qkv",
|
||||
"attention.out": "attn_output",
|
||||
"norm2": "ffn_norm",
|
||||
"depthwise": "ssm_conv1d",
|
||||
"expand": "ffn_up",
|
||||
"project": "ffn_down",
|
||||
}
|
||||
|
||||
# mmproj stacks: reference module -> prefix
|
||||
STACK_PREFIX = {
|
||||
"prosody_stack": "a.gen.prosody.",
|
||||
"acoustic_stack": "a.gen.acoustic.",
|
||||
}
|
||||
|
||||
# other mmproj tensors: reference module -> name
|
||||
MMPROJ_MAP = {
|
||||
"tokens": "a.gen.tok_embd",
|
||||
"stress": "a.gen.stress_embd",
|
||||
"duration.conv1": "a.gen.dur.conv1",
|
||||
"duration.conv2": "a.gen.dur.conv2",
|
||||
"duration.norm1": "a.gen.dur.norm1",
|
||||
"duration.norm2": "a.gen.dur.norm2",
|
||||
"duration.out": "a.gen.dur.out",
|
||||
"frame_position": "a.gen.frame_pos",
|
||||
"prosody_head": "a.gen.prosody_head",
|
||||
"prosody_projection": "a.gen.prosody_proj",
|
||||
"vocoder.harmonic": "a.gen.voc.harmonic",
|
||||
"vocoder.noise": "a.gen.voc.noise",
|
||||
}
|
||||
|
||||
|
||||
def _is_tiny_voca_checkpoint(dir_model: Path) -> bool:
|
||||
if not (dir_model / CHECKPOINT).is_file() or not (dir_model / "config.json").is_file():
|
||||
return False
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
return config.get("phonemizer", {}).get("name") == "dizione" and "vocab" in config and "model" in config
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_tiny_voca_checkpoint)
|
||||
def _load_tiny_voca_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected tiny-voca checkpoint")
|
||||
with open(dir_model / "config.json", encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
c = config["model"]
|
||||
return {
|
||||
"architectures": ["TinyVocaModel"],
|
||||
"model_type": "tiny-voca",
|
||||
"num_hidden_layers": c["encoder_layers"],
|
||||
"hidden_size": c["channels"],
|
||||
"intermediate_size": c["feedforward"],
|
||||
"num_attention_heads": c["heads"],
|
||||
"num_key_value_heads": c["heads"],
|
||||
"max_position_embeddings": c["max_tokens"],
|
||||
"vocab_size": c["vocab_size"],
|
||||
"layer_norm_eps": 1e-5, # torch LayerNorm default
|
||||
"rope_theta": 10000.0,
|
||||
"tiny_voca": config,
|
||||
# the mmproj tensors are named explicitly, they do not go through the block tensor map
|
||||
"audio_config": {"num_hidden_layers": 0},
|
||||
}
|
||||
|
||||
|
||||
def _table_rows(path: Path) -> Iterable[list[str]]:
|
||||
# dizione tables: columns separated by tabs or two or more spaces, "#" comments
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.split("#", 1)[0].rstrip() if line.lstrip().startswith("#") else line.rstrip()
|
||||
if line.strip():
|
||||
yield [c.strip() for c in re.split(r"\t+| {2,}", line.strip())]
|
||||
|
||||
|
||||
def _g2p_dicts(dir_model: Path) -> dict[str, dict[str, Any]] | None:
|
||||
lexicon_path = dir_model / "phonemizer" / "lexicon.tsv"
|
||||
spec = importlib.util.find_spec("dizione")
|
||||
if not lexicon_path.is_file() or spec is None or spec.origin is None:
|
||||
logger.warning("dizione or phonemizer/lexicon.tsv not found, the model will take phonemes as prompt "
|
||||
"(pip install git+https://github.com/n4ze3m/dizione)")
|
||||
return None
|
||||
data = Path(spec.origin).parent / "data"
|
||||
lexicon: dict[str, str] = {}
|
||||
for line in lexicon_path.read_text(encoding="utf-8").splitlines():
|
||||
if "\t" in line:
|
||||
word, phones = line.split("\t", 1)
|
||||
lexicon[word] = phones.strip()
|
||||
|
||||
# word -> [stress class, form, final form or ""], see function_words.txt
|
||||
functions = {r[0]: [r[1], r[2], r[3] if len(r) > 3 else ""] for r in _table_rows(data / "function_words.txt")}
|
||||
# words -> [class, forms (one, or one per word), final form or ""], see phrases.txt
|
||||
phrases = {r[0]: [r[1], [f.strip() for f in r[2].split("|")], r[3] if len(r) > 3 else ""] for r in _table_rows(data / "phrases.txt")}
|
||||
letters = {r[0]: r[1] for r in _table_rows(data / "letters.txt")}
|
||||
# a value is phones, or CMUdict notation (converted by the template, like the output of the unknown-word model)
|
||||
exceptions = {r[0].lower(): r[1] for r in _table_rows(data / "exceptions.txt") if " " not in r[0]}
|
||||
# a name of several words: "w1 w2 ..." -> one pronunciation per word; first word -> the longest name it starts
|
||||
names: dict[str, Any] = {}
|
||||
name_starts: dict[str, int] = {}
|
||||
for r in _table_rows(data / "exceptions.txt"):
|
||||
words = r[0].lower().split()
|
||||
if len(words) > 1:
|
||||
names[" ".join(words)] = [part.strip() for part in r[1].split("|")]
|
||||
name_starts[words[0]] = max(name_starts.get(words[0], 0), len(words))
|
||||
# phoneme (CMUdict, no stress) -> {letters: log probability}: aligns the letters of a word to its phonemes
|
||||
with open(dir_model / "phonemizer" / "align.json", encoding="utf-8") as f:
|
||||
align = json.load(f)
|
||||
# abbreviation -> [when, words]
|
||||
abbreviations = {r[0]: [r[1], r[2]] for r in _table_rows(data / "abbreviations.txt")}
|
||||
# chat spelling -> [words to say ("*LOL": letter by letter, "-": nothing), only as a reply by itself], see chat.txt
|
||||
chat = {r[0].lower(): [r[1] if len(r) > 1 else "-", len(r) > 2] for r in _table_rows(data / "chat.txt")}
|
||||
# accented latin letter -> plain letter(s): "café" is in the dictionary as "cafe" (dizione: _plain_letter)
|
||||
accents = {"ß": "ss", "æ": "ae", "Æ": "Ae", "œ": "oe", "Œ": "Oe", "ø": "o", "Ø": "O", "đ": "d", "ł": "l", "Ł": "L"}
|
||||
for cp in range(0x80, 0x2000):
|
||||
char = chr(cp)
|
||||
if char.isalpha() and char not in accents:
|
||||
plain = "".join(c for c in unicodedata.normalize("NFKD", char) if not unicodedata.combining(c))
|
||||
if plain.isascii() and plain.isalpha():
|
||||
accents[char] = plain
|
||||
# compatibility forms of the text (NFKC), a character at a time: "…" is "...", "fi" is "fi", "№" is "No", full
|
||||
# width letters. Sequences that NFKC would compose are left as they are
|
||||
nfkc: dict[str, str] = {}
|
||||
for cp in list(range(0x80, 0x3000)) + list(range(0xFB00, 0xFB07)) + list(range(0xFF01, 0xFF5F)):
|
||||
if unicodedata.normalize("NFKC", chr(cp)) != chr(cp):
|
||||
nfkc[chr(cp)] = unicodedata.normalize("NFKC", chr(cp))
|
||||
# units after a number: name -> [one, many]; "exact" ones are written in one letter case only ("m", not "M")
|
||||
units: dict[str, Any] = {}
|
||||
exact_units: dict[str, Any] = {}
|
||||
for r in _table_rows(data / "units.txt"):
|
||||
if len(r) > 3 and r[3] == "exact":
|
||||
exact_units[r[0]] = [r[1], r[2]]
|
||||
else:
|
||||
units[r[0].lower()] = [r[1], r[2]]
|
||||
# the unit names as a pattern of tts_norm_scan(), the longest first; a unit does not stop inside a word
|
||||
unit_names = sorted(set(units) | set(exact_units), key=lambda name: (-len(name), name))
|
||||
# (the pattern ignores the case of ASCII letters only, the greek letter is written in both)
|
||||
unit_pattern = "|".join("".join("[μΜ]" if c == "μ" else c if c.isalnum() else "%" + c for c in name) + "(?![A-Za-z])" for name in unit_names)
|
||||
# endings of file names and addresses: "word" (dot com) or "letters" (dot P D F)
|
||||
endings = {r[0]: (r[1] if len(r) > 1 else "") for r in _table_rows(data / "endings.txt")}
|
||||
out = {
|
||||
"lexicon": lexicon,
|
||||
"functions": functions,
|
||||
"phrases": phrases,
|
||||
"letters": letters,
|
||||
"exceptions": exceptions,
|
||||
"abbreviations": abbreviations,
|
||||
"align": align,
|
||||
"names": names,
|
||||
"chat": chat,
|
||||
"accents": accents,
|
||||
"nfkc": nfkc,
|
||||
"units": units,
|
||||
"exact_units": exact_units,
|
||||
"endings": endings,
|
||||
"patterns": {"units": unit_pattern},
|
||||
"name_starts": name_starts,
|
||||
}
|
||||
out.update(_homograph_dicts(dir_model, data))
|
||||
return out
|
||||
|
||||
|
||||
def _homograph_dicts(dir_model: Path, data: Path) -> dict[str, dict[str, Any]]:
|
||||
# words spelled alike and pronounced differently (dizione homographs.py): a linear classifier on the neighbouring
|
||||
# words, and hand-written rules that are tried first. Weights are stored x10000 (they have four decimals), so the
|
||||
# template adds integers.
|
||||
path = dir_model / "phonemizer" / "homographs.json"
|
||||
if not path.is_file():
|
||||
return {"homographs": {}, "homograph_shared": {}, "homograph_plurals": {}, "homograph_rules": {}}
|
||||
with open(path, encoding="utf-8") as f:
|
||||
source = json.load(f)
|
||||
|
||||
def fixed(x: float) -> int:
|
||||
value = round(x * 10000)
|
||||
if abs(value - x * 10000) > 1e-6:
|
||||
raise ValueError(f"homographs.json: weight {x} has more than four decimals")
|
||||
return value
|
||||
|
||||
words: dict[str, Any] = {}
|
||||
for word, entry in source["words"].items():
|
||||
model = entry["model"]
|
||||
noun_verb = model["kind"] == "noun-verb"
|
||||
if not noun_verb and model["kind"] != "senses":
|
||||
raise ValueError(f"homographs.json: unknown model kind {model['kind']!r}")
|
||||
words[word] = {
|
||||
# sense: [id, role, phones]
|
||||
"senses": [[s["id"], s["role"], s["phones"]] for s in entry["senses"]],
|
||||
# noun-verb: one score for the verb reading; else one score per sense
|
||||
"noun_verb": noun_verb,
|
||||
"bias": [fixed(model["bias"])] if noun_verb else [fixed(b) for b in model["bias"]],
|
||||
"weights": [{k: fixed(v) for k, v in w.items()} for w in ([model["weights"]] if noun_verb else model["weights"])],
|
||||
}
|
||||
# the weights all noun-or-verb homographs share; "" is the bias
|
||||
shared = {k: fixed(v) for k, v in source["pooled"]["weights"].items()}
|
||||
shared[""] = fixed(source["pooled"]["bias"])
|
||||
# word -> [[sense id, [[condition, [words]], ...]], ...], see homograph_rules.txt
|
||||
rules: dict[str, Any] = {}
|
||||
for r in _table_rows(data / "homograph_rules.txt"):
|
||||
conditions = []
|
||||
for part in r[2].split(";"):
|
||||
name, _, listed = part.strip().partition(":")
|
||||
conditions.append([name, listed.split()])
|
||||
rules.setdefault(r[0], []).append([r[1], conditions])
|
||||
return {
|
||||
"homographs": words,
|
||||
"homograph_shared": shared,
|
||||
"homograph_plurals": {base: 1 for base in source.get("plural_bases", ())},
|
||||
"homograph_rules": rules,
|
||||
}
|
||||
|
||||
|
||||
def _load_state(dir_model: Path) -> dict[str, Tensor]:
|
||||
ckpt = torch.load(dir_model / CHECKPOINT, map_location="cpu", weights_only=True)
|
||||
if ckpt.get("format_version") != 4:
|
||||
raise ValueError(f"unsupported tiny-voca checkpoint format {ckpt.get('format_version')}")
|
||||
return ckpt["model"]
|
||||
|
||||
|
||||
def _block_name(name: str) -> tuple[int, str] | None:
|
||||
# "blocks.<i>.<module>.<weight|bias>" -> (i, "blk.<i>.<libllama name>.<weight|bias>")
|
||||
parts = name.split(".")
|
||||
if len(parts) < 4 or parts[0] != "blocks":
|
||||
return None
|
||||
bid = int(parts[1])
|
||||
module, suffix = ".".join(parts[2:-1]), parts[-1]
|
||||
return bid, f"blk.{bid}.{BLOCK_MAP[module]}.{suffix}"
|
||||
|
||||
|
||||
def _block_tensors(new_name: str, t: Tensor) -> Iterable[tuple[str, Tensor]]:
|
||||
# depthwise conv [C, 1, K] -> [C, K], the layout of ggml_ssm_conv
|
||||
if ".ssm_conv1d.weight" in new_name:
|
||||
yield (new_name, t.squeeze(1))
|
||||
return
|
||||
# separate q, k, v: a model that libllama builds from metadata (the mmproj stacks) has no fused qkv
|
||||
if ".attn_qkv." in new_name:
|
||||
for part, chunk in zip(("attn_q", "attn_k", "attn_v"), t.chunk(3, dim=0)):
|
||||
yield (new_name.replace("attn_qkv", part), chunk.contiguous())
|
||||
return
|
||||
yield (new_name, t)
|
||||
|
||||
|
||||
@ModelBase.register("TinyVocaModel")
|
||||
@ModelBase.example("n4ze3m/tiny-voca")
|
||||
class TinyVocaTextModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.TINY_VOCA
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
del remote_hf_model_id
|
||||
state = _load_state(self.dir_model)
|
||||
out: dict[str, Callable[[], Tensor]] = {}
|
||||
for name, t in state.items():
|
||||
if name == "tokens.weight" or name.startswith("encoder."):
|
||||
out[name] = (lambda t=t: t)
|
||||
return out
|
||||
|
||||
def set_vocab(self):
|
||||
config = self.hparams["tiny_voca"]
|
||||
n_vocab = config["model"]["vocab_size"]
|
||||
tokens: list[str] = [""] * n_vocab
|
||||
toktypes = [gguf.TokenType.NORMAL] * n_vocab
|
||||
for text, i in config["vocab"].items():
|
||||
tokens[i] = text
|
||||
if text.startswith("<") and text.endswith(">"):
|
||||
toktypes[i] = gguf.TokenType.CONTROL
|
||||
assert all(tokens)
|
||||
|
||||
# the prompt is split into phonemes by the audio pipeline, see mtmd-helper-gen.cpp
|
||||
self.gguf_writer.add_tokenizer_model("llama")
|
||||
self.gguf_writer.add_tokenizer_pre("default")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores([0.0] * n_vocab)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
self.gguf_writer.add_bos_token_id(config["vocab"]["<bos>"])
|
||||
self.gguf_writer.add_eos_token_id(config["vocab"]["<eos>"])
|
||||
self.gguf_writer.add_pad_token_id(config["vocab"]["<pad>"])
|
||||
self.gguf_writer.add_unk_token_id(config["vocab"]["<unk>"])
|
||||
self.gguf_writer.add_add_bos_token(False)
|
||||
self.gguf_writer.add_add_eos_token(False)
|
||||
|
||||
dicts = _g2p_dicts(self.dir_model)
|
||||
if dicts is not None:
|
||||
template_path = Path(__file__).parent.parent / "models" / "templates" / "n4ze3m-tiny-voca-tts.jinja"
|
||||
with open(template_path, encoding="utf-8") as f:
|
||||
self.gguf_writer.add_chat_template([{"name": "tts", "template": f.read()}])
|
||||
self.gguf_writer.add_tts_dicts(dicts)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
self.gguf_writer.add_causal_attention(False)
|
||||
self.gguf_writer.add_pooling_type(gguf.PoolingType.NONE)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
del bid
|
||||
if name == "tokens.weight":
|
||||
yield ("token_embd.weight", data_torch)
|
||||
return
|
||||
if name.startswith("encoder.norm."):
|
||||
yield ("output_norm." + name.rsplit(".", 1)[1], data_torch)
|
||||
return
|
||||
res = _block_name(name.removeprefix("encoder."))
|
||||
if res is None:
|
||||
raise ValueError(f"unexpected tensor {name}")
|
||||
yield from _block_tensors(res[1], data_torch)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
del name, bid, n_dims
|
||||
# ggml_ssm_conv takes F32 weights; token_embd is also read row by row on the host side
|
||||
if ".ssm_conv1d." in new_name or new_name == "token_embd.weight":
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return False
|
||||
|
||||
|
||||
@ModelBase.register("TinyVocaModel")
|
||||
@ModelBase.example("n4ze3m/tiny-voca")
|
||||
class TinyVocaMmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = False
|
||||
|
||||
def get_audio_config(self) -> dict[str, Any] | None:
|
||||
return self.global_config.get("audio_config")
|
||||
|
||||
def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]:
|
||||
del remote_hf_model_id
|
||||
state = _load_state(self.dir_model)
|
||||
out: dict[str, Callable[[], Tensor]] = {}
|
||||
for name, t in state.items():
|
||||
if name.startswith("encoder.") or name in ("voices.weight", "frame_voices.weight"):
|
||||
continue
|
||||
out[name] = (lambda t=t: t)
|
||||
return out
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
return item
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
c = self.global_config["tiny_voca"]["model"]
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.TINY_VOCA)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(c["channels"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(c["feedforward"])
|
||||
self.gguf_writer.add_gen_audio_block_count(c["acoustic_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(c["heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5)
|
||||
self.gguf_writer.add_gen_audio_default_voice(DEFAULT_VOICE)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
del bid
|
||||
# from generate_extra_tensors(), named already
|
||||
if name.startswith(("voice.", "a.gen.")):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
stack, _, rest = name.partition(".")
|
||||
if stack in STACK_PREFIX:
|
||||
prefix = STACK_PREFIX[stack]
|
||||
if rest.startswith("norm."):
|
||||
yield (prefix + "output_norm." + rest.rsplit(".", 1)[1], data_torch)
|
||||
return
|
||||
res = _block_name(rest)
|
||||
if res is None:
|
||||
raise ValueError(f"unexpected tensor {name}")
|
||||
yield from _block_tensors(prefix + res[1], data_torch)
|
||||
return
|
||||
module, suffix = name.rsplit(".", 1)
|
||||
if module not in MMPROJ_MAP:
|
||||
raise ValueError(f"unexpected tensor {name}")
|
||||
yield (MMPROJ_MAP[module] + "." + suffix, data_torch)
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
del name, bid
|
||||
# the blocks of the stacks are loaded by libllama, like the text model
|
||||
if new_name.startswith(tuple(STACK_PREFIX.values())) and ".ssm_conv1d." not in new_name and n_dims >= 2:
|
||||
return False
|
||||
# the rest is small and partly read on the host
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# the unknown-word model of dizione (letters -> CMUdict phonemes), run on the host by mtmd_gen_audio_g2p()
|
||||
g2p_dir = self.dir_model / "phonemizer"
|
||||
if (g2p_dir / "g2p.npz").is_file():
|
||||
with open(g2p_dir / "g2p.json", encoding="utf-8") as f:
|
||||
settings = json.load(f)
|
||||
if settings["letters"] != G2P_LETTERS or settings["symbols"] != G2P_SYMBOLS or settings["convs"] != 0 or settings["slots"] != 2:
|
||||
raise ValueError("unsupported g2p model in phonemizer/g2p.json")
|
||||
with np.load(g2p_dir / "g2p.npz") as arrays:
|
||||
for name in arrays.files:
|
||||
yield ("a.gen.g2p." + name.replace("__", "."), torch.from_numpy(arrays[name].astype(np.float32)))
|
||||
config = self.global_config["tiny_voca"]
|
||||
state = _load_state(self.dir_model)
|
||||
# a voice row is the embedding added to the encoder input, then the one added to the frames
|
||||
for i, voice in enumerate(config["voices"]):
|
||||
yield (f"voice.{voice}", torch.cat([state["voices.weight"][i], state["frame_voices.weight"][i]]).float())
|
||||
# class of each token: 0 = pause (special or punctuation), 1 = consonant or <unk>, 2 = vowel
|
||||
vocab = config["vocab"]
|
||||
cls = torch.zeros(len(vocab), dtype=torch.float32)
|
||||
for p in CONSONANTS + ("<unk>",):
|
||||
cls[vocab[p]] = 1.0
|
||||
for p in VOWELS:
|
||||
cls[vocab[p]] = 2.0
|
||||
yield ("a.gen.token_class", cls)
|
||||
@@ -0,0 +1,314 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, MmprojModel, TextModel, LazyTorchTensor, gguf, logger
|
||||
from .drama3 import fold_snake
|
||||
from .minicpm import MiniCPMModel
|
||||
|
||||
# sample rate of the AudioVAE decoder output, picks the row of the sample-rate conditioning
|
||||
VAE_OUT_SAMPLE_RATE = 48000
|
||||
|
||||
|
||||
def _load_config(dir_model: Path) -> dict[str, Any] | None:
|
||||
path = dir_model / "config.json"
|
||||
if not path.is_file():
|
||||
return None
|
||||
with open(path, encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
return config if config.get("architecture") == "voxcpm2" else None
|
||||
|
||||
|
||||
def _is_voxcpm2_checkpoint(dir_model: Path) -> bool:
|
||||
return _load_config(dir_model) is not None
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_voxcpm2_checkpoint)
|
||||
def _load_voxcpm2_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected VoxCPM2 checkpoint")
|
||||
config = _load_config(dir_model)
|
||||
assert config is not None
|
||||
if config["lm_config"].get("use_mup"):
|
||||
raise NotImplementedError("VoxCPM2: use_mup = true is not supported")
|
||||
if config["dit_config"].get("mean_mode"):
|
||||
raise NotImplementedError("VoxCPM2: dit mean_mode = true is not supported")
|
||||
if not config.get("residual_lm_no_rope"):
|
||||
raise NotImplementedError("VoxCPM2: residual LM with RoPE is not supported")
|
||||
for key in ("hidden_dim", "ffn_dim", "num_heads", "num_layers", "kv_channels"):
|
||||
if config["encoder_config"][key] != config["dit_config"][key]:
|
||||
raise NotImplementedError(f"VoxCPM2: LocEnc and LocDiT differ in {key}")
|
||||
return {**config, "architectures": ["VoxCPM2Model"], "text_config": config["lm_config"], "audio_config": config["dit_config"]}
|
||||
|
||||
|
||||
@ModelBase.register("VoxCPM2Model")
|
||||
@ModelBase.example("openbmb/VoxCPM2")
|
||||
class VoxCPM2TextModel(MiniCPMModel):
|
||||
model_arch = gguf.MODEL_ARCH.MINICPM
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
# the text-semantic LM is a plain MiniCPM, the rest goes to the mmproj
|
||||
if not name.startswith("base_lm."):
|
||||
return None
|
||||
return super().filter_tensors(("model." + name.removeprefix("base_lm."), gen))
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
TextModel.set_gguf_parameters(self)
|
||||
# use_mup = false: no embedding, residual or logit scaling
|
||||
self.gguf_writer.add_embedding_scale(1.0)
|
||||
self.gguf_writer.add_residual_scale(1.0)
|
||||
self.gguf_writer.add_logit_scale(1.0)
|
||||
|
||||
def set_vocab(self):
|
||||
# the checkpoint has no tokenizer.model and its tokenizer class is remote code, so read tokenizer.json
|
||||
# SPM-style BPE: a merged token scores minus its merge rank, so the SPM tokenizer merges in the BPE order
|
||||
with open(self.dir_model / "tokenizer.json", encoding="utf-8") as f:
|
||||
tok = json.load(f)
|
||||
with open(self.dir_model / "tokenizer_config.json", encoding="utf-8") as f:
|
||||
tok_cfg = json.load(f)
|
||||
model = tok["model"]
|
||||
if model["type"] != "BPE" or not model.get("byte_fallback"):
|
||||
raise NotImplementedError("VoxCPM2: unexpected tokenizer type")
|
||||
|
||||
vocab: dict[str, int] = model["vocab"]
|
||||
vocab_size = self.hparams["vocab_size"]
|
||||
tokens = [f"[PAD{i}]" for i in range(vocab_size)]
|
||||
scores = [-1e9] * vocab_size
|
||||
toktypes = [gguf.TokenType.UNUSED] * vocab_size
|
||||
for text, i in vocab.items():
|
||||
tokens[i] = text
|
||||
toktypes[i] = gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-Fa-f]{2}>", text) else gguf.TokenType.NORMAL
|
||||
for rank, merge in enumerate(model["merges"]):
|
||||
a, b = merge.split(" ") if isinstance(merge, str) else merge
|
||||
i = vocab[a + b]
|
||||
scores[i] = max(scores[i], -float(rank))
|
||||
|
||||
# the special tokens (e.g. <|audio_start|>) are only listed in tokenizer_config.json
|
||||
specials = {int(i): t["content"] for i, t in tok_cfg.get("added_tokens_decoder", {}).items() if t.get("special")}
|
||||
specials.update({t["id"]: t["content"] for t in tok["added_tokens"] if t.get("special")})
|
||||
for i, text in specials.items():
|
||||
tokens[i] = text
|
||||
toktypes[i] = gguf.TokenType.CONTROL
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("llama")
|
||||
self.gguf_writer.add_tokenizer_pre("default")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores(scores)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
|
||||
@ModelBase.register("VoxCPM2Model")
|
||||
@ModelBase.example("openbmb/VoxCPM2")
|
||||
class VoxCPM2MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True # AudioVAE encoder, for voice cloning
|
||||
|
||||
_LAYER_MAP = {
|
||||
"input_layernorm.weight": "ln1",
|
||||
"self_attn.q_proj.weight": "attn_q",
|
||||
"self_attn.k_proj.weight": "attn_k",
|
||||
"self_attn.v_proj.weight": "attn_v",
|
||||
"self_attn.o_proj.weight": "attn_out",
|
||||
"post_attention_layernorm.weight": "ln2",
|
||||
"mlp.gate_proj.weight": "ffn_gate",
|
||||
"mlp.up_proj.weight": "ffn_up",
|
||||
"mlp.down_proj.weight": "ffn_down",
|
||||
}
|
||||
|
||||
# libllama names, mtmd loads the residual LM as a text model
|
||||
_RES_LAYER_MAP = {
|
||||
"input_layernorm.weight": "attn_norm",
|
||||
"self_attn.q_proj.weight": "attn_q",
|
||||
"self_attn.k_proj.weight": "attn_k",
|
||||
"self_attn.v_proj.weight": "attn_v",
|
||||
"self_attn.o_proj.weight": "attn_output",
|
||||
"post_attention_layernorm.weight": "ffn_norm",
|
||||
"mlp.gate_proj.weight": "ffn_gate",
|
||||
"mlp.up_proj.weight": "ffn_up",
|
||||
"mlp.down_proj.weight": "ffn_down",
|
||||
}
|
||||
|
||||
_PROJ_MAP = {
|
||||
"fsq_layer.in_proj": "a.gen.fsq.in",
|
||||
"fsq_layer.out_proj": "a.gen.fsq.out",
|
||||
"enc_to_lm_proj": "a.gen.enc_to_lm",
|
||||
"lm_to_dit_proj": "a.gen.lm_to_dit",
|
||||
"res_to_dit_proj": "a.gen.res_to_dit",
|
||||
"fusion_concat_proj": "a.gen.fusion",
|
||||
"stop_proj": "a.gen.stop_proj",
|
||||
"stop_head": "a.gen.stop_head",
|
||||
"feat_encoder.in_proj": "a.gen.enc.in_proj",
|
||||
"feat_decoder.estimator.in_proj": "a.gen.dit.in_proj",
|
||||
"feat_decoder.estimator.cond_proj": "a.gen.dit.cond_proj",
|
||||
"feat_decoder.estimator.out_proj": "a.gen.dit.out_proj",
|
||||
"feat_decoder.estimator.time_mlp.linear_1": "a.gen.dit.time_mlp.0",
|
||||
"feat_decoder.estimator.time_mlp.linear_2": "a.gen.dit.time_mlp.1",
|
||||
}
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
lm = self.global_config["lm_config"]
|
||||
dit = self.global_config["dit_config"]
|
||||
vae = self.global_config["audio_vae_config"]
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
|
||||
# LocEnc + LocDiT + residual LM + AudioVAE decoder, LocEnc has the hparams of LocDiT
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.VOXCPM2_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(dit["hidden_dim"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(dit["ffn_dim"])
|
||||
self.gguf_writer.add_gen_audio_block_count(dit["num_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(dit["num_heads"])
|
||||
self.gguf_writer.add_gen_audio_head_count_kv(lm["num_key_value_heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(lm["rms_norm_eps"])
|
||||
|
||||
# AudioVAE encoder: raw waveform -> latents, LocEnc in the gen model then makes the backbone rows
|
||||
self.gguf_writer.add_clip_has_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.VOXCPM2_SPKENC)
|
||||
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_audio_num_mel_bins(1)
|
||||
self.gguf_writer.add_audio_block_count(0)
|
||||
self.gguf_writer.add_audio_embedding_length(vae["encoder_dim"])
|
||||
self.gguf_writer.add_audio_feed_forward_length(vae["encoder_dim"])
|
||||
self.gguf_writer.add_audio_head_count(1)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(lm["rms_norm_eps"])
|
||||
# the ODE solver, the RoPE factors and the VAE layout are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# the AudioVAE convs and the 1D tensors stay F32
|
||||
if new_name.startswith(("a.vae.", "a.gen.vae.")) or n_dims < 2:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
if item[0].startswith("base_lm."):
|
||||
return None
|
||||
return item
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# already named by generate_extra_tensors()
|
||||
if name.startswith("a."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
# dt is always 0 (mean_mode = false), delta_time_mlp gives a constant that is folded into the time_mlp bias
|
||||
if name.startswith("feat_decoder.estimator.delta_time_mlp."):
|
||||
return
|
||||
if name == "feat_decoder.estimator.time_mlp.linear_2.bias":
|
||||
data_torch = LazyTorchTensor.to_eager(data_torch).float() + self._dtime_const()
|
||||
|
||||
if name == "feat_encoder.special_token":
|
||||
yield ("a.gen.enc.special_token", data_torch.reshape(-1))
|
||||
return
|
||||
|
||||
for src, dst in (("residual_lm.", "a.gen.res"), ("feat_encoder.encoder.", "a.gen.enc"),
|
||||
("feat_decoder.estimator.decoder.", "a.gen.code")):
|
||||
if not name.startswith(src):
|
||||
continue
|
||||
key = name.removeprefix(src)
|
||||
if key == "norm.weight":
|
||||
yield (f"{dst}.output_norm.weight" if dst == "a.gen.res" else f"{dst}.post_ln.weight", data_torch)
|
||||
return
|
||||
assert bid is not None, name
|
||||
key = key.removeprefix(f"layers.{bid}.")
|
||||
layer_map = self._RES_LAYER_MAP if dst == "a.gen.res" else self._LAYER_MAP
|
||||
yield (f"{dst}.blk.{bid}.{layer_map[key]}.weight", data_torch)
|
||||
return
|
||||
|
||||
base, _, suffix = name.rpartition(".")
|
||||
yield (f"{self._PROJ_MAP[base]}.{suffix}", data_torch)
|
||||
|
||||
# delta_time_mlp(SinusoidalPosEmb(0)), see VoxCPMLocDiTV2.forward()
|
||||
def _dtime_const(self) -> Tensor:
|
||||
d = {k: LazyTorchTensor.to_eager(self.model_tensors[f"feat_decoder.estimator.delta_time_mlp.{k}"]()).float()
|
||||
for k in ("linear_1.weight", "linear_1.bias", "linear_2.weight", "linear_2.bias")}
|
||||
dim = d["linear_1.weight"].shape[1]
|
||||
emb = torch.cat([torch.zeros(dim // 2), torch.ones(dim // 2)]) # sin(0), cos(0)
|
||||
h = torch.nn.functional.silu(d["linear_1.weight"] @ emb + d["linear_1.bias"])
|
||||
return d["linear_2.weight"] @ h + d["linear_2.bias"]
|
||||
|
||||
@staticmethod
|
||||
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
|
||||
return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# LongRoPE factors of LocEnc and LocDiT, picked as in MiniCPMLongRoPE._set_cos_sin_cache()
|
||||
lm = self.global_config["lm_config"]
|
||||
rope = lm["rope_scaling"]
|
||||
if rope.get("type") != "longrope":
|
||||
raise NotImplementedError(f"VoxCPM2: unexpected rope_scaling {rope.get('type')}")
|
||||
if lm["max_position_embeddings"] > rope["original_max_position_embeddings"]:
|
||||
raise NotImplementedError("VoxCPM2: the long RoPE factors also scale cos/sin, not supported")
|
||||
yield ("a.gen.rope_factors", torch.tensor(rope["short_factor"], dtype=torch.float32))
|
||||
|
||||
path = self.dir_model / "audiovae.pth"
|
||||
logger.info(f"gguf: loading AudioVAE from '{path}'")
|
||||
sd = torch.load(str(path), map_location="cpu", mmap=True, weights_only=True)
|
||||
sd = sd.get("state_dict", sd)
|
||||
vae = self.global_config["audio_vae_config"]
|
||||
|
||||
def conv(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
yield (f"{dst}.weight", self._wn(sd, src))
|
||||
yield (f"{dst}.bias", sd[src + ".bias"].float())
|
||||
|
||||
def snake(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
alpha, beta = fold_snake(sd[src + ".alpha"])
|
||||
yield (f"{dst}.alpha", alpha)
|
||||
yield (f"{dst}.beta", beta)
|
||||
|
||||
def res_units(dst: str, src: str, first: int) -> Iterable[tuple[str, Tensor]]:
|
||||
# dilation 1, 3, 9: Snake, depthwise conv k = 7, Snake, conv k = 1
|
||||
for r in range(3):
|
||||
s = f"{src}.{first + r}.block"
|
||||
d = f"{dst}.res.{r}"
|
||||
yield from snake(f"{d}.snake1", f"{s}.0")
|
||||
yield from conv(f"{d}.conv1", f"{s}.1")
|
||||
yield from snake(f"{d}.snake2", f"{s}.2")
|
||||
yield from conv(f"{d}.conv2", f"{s}.3")
|
||||
|
||||
# encoder: conv, then per stride (3 residual units, Snake, strided conv), then fc_mu
|
||||
p = "a.vae.enc"
|
||||
yield from conv(f"{p}.conv_in", "encoder.block.0")
|
||||
for b in range(len(vae["encoder_rates"])):
|
||||
s = f"encoder.block.{b + 1}.block"
|
||||
yield from res_units(f"{p}.blk.{b}", s, 0)
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.3")
|
||||
yield from conv(f"{p}.blk.{b}.down", f"{s}.4")
|
||||
yield from conv(f"{p}.fc_mu", "encoder.fc_mu")
|
||||
|
||||
# decoder: depthwise conv, conv k = 1, then per stride (sample-rate scale and bias, Snake, transposed conv, 3 residual units), then Snake, conv
|
||||
bounds = vae["sr_bin_boundaries"]
|
||||
sr_idx = sum(1 for b in bounds if b < VAE_OUT_SAMPLE_RATE) # torch.bucketize
|
||||
if vae["out_sample_rate"] != VAE_OUT_SAMPLE_RATE:
|
||||
raise NotImplementedError(f"VoxCPM2: unexpected out_sample_rate {vae['out_sample_rate']}")
|
||||
p = "a.gen.vae"
|
||||
yield from conv(f"{p}.conv_in_dw", "decoder.model.0")
|
||||
yield from conv(f"{p}.conv_in", "decoder.model.1")
|
||||
n_up = len(vae["decoder_rates"])
|
||||
for b in range(n_up):
|
||||
c = f"decoder.sr_cond_model.{b + 2}"
|
||||
yield (f"{p}.blk.{b}.sr_scale", sd[c + ".scale_embed.weight"][sr_idx].float())
|
||||
yield (f"{p}.blk.{b}.sr_bias", sd[c + ".bias_embed.weight"][sr_idx].float())
|
||||
s = f"decoder.model.{b + 2}.block"
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.0")
|
||||
yield from conv(f"{p}.blk.{b}.up", f"{s}.1")
|
||||
yield from res_units(f"{p}.blk.{b}", s, 2)
|
||||
yield from snake(f"{p}.post_snake", f"decoder.model.{n_up + 2}")
|
||||
yield from conv(f"{p}.conv_out", f"decoder.model.{n_up + 3}")
|
||||
|
||||
assert math.prod(vae["encoder_rates"]) * VAE_OUT_SAMPLE_RATE == math.prod(vae["decoder_rates"]) * vae["sample_rate"]
|
||||
@@ -0,0 +1,144 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import torch
|
||||
|
||||
from typing import Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("ZayaForCausalLM")
|
||||
@ModelBase.example("Zyphra/ZAYA1-8B")
|
||||
class ZayaModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.ZAYA1
|
||||
|
||||
# checkpoint names without a .weight/.bias suffix
|
||||
_RENAME = {
|
||||
"model.input_hidden_states_scale": "model.input_hidden_states.weight",
|
||||
"model.input_hidden_states_bias": "model.input_hidden_states.bias",
|
||||
}
|
||||
|
||||
# the two kv heads of attn_v: head 0 = current token, head 1 = previous token
|
||||
_V_PARTS = (
|
||||
"self_attn.qkv_proj.v_proj_current.weight",
|
||||
"self_attn.qkv_proj.v_proj_delayed.weight",
|
||||
)
|
||||
|
||||
# (hs + b_hs) * s_hs + (res + b_res) * s_res = hs * s_hs + res * s_res + (b_hs * s_hs + b_res * s_res)
|
||||
_RES_SCALES = {
|
||||
"post_attention_residual_scale.": (gguf.MODEL_TENSOR.ATTN_RES_SCALE_HS, gguf.MODEL_TENSOR.ATTN_RESID_SCALE),
|
||||
"post_mlp_residual_scale.": (gguf.MODEL_TENSOR.FFN_RES_SCALE_HS, gguf.MODEL_TENSOR.FFN_RESID_SCALE),
|
||||
}
|
||||
_RES_PARTS = ("hidden_states_scale", "hidden_states_bias", "residual_scale", "residual_bias")
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
# added tokens overlap the base vocab, so count the token list itself
|
||||
self._vocab = gguf.LlamaHfVocab(self.dir_model)
|
||||
self._n_vocab = self._vocab.vocab_size
|
||||
self._parts: dict[tuple[int, str], dict[str, Tensor]] = {}
|
||||
self.rope_parameters = self.hparams["rope_parameters"]["hybrid"]
|
||||
|
||||
def set_vocab(self):
|
||||
# added special tokens overlap the base vocab, so mark them as control here
|
||||
with open(self.dir_model / "tokenizer.json", "r", encoding="utf-8") as f:
|
||||
added = {t["id"]: t["special"] for t in json.load(f)["added_tokens"]}
|
||||
|
||||
tokens, scores, toktypes = [], [], []
|
||||
for i, (text, score, toktype) in enumerate(self._vocab.all_tokens()):
|
||||
tokens.append(text)
|
||||
scores.append(score)
|
||||
if added.get(i):
|
||||
toktype = gguf.TokenType.CONTROL
|
||||
toktypes.append(toktype)
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("gemma4")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores(scores)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
self.gguf_writer.add_add_space_prefix(False)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hparams = self.hparams
|
||||
super().set_gguf_parameters()
|
||||
|
||||
self.gguf_writer.add_rope_dimension_count(int(self.rope_parameters["partial_rotary_factor"] * hparams["head_dim"]))
|
||||
self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
|
||||
self.gguf_writer.add_ssm_conv_kernel(hparams["cca_time0"])
|
||||
self.gguf_writer.add_router_hidden_size(hparams["router_hidden_size"])
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
# skip the draft head tensors now, they will be added as part of the TiDAR changes
|
||||
if name.startswith(("diffusion_output_layer.", "diffusion_markov_head.")):
|
||||
return None
|
||||
|
||||
return super().filter_tensors(item)
|
||||
|
||||
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
|
||||
# the router picks one expert, so its matmuls stay F32 like ffn_gate_inp
|
||||
if any(self.match_model_tensor_name(new_name, key, bid) for key in (
|
||||
gguf.MODEL_TENSOR.CCA_CONV_GRP,
|
||||
gguf.MODEL_TENSOR.ROUTER_FC1,
|
||||
gguf.MODEL_TENSOR.ROUTER_FC2,
|
||||
gguf.MODEL_TENSOR.ROUTER_OUT)):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if bid is not None:
|
||||
local = name.removeprefix(f"model.layers.{bid}.")
|
||||
if local in self._V_PARTS:
|
||||
p = self._parts.setdefault((bid, "v"), {})
|
||||
p[local] = data_torch
|
||||
if len(p) == len(self._V_PARTS):
|
||||
del self._parts[(bid, "v")]
|
||||
v = torch.cat([p[n] for n in self._V_PARTS], dim=0)
|
||||
yield from super().modify_tensors(v, f"model.layers.{bid}.{self._V_PARTS[0]}", bid)
|
||||
return
|
||||
for prefix, (key_hs, key_res) in self._RES_SCALES.items():
|
||||
if local.startswith(prefix) and local[len(prefix):] in self._RES_PARTS:
|
||||
p = self._parts.setdefault((bid, prefix), {})
|
||||
p[local[len(prefix):]] = data_torch
|
||||
if len(p) == len(self._RES_PARTS):
|
||||
del self._parts[(bid, prefix)]
|
||||
s_hs, b_hs, s_res, b_res = (p[n].float() for n in self._RES_PARTS)
|
||||
yield self.format_tensor_name(key_hs, bid, ".weight"), s_hs
|
||||
yield self.format_tensor_name(key_res, bid, ".weight"), s_res
|
||||
yield self.format_tensor_name(key_res, bid, ".bias"), b_hs * s_hs + b_res * s_res
|
||||
return
|
||||
|
||||
name = self._RENAME.get(name, name)
|
||||
if name.endswith("balancing_biases"):
|
||||
name += ".bias"
|
||||
elif not name.endswith((".weight", ".bias")):
|
||||
name += ".weight"
|
||||
new_name = self.map_tensor_name(name)
|
||||
|
||||
def is_tensor(key: gguf.MODEL_TENSOR) -> bool:
|
||||
return self.match_model_tensor_name(new_name, key, bid)
|
||||
|
||||
if any(is_tensor(key) for key in (gguf.MODEL_TENSOR.ATTN_Q, gguf.MODEL_TENSOR.ATTN_K)):
|
||||
# through the base class, which fuses q, k and v with --fuse-qkv
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
return
|
||||
if is_tensor(gguf.MODEL_TENSOR.TOKEN_EMBD):
|
||||
data_torch = data_torch[:self._n_vocab]
|
||||
elif is_tensor(gguf.MODEL_TENSOR.SSM_CONV1D):
|
||||
data_torch = data_torch.squeeze(1)
|
||||
elif is_tensor(gguf.MODEL_TENSOR.CCA_CONV_GRP):
|
||||
# [n_group*out, in, kernel] -> [kernel, n_group, out, in]
|
||||
n_group = self.hparams["num_attention_heads"] + self.hparams["num_key_value_heads"]
|
||||
n_out, n_in, n_k = data_torch.shape
|
||||
data_torch = data_torch.reshape(n_group, n_out // n_group, n_in, n_k).permute(3, 0, 1, 2).contiguous()
|
||||
|
||||
yield new_name, data_torch
|
||||
@@ -100,6 +100,12 @@ The OpenCL backend has the following CMake options that control the behavior of
|
||||
| `GGML_OPENCL_USE_ADRENO_KERNELS` | `ON` | Use kernels optimized for Adreno. |
|
||||
| `GGML_OPENCL_USE_ADRENO_BIN_KERNELS` | `OFF` | Allow using binary kernel lib for Adreno. |
|
||||
|
||||
## Decomposed Flash Attention
|
||||
|
||||
The P8 path rounds scratch row pitches for P and transposed V up to a multiple of 32 KiB before adding the existing 64-byte padding. This reduces cache conflicts when the KV length exceeds 16384 and the V head size is at most 256. Larger V heads keep the previous layout. The extra scratch space is less than 32 KiB per row at the default granule.
|
||||
|
||||
`GGML_OPENCL_FA_KV_PITCH_ROUND` sets the rounding granule in bytes to a positive power of two; `0` restores the previous layout. `GGML_OPENCL_FA_KV_PITCH_ROUND_MIN` sets the KV-length threshold (default `16384`), and `GGML_OPENCL_FA_KV_PITCH_ROUND_MAX_DV` sets the largest V head size to round (default `256`). These settings do not change the existing attention dispatch conditions.
|
||||
|
||||
## Program Binary Cache
|
||||
|
||||
Compiled `cl_program` binaries are cached on disk, so subsequent runs skip the expensive
|
||||
|
||||
+416
-193
@@ -3,9 +3,9 @@
|
||||
> [!NOTE]
|
||||
> Performance and memory optimizations, accuracy validation, broader quantization coverage, broader operator and model support are work in progress.
|
||||
|
||||
[OpenVINO](https://docs.openvino.ai/) is an open-source toolkit for optimizing and deploying high-performance AI inference, specifically designed for Intel hardware, including CPUs, GPUs, and NPUs, in the cloud, on-premises, and on the edge. [OpenVINO backend for llama.cpp](../../ggml/src/ggml-openvino) enables hardware-accelerated inference on **Intel® CPUs, GPUs, and NPUs** while remaining compatible with the existing **GGUF model ecosystem**. The backend translates GGML compute graphs into OpenVINO graphs and leverages graph compilation, kernel fusion, and device-specific optimizations to improve inference performance on supported Intel hardware.
|
||||
[OpenVINO](https://docs.openvino.ai/) is an open-source toolkit for optimizing and deploying high-performance AI inference, specifically designed for Intel hardware, including CPUs, GPUs (integrated and discrete), and NPUs, in the cloud, on-premises, and on the edge. The [OpenVINO backend](../../ggml/src/ggml-openvino) runs the same GGUF model files on **Intel® CPUs, GPUs and NPUs**, without changes to the model or the rest of llama.cpp.
|
||||
|
||||
The OpenVINO backend is implemented in `ggml/src/ggml-openvino` and provides a translation layer for core GGML operations. The OpenVINO backend replaces the standard GGML graph execution path with Intel's OpenVINO inference engine. This approach allows the same GGUF model file to run on Intel CPUs, Intel GPUs (integrated and discrete), and Intel NPUs without changes to the model or the rest of the llama.cpp stack. When a `ggml_cgraph` is dispatched to OpenVINO backend, it:
|
||||
When a `ggml_cgraph` is dispatched to the OpenVINO backend, it:
|
||||
|
||||
- Walks the GGML graph and identifies inputs, outputs, weights, and KV cache tensors.
|
||||
- Translates the GGML operations into an `ov::Model` using OpenVINO's frontend API.
|
||||
@@ -16,9 +16,10 @@ For guidance on contributing to the OpenVINO backend, see the [OpenVINO Backend
|
||||
|
||||
## Contents
|
||||
|
||||
- [Quick Start](#quick-start)
|
||||
- [Supported Devices](#supported-devices)
|
||||
- [Supported Model Precisions](#supported-model-precisions)
|
||||
- [Supported Llama.cpp Tools](#supported-llamacpp-tools)
|
||||
- [Supported llama.cpp Tools](#supported-llamacpp-tools)
|
||||
- [Validated Models](#validated-models)
|
||||
- [Build Instructions](#build-instructions)
|
||||
- [0. Prerequisites](#0-prerequisites)
|
||||
@@ -29,10 +30,44 @@ For guidance on contributing to the OpenVINO backend, see the [OpenVINO Backend
|
||||
- [3. Download Sample Model](#3-download-sample-model)
|
||||
- [4. Run Inference with OpenVINO Backend](#4-run-inference-with-openvino-backend)
|
||||
- [5. Docker Build](#5-docker-build)
|
||||
- [GGML OpenVINO Backend Runtime Configurations](#ggml-openvino-backend-runtime-configurations)
|
||||
- [Runtime Configuration](#runtime-configuration)
|
||||
- [Basic](#basic)
|
||||
- [Plugin properties](#plugin-properties)
|
||||
- [Advanced tuning](#advanced-tuning)
|
||||
- [Compiled model cache](#compiled-model-cache)
|
||||
- [Deprecated](#deprecated)
|
||||
- [Debugging and development](#debugging-and-development)
|
||||
- [Example Usage](#example-usage)
|
||||
- [GPU Inference with Profiling](#gpu-inference-with-profiling)
|
||||
- [Known Limitations](#known-limitations)
|
||||
- [Work in Progress](#work-in-progress)
|
||||
|
||||
## Quick Start
|
||||
|
||||
On Ubuntu, with the [prerequisites](#0-prerequisites) and the [OpenVINO Runtime](#1-install-openvino-runtime) installed:
|
||||
|
||||
```bash
|
||||
# 1. Build
|
||||
git clone https://github.com/ggml-org/llama.cpp
|
||||
cd llama.cpp
|
||||
source /opt/intel/openvino/setupvars.sh
|
||||
cmake -B build/ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON
|
||||
cmake --build build/ReleaseOV --parallel
|
||||
|
||||
# 2. Download a model
|
||||
mkdir -p ~/models/
|
||||
wget https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf \
|
||||
-O ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
|
||||
# 3. Run on CPU, GPU or NPU
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024
|
||||
```
|
||||
|
||||
The [Ubuntu Build Script](#ubuntu-build-script) and the [Windows Build Script](#windows-build-script) do the setup in one step: they install the prerequisites and the OpenVINO Runtime, and build llama.cpp with the OpenVINO backend.
|
||||
|
||||
See [Build Instructions](#build-instructions) for Windows and Docker, and [Runtime Configuration](#runtime-configuration) for the environment variables.
|
||||
|
||||
## Supported Devices
|
||||
|
||||
OpenVINO backend supports the following hardware:
|
||||
@@ -41,7 +76,7 @@ OpenVINO backend supports the following hardware:
|
||||
- Intel GPUs (integrated and discrete)
|
||||
- Intel NPUs
|
||||
|
||||
Although OpenVINO supports a wide range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), the llama.cpp OpenVINO backend has been validated specifically on AI PCs such as the Intel® Core™ Ultra Series 1 and Series 2.
|
||||
Although OpenVINO supports a wide range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), the llama.cpp OpenVINO backend has been validated specifically on AI PCs such as the Intel® Core™ Ultra Series 2 (Lunar Lake) and Series 3 (Panther Lake).
|
||||
|
||||
## Supported Model Precisions
|
||||
|
||||
@@ -51,101 +86,124 @@ Although OpenVINO supports a wide range of [Intel hardware](https://docs.openvin
|
||||
- `Q4_0`
|
||||
- `Q4_1`
|
||||
- `Q4_K`
|
||||
- `Q4_K_M`
|
||||
- `Q5_K` (converted to `Q8_0_C` at runtime by default)
|
||||
- `Q6_K` (converted to `Q8_0_C` at runtime by default)
|
||||
- `Q5_0` (converted at runtime, see the details below)
|
||||
- `Q5_1`
|
||||
- `Q5_K` (converted at runtime, see the details below)
|
||||
- `Q6_K` (converted at runtime, see the details below)
|
||||
|
||||
> [!NOTE]
|
||||
> Accuracy validation and performance optimizations for quantized models are a work in progress.
|
||||
`Q4_K_M` files mix `Q4_K` tensors with `Q6_K` tensors, and sometimes `Q5_K` tensors.
|
||||
|
||||
**CPU and GPU Quantization Details:**
|
||||
- `Q5_K` and `Q6_K` tensors are converted to `Q8_0_C`
|
||||
<details>
|
||||
<summary>Quantization details for each device</summary>
|
||||
|
||||
**NPU Quantization Details:**
|
||||
- Primary supported quantization scheme is `Q4_0`
|
||||
- `Q6_K` tensors are requantized to `Q4_0_128` in general. For embedding weights, `Q6_K` tensors are requantized to `Q8_0_C` except for the token embedding matrix which is dequantized to fp16
|
||||
**All devices:**
|
||||
- `FP16` and `BF16` tensors are used as they are.
|
||||
- A quantized token embedding (`token_embd.weight`) and output weight (`output.weight`) are converted to `Q8_0_C`.
|
||||
|
||||
**CPU and GPU:**
|
||||
- `Q4_0`, `Q4_1`, `Q4_K`, `Q5_1` and `Q8_0` tensors keep their layout: weights, scales and zero points are extracted directly.
|
||||
- `Q5_K` and `Q6_K` tensors are converted to `Q8_0_C`. `GGML_OPENVINO_REQUANT_KQUANT` selects a 4-bit layout instead.
|
||||
- `Q5_0` tensors are converted to `Q4_0_64`.
|
||||
|
||||
**GPU:**
|
||||
- `Q4_K`, `Q5_K` and `Q6_K` MoE expert tensors are converted to `Q4_1_64`. This lets the GPU plugin use its fused MoE op, which is much faster for prompt processing. `GGML_OPENVINO_REQUANT_KQUANT` does not change this. `GGML_OPENVINO_MOE_OP=0` turns the fused MoE op off.
|
||||
|
||||
**NPU:**
|
||||
- All other quantized tensors are requantized to `Q4_0_128`.
|
||||
- For a `Q6_K` token embedding, `GGML_OPENVINO_NPU_TOKEN_EMBD_QUANT` selects another layout.
|
||||
- The validated models use `Q4_K_M` files.
|
||||
|
||||
**Layout names:**
|
||||
- `Q8_0_C`: 8-bit, one scale per row.
|
||||
- `Q4_0_128` and `Q4_0_64`: 4-bit, symmetric, one scale per 128 or 64 weights.
|
||||
|
||||
**Additional Notes:**
|
||||
- Both `Q4_0` and `Q4_1` models use `Q6_K` for the token embedding tensor and the final matmul weight tensor (often the same tensor)
|
||||
- `Q4_0` models may produce some `Q4_1` tensors if an imatrix is provided during quantization using `llama-quantize`
|
||||
- `Q4_K_M` models may include both `Q6_K` and `Q5_K` tensors (observed in Phi-3)
|
||||
- `Q5_1` tensors are dequantized natively (weights, scales, and zero-points extracted directly)
|
||||
|
||||
## Supported Llama.cpp Tools
|
||||
</details>
|
||||
|
||||
The OpenVINO backend integrates with the standard llama.cpp tools listed below.
|
||||
However, all the tools coverage across all devices is not uniform and exhaustive validation is work in progress.
|
||||
## Supported llama.cpp Tools
|
||||
|
||||
The OpenVINO backend works with the standard llama.cpp tools listed below. Tool coverage is not uniform across devices.
|
||||
|
||||
- llama-bench
|
||||
- llama-cli
|
||||
- llama-completion
|
||||
- llama-embedding
|
||||
- llama-perplexity
|
||||
- llama-run
|
||||
- llama-server
|
||||
- llama-simple
|
||||
|
||||
## Validated Models
|
||||
|
||||
Although, the validated models below were tested with `llama-cli` using the `Q4_K_M` quantization format on Intel® Core™ Ultra Series 2 (Lunar Lake), the OpenVINO backend is expected to work across a broader range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), [supported model precisions](#supported-model-precisions), [supported llama.cpp tools](#supported-llamacpp-tools) and additional model architectures.
|
||||
|
||||
> [!NOTE]
|
||||
> Extensive accuracy validation, performance optimizations, and broader architecture coverage are work in progress.
|
||||
The models below were tested with `llama-cli` and `Q4_K_M` files on an Intel® Core™ Ultra Series 2 (Lunar Lake). The OpenVINO backend is expected to work on a broader range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), [model precisions](#supported-model-precisions), [llama.cpp tools](#supported-llamacpp-tools) and model architectures.
|
||||
|
||||
**Legend & Test Configuration:**
|
||||
- **Status:** ✓ = Passed | ~ = Accuracy issues | ✗ = Failed or Unsupported
|
||||
- **Execution Modes:**
|
||||
- **SL** = Stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`)
|
||||
- **SF** = Stateful (`GGML_OPENVINO_STATEFUL_EXECUTION=1`)
|
||||
- Note: The NPU operates in stateless mode only.
|
||||
- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel Graphics Compiler 2.41.5 | Intel OpenCL GPU Driver 26.35.39758.10-0 | Intel NPU Driver 1.38.0.
|
||||
- **Execution mode:** stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`) on every device.
|
||||
- **Validation system:** Intel® Core™ Ultra 7 258V (Lunar Lake) | 32 GB RAM | Ubuntu 26.04.1 | Intel Graphics Compiler 2.42.6 | Intel OpenCL GPU Driver 26.40.40196.8-0 | Intel NPU Driver 1.38.0.
|
||||
- See [Known Limitations](#known-limitations) for context on observed failures.
|
||||
|
||||
| Model | CPU (SL / SF) | GPU (SL / SF) | NPU (SL) |
|
||||
<details open>
|
||||
<summary><b>Validated models</b></summary>
|
||||
|
||||
| Model | CPU | GPU | NPU |
|
||||
| :--- | :---: | :---: | :---: |
|
||||
| [bartowski/Llama-3.2-1B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Llama-3.2-3B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Meta-Llama-3.1-8B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Llama-3.2-1B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Llama-3.2-3B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Meta-Llama-3.1-8B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3.5-0.8B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-0.8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [bartowski/Qwen_Qwen3.5-2B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [bartowski/Qwen_Qwen3.5-4B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-4B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3.5-0.8B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-0.8B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3.5-2B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3.5-4B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-4B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ~ |
|
||||
| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✓ | ✗ / ✗ | ✓ |
|
||||
| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ | ✓ | ✗ |
|
||||
| | | | |
|
||||
| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/microsoft_Phi-4-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/microsoft_Phi-4-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [bartowski/Mistral-7B-Instruct-v0.3-Q4_K_M](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [QuantFactory/Ministral-3b-instruct.Q4_K_M](https://huggingface.co/QuantFactory/Ministral-3b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Ministral-8B-Instruct-2410-Q4_K_M](https://huggingface.co/bartowski/Ministral-8B-Instruct-2410-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Mistral-7B-Instruct-v0.3-Q4_K_M](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [QuantFactory/Ministral-3b-instruct.Q4_K_M](https://huggingface.co/QuantFactory/Ministral-3b-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Ministral-8B-Instruct-2410-Q4_K_M](https://huggingface.co/bartowski/Ministral-8B-Instruct-2410-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ~ / ~ | ✓ |
|
||||
| [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ / ✓ | ~ / ~ | ~ |
|
||||
| [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ | ✓ | ~ |
|
||||
| [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ | ✓ | ✓ |
|
||||
| [openbmb/MiniCPM-V-4_5-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-4_5-gguf) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [LiquidAI/LFM2.5-2.6B-Q4_K_M](https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF) | ✓ | ✓ | ✗ |
|
||||
| [bartowski/inclusionAI_Ling-mini-2.0-Q4_K_M](https://huggingface.co/bartowski/inclusionAI_Ling-mini-2.0-GGUF) | ✓ | ✓ | ✗ |
|
||||
| [tiiuae/Falcon-H1-0.5B-Instruct-Q4_K_M](https://huggingface.co/tiiuae/Falcon-H1-0.5B-Instruct-GGUF) | ✓ | ✓ | ✗ |
|
||||
| | | | |
|
||||
| [gpustack/bge-m3-Q4_K_M.gguf](https://huggingface.co/gpustack/bge-m3-GGUF) | ✓ | ✗ | ✗ |
|
||||
| [Qwen/Qwen3-Embedding-0.6B-Q8_0](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B-GGUF) | ✓ | ✓ | ✗ |
|
||||
| [ggml-org/bge-m3-Q8_0](https://huggingface.co/ggml-org/bge-m3-Q8_0-GGUF) | ✓ | ✓ | ✗ |
|
||||
| [nomic-ai/nomic-embed-text-v1.5-Q8_0](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF) | ✓ | ✓ | ✗ |
|
||||
|
||||
The embedding models were tested with `llama-embedding` and `Q8_0` files.
|
||||
|
||||
</details>
|
||||
|
||||
## Build Instructions
|
||||
|
||||
@@ -154,39 +212,51 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
|
||||
- Linux or Windows system with Intel hardware (CPU, GPU, or NPU)
|
||||
- **For Intel GPU or NPU Usage**: Install the appropriate hardware drivers for your Intel GPU or NPU. For detailed instructions, see: [Additional Configurations for Hardware Acceleration](https://docs.openvino.ai/2026/get-started/install-openvino/configurations.html).
|
||||
|
||||
- **Linux:**
|
||||
- Git, CMake, and Ninja software tools are needed for building.
|
||||
```bash
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential libcurl4-openssl-dev libtbb12 cmake ninja-build python3-pip curl wget tar
|
||||
```
|
||||
- OpenCL
|
||||
```bash
|
||||
sudo apt install ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd
|
||||
```
|
||||
<details open>
|
||||
<summary><b>Linux</b></summary>
|
||||
|
||||
- **Windows:**
|
||||
- Download and install [Microsoft Visual Studio 2022 Build Tools](https://aka.ms/vs/17/release/vs_BuildTools.exe). During installation, select the **"Desktop development with C++"** workload.
|
||||
Build tools (Git, CMake, Ninja):
|
||||
|
||||
- Install required tools:
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
winget install Git.Git
|
||||
winget install GNU.Wget
|
||||
winget install Ninja-build.Ninja
|
||||
```
|
||||
```bash
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential libcurl4-openssl-dev libtbb12 cmake ninja-build python3-pip curl wget tar
|
||||
```
|
||||
|
||||
- Install **OpenCL** using **vcpkg**:
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
cd C:\
|
||||
git clone https://github.com/microsoft/vcpkg
|
||||
cd vcpkg
|
||||
.\bootstrap-vcpkg.bat
|
||||
.\vcpkg install opencl
|
||||
# Optional but recommended: Integrate vcpkg with Visual Studio / CMake:
|
||||
.\vcpkg integrate install
|
||||
```
|
||||
OpenCL:
|
||||
|
||||
```bash
|
||||
sudo apt install ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Windows</b></summary>
|
||||
|
||||
1. Download and install [Microsoft Visual Studio 2022 Build Tools](https://aka.ms/vs/17/release/vs_BuildTools.exe). During installation, select the **"Desktop development with C++"** workload.
|
||||
2. Install the required tools:
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
winget install Git.Git
|
||||
winget install GNU.Wget
|
||||
winget install Ninja-build.Ninja
|
||||
```
|
||||
|
||||
3. Install **OpenCL** using **vcpkg**:
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
cd C:\
|
||||
git clone https://github.com/microsoft/vcpkg
|
||||
cd vcpkg
|
||||
.\bootstrap-vcpkg.bat
|
||||
.\vcpkg install opencl
|
||||
# Optional but recommended: Integrate vcpkg with Visual Studio / CMake:
|
||||
.\vcpkg integrate install
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
### 1. Install OpenVINO Runtime
|
||||
|
||||
@@ -199,21 +269,25 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
|
||||
|
||||
### 2. Build llama.cpp with OpenVINO Backend
|
||||
|
||||
Clone llama.cpp repo and build :
|
||||
Clone the llama.cpp repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/ggml-org/llama.cpp
|
||||
cd llama.cpp
|
||||
```
|
||||
|
||||
- **Linux:**
|
||||
Build on Linux:
|
||||
|
||||
```bash
|
||||
source /opt/intel/openvino/setupvars.sh
|
||||
cmake -B build/ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON
|
||||
cmake --build build/ReleaseOV --parallel
|
||||
```
|
||||
|
||||
- **Windows:** Open **x64 Native Tools Command Prompt for VS** (so the MSVC toolchain is on `PATH`), then run:
|
||||
<details>
|
||||
<summary><b>Build on Windows</b></summary>
|
||||
|
||||
Open **x64 Native Tools Command Prompt for VS** (so the MSVC toolchain is on `PATH`), then run:
|
||||
|
||||
```cmd
|
||||
C:\Intel\openvino\setupvars.bat
|
||||
@@ -224,9 +298,11 @@ cmake --build build\ReleaseOV --parallel
|
||||
> [!NOTE]
|
||||
> The Windows install path is `C:\Intel\openvino` (no spaces) to avoid quoting problems some CMake/Ninja toolchains have with `C:\Program Files (x86)\...`. Adjust to wherever you installed OpenVINO Runtime. From `cmd`, run `C:\Intel\openvino\setupvars.bat`; from PowerShell, run `& "C:\Intel\openvino\setupvars.ps1"` instead. Once the build is finished you can launch the binaries from any `cmd` or `PowerShell` window after sourcing the matching `setupvars` script for that shell.
|
||||
|
||||
</details>
|
||||
|
||||
#### Ubuntu Build Script
|
||||
|
||||
For Ubuntu24 users, the following shell script automates the prerequisite installs (build tools, OpenCL ICD), the OpenVINO Runtime download/extract/setup, and the Ninja-based llama.cpp build.
|
||||
For Ubuntu users, the following shell script automates the prerequisite installs (build tools, OpenCL ICD), the OpenVINO Runtime download/extract/setup, and the Ninja-based llama.cpp build.
|
||||
Save the following as `build-llamacpp-ov.sh` next to where you want the `llama.cpp` folder to land, then run it:
|
||||
|
||||
```bash
|
||||
@@ -556,81 +632,102 @@ endlocal
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
### 3. Download Sample Model
|
||||
|
||||
Download sample model for testing.
|
||||
Download a sample model for testing.
|
||||
|
||||
```bash
|
||||
# Linux
|
||||
mkdir -p ~/models/
|
||||
wget https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf \
|
||||
-O ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary><b>Windows</b></summary>
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
mkdir C:\models
|
||||
Invoke-WebRequest -Uri https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf -OutFile C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
```
|
||||
|
||||
# Windows Command Line
|
||||
```cmd
|
||||
REM Windows Command Prompt
|
||||
mkdir C:\models
|
||||
curl -L https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf -o C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
### 4. Run Inference with OpenVINO Backend
|
||||
|
||||
When using the OpenVINO backend, the first inference token may have slightly higher latency due to on-the-fly conversion to the OpenVINO graph. Subsequent tokens and runs will be faster.
|
||||
The first run of a model converts and compiles it for the device. This takes seconds on CPU and GPU, and can take minutes on the NPU for large models. Later tokens are faster, and the [compiled model cache](#compiled-model-cache) makes later runs start faster.
|
||||
|
||||
> [!NOTE]
|
||||
> Default context size is set to the model training context, which may be very large. For example, 131072 for Llama 3.2 1B, which may result in lower performance, especially on edge/laptop devices. Use `-c` to limit context size in supported llama.cpp tools for better performance. For example, `-c 512`.
|
||||
> Default context size is set to the model training context, which may be very large. For example, 131072 for Llama 3.2 1B, which may result in lower performance, especially on edge/laptop devices. Use `-c` to limit context size in supported llama.cpp tools for better performance:
|
||||
> - CPU and GPU: for example `-c 1024`.
|
||||
> - NPU: keep the context small, for example `-c 512`, or `-c 256` for larger models. Decode may slow down as the context grows.
|
||||
|
||||
```bash
|
||||
# If device is unset or unavailable, defaults to CPU.
|
||||
# If the system has multiple GPUs, use GPU.0 or GPU.1 to explicitly target a specific GPU.
|
||||
|
||||
# Linux
|
||||
# If the device is unset, the CPU is used.
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
# Optional: enable stateful execution for improved GPU performance (recommended).
|
||||
export GGML_OPENVINO_STATEFUL_EXECUTION=1
|
||||
# To run llama-simple:
|
||||
./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -n 50 "The story of AI is "
|
||||
# To run in chat mode:
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024
|
||||
# To run llama-bench, -fa 1 is needed
|
||||
GGML_OPENVINO_STATEFUL_EXECUTION=1 GGML_OPENVINO_DEVICE=GPU ./build/ReleaseOV/bin/llama-bench -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -fa 1
|
||||
# To run llama-bench:
|
||||
GGML_OPENVINO_DEVICE=GPU ./build/ReleaseOV/bin/llama-bench -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
|
||||
# NPU: keep context small to avoid failures from very large model context windows.
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 512
|
||||
```
|
||||
|
||||
# Windows Command Line
|
||||
<details>
|
||||
<summary><b>Windows</b></summary>
|
||||
|
||||
```cmd
|
||||
REM Windows Command Prompt
|
||||
set GGML_OPENVINO_DEVICE=GPU
|
||||
# Optional: enable stateful execution for improved GPU performance (recommended).
|
||||
set GGML_OPENVINO_STATEFUL_EXECUTION=1
|
||||
REM To run llama-simple:
|
||||
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
|
||||
REM To run in chat mode:
|
||||
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 1024
|
||||
REM To run llama-bench:
|
||||
build\ReleaseOV\bin\llama-bench.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf"
|
||||
|
||||
REM NPU: keep the context small.
|
||||
set GGML_OPENVINO_DEVICE=NPU
|
||||
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 512
|
||||
```
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
$env:GGML_OPENVINO_DEVICE = "GPU"
|
||||
$env:GGML_OPENVINO_STATEFUL_EXECUTION = "1"
|
||||
|
||||
# To run llama-simple
|
||||
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
|
||||
# To run in chat mode:
|
||||
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 1024
|
||||
# To run llama-bench, -fa 1 is needed
|
||||
build\ReleaseOV\bin\llama-bench.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -fa 1
|
||||
|
||||
# NPU: keep context small to avoid failures from very large model context windows.
|
||||
# Windows Command Line
|
||||
set GGML_OPENVINO_DEVICE=NPU
|
||||
# Windows PowerShell
|
||||
# NPU: keep the context small.
|
||||
$env:GGML_OPENVINO_DEVICE = "NPU"
|
||||
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 512
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
> [!NOTE]
|
||||
> On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU. A device that is not available is an error (no fallback to CPU), and the error message lists the available OpenVINO devices with their names. Run `llama-cli --list-devices` to see the valid values: each OpenVINO device shows the `GGML_OPENVINO_DEVICE=<value>` to set, and `(selected)` marks the active one. Select the OpenVINO device with this variable, not with `-dev`. See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html) for more details.
|
||||
> - Select the OpenVINO device with `GGML_OPENVINO_DEVICE`, not with `-dev`.
|
||||
> - On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU.
|
||||
> - To split a model across several GPUs, set a comma-separated list, e.g. `GPU.0,GPU.1`, and use `-sm layer`.
|
||||
> - A device that is not available is an error (no fallback to CPU). The error message lists the available OpenVINO devices with their names.
|
||||
> - Run `llama-cli --list-devices` to see the valid values: each OpenVINO device shows the `GGML_OPENVINO_DEVICE=<value>` to set, and `(selected)` marks the active one.
|
||||
> - See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html) for more details.
|
||||
|
||||
### 5. Docker Build
|
||||
|
||||
You can build and run llama.cpp with OpenVINO backend using Docker.
|
||||
|
||||
<details>
|
||||
<summary><b>Build the images</b></summary>
|
||||
|
||||
```bash
|
||||
# Build the base runtime image with compiled shared libraries and minimal dependencies.
|
||||
docker build -t llama-openvino:base -f .devops/openvino.Dockerfile .
|
||||
@@ -648,9 +745,12 @@ docker build --target=server -t llama-openvino:server -f .devops/openvino.Docker
|
||||
docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --target=server -t llama-openvino:server -f .devops/openvino.Dockerfile .
|
||||
```
|
||||
|
||||
Run llama.cpp with OpenVINO backend Docker container.
|
||||
Save sample models in `~/models` as [shown above](#3-download-sample-model). It will be mounted to the container in the examples below.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Run a container</b> (CPU, GPU, NPU)</summary>
|
||||
|
||||
Run llama.cpp with OpenVINO backend Docker container. Save sample models in `~/models` as [shown above](#3-download-sample-model). It will be mounted to the container in the examples below.
|
||||
|
||||
```bash
|
||||
# Run Docker container
|
||||
@@ -659,19 +759,20 @@ docker run --rm -it -v ~/models:/models llama-openvino:light --no-warmup -c 1024
|
||||
# With Intel GPU access (iGPU or dGPU)
|
||||
docker run --rm -it -v ~/models:/models \
|
||||
--device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
|
||||
--env=GGML_OPENVINO_DEVICE=GPU --env=GGML_OPENVINO_STATEFUL_EXECUTION=1 \
|
||||
--env=GGML_OPENVINO_DEVICE=GPU \
|
||||
llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
|
||||
# With Intel NPU access
|
||||
docker run --rm -it -v ~/models:/models \
|
||||
--device=/dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
|
||||
--device=/dev/accel --group-add=$(stat -c "%g" /dev/accel/accel* | head -n 1) -u $(id -u):$(id -g) \
|
||||
--env=GGML_OPENVINO_DEVICE=NPU \
|
||||
llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
llama-openvino:light --no-warmup -c 512 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
```
|
||||
|
||||
Run Llama.cpp Server with OpenVINO Backend.
|
||||
> [!NOTE]
|
||||
> `llama-server` with OpenVINO backend supports only one chat session/thread, when `GGML_OPENVINO_STATEFUL_EXECUTION=1` is enabled.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Run llama-server</b> (container or local build) and test it</summary>
|
||||
|
||||
```bash
|
||||
# Run the llama-openvino:server Docker container (CPU)
|
||||
@@ -685,9 +786,9 @@ llama-openvino:server --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_
|
||||
|
||||
# Run the llama-openvino:server Docker container with Intel NPU access
|
||||
docker run --rm -it -v ~/models:/models \
|
||||
--device=/dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
|
||||
--device=/dev/accel --group-add=$(stat -c "%g" /dev/accel/accel* | head -n 1) -u $(id -u):$(id -g) \
|
||||
-p 8080:8080 --env=GGML_OPENVINO_DEVICE=NPU \
|
||||
llama-openvino:server --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --host 0.0.0.0
|
||||
llama-openvino:server --no-warmup -c 512 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --host 0.0.0.0
|
||||
|
||||
# Or Using llama-server executable
|
||||
./build/ReleaseOV/bin/llama-server -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --port 8080 -c 1024
|
||||
@@ -706,101 +807,223 @@ curl -X POST "http://localhost:8080/v1/chat/completions" -H "Content-Type: appli
|
||||
-d '{"messages":[{"role":"user","content":"Write a poem about OpenVINO"}],"max_tokens":100}' | jq .
|
||||
```
|
||||
|
||||
## GGML OpenVINO Backend Runtime Configurations
|
||||
</details>
|
||||
|
||||
The OpenVINO backend can be configured using the following environment variables at runtime to control device selection, caching, debugging, and profiling behavior.
|
||||
Boolean flags follow a uniform convention: set to a **positive integer** (e.g. `1`) to enable; unset, empty, `0`, negative, or non-numeric values are treated as disabled.
|
||||
## Runtime Configuration
|
||||
|
||||
| Variable | Type | Default | Description |
|
||||
|-----------------------------------|-----------|------------|-------------------------------------------------------------------------------------------------------------|
|
||||
| `GGML_OPENVINO_DEVICE` | String | `CPU` | Specify the target device (CPU, GPU, NPU). On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU. A device that is not available is an error (no fallback to CPU), and the error message lists the available OpenVINO devices with their names. See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html). When set to **NPU**, static compilation mode is enabled for optimal performance. |
|
||||
| `GGML_OPENVINO_CACHE_DIR` | String | `not set` | Directory for OpenVINO's separate plugin cache. On NPU, this sets `NPUW_CACHE_DIR`. |
|
||||
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` | String | `not set` | Directory for standalone compiled blobs with weights. Dynamic CPU/GPU graphs can import matching blobs on later runs. |
|
||||
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY` | Boolean | `0` | Require an existing compiled blob and skip weight uploads and compilation. Requires Linux or Windows mmap loading and a full dynamic CPU/GPU graph on OpenVINO. |
|
||||
| `GGML_OPENVINO_PREFILL_CHUNK_SIZE`| Integer | `256` | Token chunk size for **NPU** prefill (NPU-only; ignored on CPU/GPU). Must be a positive integer; otherwise the default is used. |
|
||||
| `GGML_OPENVINO_NPU_COMPILE_CONFIG` | String | `not set` | NPU-only compiler mode parameters forwarded to OpenVINO as `NPU_COMPILATION_MODE_PARAMS`, for example `optimization-level=3`. |
|
||||
| `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Keep KV and supported recurrent caches inside the model. Single-slot CPU/GPU execution only. |
|
||||
| `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. |
|
||||
| `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. |
|
||||
| `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | Boolean | `0` | Disable the stateful KV-state sequence-axis relayout (relayout is on by default). It moves the KV state sequence axis from dim 1 to dim 2, so the GPU plugin can append new tokens in place instead of copying the whole state every token, and the reader side no longer transposes the whole accumulated state. Set to `1` to disable. |
|
||||
| `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. |
|
||||
| `GGML_OPENVINO_MEMORY_OPTIMIZE` | Boolean | `0` | Umbrella switch for compile-time memory reductions. Enables `GGML_OPENVINO_REDUCE_COMPILE_MEM` and, on GPU, `GGML_OPENVINO_RELEASE_WEIGHTS` unless those fine-grained variables are explicitly set. |
|
||||
| `GGML_OPENVINO_REDUCE_COMPILE_MEM`| Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` | Reduce compile-time host memory use by streaming weight requantization and avoiding extra weight-node materialization where possible. Set explicitly to override the umbrella switch. |
|
||||
| `GGML_OPENVINO_RELEASE_WEIGHTS` | Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` on GPU | GPU-only. Release host weight buffers after the compiled model cache can reuse the device/plugin copy. Requires stable graph shapes; dynamic workloads that need recompilation should leave this disabled. |
|
||||
| `GGML_OPENVINO_SPILL_DIR` | String | `not set` | Directory for a disk-backed weight buffer. When set, the repacked weight buffer is mapped from an unlinked file on this path instead of anonymous memory, so its pages are reclaimable under memory pressure instead of staying pinned, cutting the load-time host memory peak. Must point at real storage; a tmpfs mount (e.g. `/tmp` on many systems) backs it with RAM and makes the peak worse. |
|
||||
| `GGML_OPENVINO_REQUANT_KQUANT` | String | `not set` | Requantize Q6_K/Q5_K weights (and matching MoE expert weights) to a 4-bit target instead of the default Q8_0_C, trading accuracy for less memory traffic. One of `q4_asym64` (Q6_K/Q5_K only, keeps a real zero point at group 64), `q4_asym64_all` (also requantizes Q4_K), `q4_sym128` (Q6_K/Q5_K only), `q4_sym128_all` (Q4_K too, drops its per-group zero point), or `native` (no requantization). |
|
||||
| `GGML_OPENVINO_PROFILING` | Integer | `0` | `1` logs execution timing; `2` or higher also enables OpenVINO and OpenCL profiling. |
|
||||
| `GGML_OPENVINO_DEBUG_NODE` | String | `not set` | Add the named graph nodes as compiled outputs for debugging. Separate multiple names with commas. |
|
||||
| `GGML_OPENVINO_MOE_OP` | Boolean | `1` | On GPU, set to `0` to keep the unfused GatherMatmul path. |
|
||||
| `GGML_OPENVINO_DUMP_CGRAPH` | Boolean | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. |
|
||||
| `GGML_OPENVINO_DUMP_IR` | Boolean | `0` | Serialize OpenVINO IR files with timestamps. |
|
||||
| `GGML_OPENVINO_DEBUG_INPUT` | Boolean | `0` | Enable input debugging and print input tensor info. |
|
||||
| `GGML_OPENVINO_DEBUG_OUTPUT` | Boolean | `0` | Enable output debugging and print output tensor info. |
|
||||
| `GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS` | Boolean | `0` | Print tensor address map once. |
|
||||
| `GGML_OPENVINO_LOG_UNSUPPORTED_OPS`| Boolean | `0` | Log warning messages with tensor details and rejection reasons for any ops not supported by the OpenVINO backend. Emits at `WARN` level (requires `--log-verbosity >= 2`, enabled by default). |
|
||||
The backend needs no environment variable: it runs on the CPU with its defaults. Set `GGML_OPENVINO_DEVICE` to use the GPU or the NPU. All other variables are optional.
|
||||
|
||||
> [!NOTE]
|
||||
> - `GGML_OPENVINO_STATEFUL_EXECUTION` is an **Experimental** feature for managing caches internally inside the OpenVINO model on CPUs and GPUs. Use a single slot (`-np 1`). KV caches retain the append-based state layout and sequence-axis optimization. Qwen3.5 adds recurrent cache states in their GGML layouts. Qwen3.5 requires an unsplit graph with model caching enabled and no recurrent rollback. A prompt starting at position 0 resets all states. State save/restore, sequence rewind, context shift, and mid-sequence graph replacement are unsupported. Stateful execution is not effective on NPUs.
|
||||
> - `GGML_OPENVINO_LOG_UNSUPPORTED_OPS` emits logs at `WARN` level (`GGML_LOG_WARN`), which requires application log verbosity `--log-verbosity >= 2` (or `-lv 2`).
|
||||
> - With `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1`, use the same compilation settings as the export run. One directory can hold blobs for different models and settings; `GGML_OPENVINO_SPILL_DIR` does not affect the cache key and is ignored in cache-only mode. See [Compiled model cache](../../ggml/src/ggml-openvino/README.md) for the workflow and restrictions.
|
||||
```bash
|
||||
export GGML_OPENVINO_DEVICE=GPU # String
|
||||
export GGML_OPENVINO_PROFILING=1 # Integer
|
||||
export GGML_OPENVINO_EXACT_ZERO_POINT=1 # Boolean: 1 (on) or 0 (off)
|
||||
|
||||
# Show the variables in effect (log verbosity 4)
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024 -lv 4 2>&1 | grep "settings from the environment"
|
||||
```
|
||||
|
||||
- The backend reads the variables once, at start. Unset or empty means the default.
|
||||
- To lower the memory a model needs, see [Tips for Memory Optimization](../../ggml/src/ggml-openvino/dev_doc/OpenVINO-ENV-VARS.md#tips-for-memory-optimization).
|
||||
- A value that is not valid stops the backend. A `GGML_OPENVINO_*` name that the backend does not know gives a warning:
|
||||
|
||||
```text
|
||||
ggml-openvino: GGML_OPENVINO_EXACT_ZERO_POINT=yes is not valid, use 1 (on) or 0 (off)
|
||||
ggml-openvino: unknown environment variable GGML_OPENVINO_DEVCE is ignored
|
||||
```
|
||||
|
||||
### Basic
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_DEVICE` | all | String | `CPU` | Target device: `CPU`, `GPU` or `NPU`. For several GPUs, see the [device notes](#4-run-inference-with-openvino-backend). Examples: `GGML_OPENVINO_DEVICE=NPU`, `GGML_OPENVINO_DEVICE=GPU.0,GPU.1`. |
|
||||
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` | all | String | `not set` | Directory for the backend's cache of compiled models. A later start with the same model and settings loads faster. See [Compiled model cache](#compiled-model-cache). Example: `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache`. |
|
||||
| `GGML_OPENVINO_NPU_CONFIG` | NPU | String | `not set` | OpenVINO properties for the NPU plugin, as comma-separated `KEY=VALUE` pairs. See [Plugin properties](#plugin-properties). Example: `GGML_OPENVINO_NPU_CONFIG="NPU_COMPILER_TYPE=PLUGIN"`. |
|
||||
| `GGML_OPENVINO_GPU_CONFIG` | GPU | String | `not set` | OpenVINO properties for the GPU plugin, as comma-separated `KEY=VALUE` pairs. See [Plugin properties](#plugin-properties). Example: `GGML_OPENVINO_GPU_CONFIG="INFERENCE_PRECISION_HINT=f16"`. |
|
||||
| `GGML_OPENVINO_CPU_CONFIG` | CPU | String | `not set` | OpenVINO properties for the CPU plugin, as comma-separated `KEY=VALUE` pairs. See [Plugin properties](#plugin-properties). Example: `GGML_OPENVINO_CPU_CONFIG="INFERENCE_PRECISION_HINT=f16"`. |
|
||||
| `GGML_OPENVINO_PROFILING` | all | Integer | `0` | `1` logs execution timing. `2` or higher also turns on OpenVINO and OpenCL profiling. Example: `GGML_OPENVINO_PROFILING=1`. |
|
||||
|
||||
### Plugin properties
|
||||
|
||||
`GGML_OPENVINO_NPU_CONFIG`, `GGML_OPENVINO_GPU_CONFIG` and `GGML_OPENVINO_CPU_CONFIG` pass OpenVINO plugin properties to the device in their name.
|
||||
|
||||
```bash
|
||||
# Format: comma-separated KEY=VALUE pairs. The backend uses the variable of the selected device only.
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_NPU_CONFIG="NPU_COMPILER_TYPE=PLUGIN,COMPILATION_NUM_THREADS=4"
|
||||
|
||||
# CPU: compute in FP16, as the GPU and the NPU do. This helps to debug the accuracy of a quantized model.
|
||||
export GGML_OPENVINO_DEVICE=CPU
|
||||
export GGML_OPENVINO_CPU_CONFIG="INFERENCE_PRECISION_HINT=f16"
|
||||
```
|
||||
|
||||
- Property names and values are those of OpenVINO, for example `YES` and `NO`. They are not the `1` and `0` of the `GGML_OPENVINO_*` variables.
|
||||
- The backend applies the variable last, so it overrides the properties that the backend sets.
|
||||
- OpenVINO checks the properties when the model loads. If it rejects one, the load fails and the error names the variable.
|
||||
- All properties and their values are in the OpenVINO documentation: [NPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.html), [GPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html), [CPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device.html).
|
||||
- NPU: the backend sets several properties by default. They are listed in the [developer documentation](../../ggml/src/ggml-openvino/dev_doc/OpenVINO-ENV-VARS.md#npu-plugin-properties).
|
||||
- GPU and CPU: the backend sets no property, except `CACHE_DIR` and `CACHE_MODE=OPTIMIZE_SIZE` when `GGML_OPENVINO_CACHE_DIR` is set.
|
||||
|
||||
`NPU_COMPILER_TYPE` selects the NPU compiler. Both forms below work; use the first one. If both are set, `GGML_OPENVINO_NPU_CONFIG` takes precedence.
|
||||
|
||||
```bash
|
||||
# Values: DRIVER (the compiler in the NPU driver; backend default), PLUGIN (the compiler in OpenVINO),
|
||||
# PREFER_PLUGIN (OpenVINO default: PLUGIN when it is available, DRIVER if not).
|
||||
export GGML_OPENVINO_NPU_CONFIG="NPU_COMPILER_TYPE=PLUGIN" # recommended form
|
||||
export GGML_OPENVINO_NPU_COMPILER_TYPE=PLUGIN # older name, still works
|
||||
```
|
||||
|
||||
### Advanced tuning
|
||||
|
||||
The defaults fit most models. The variables below trade one property for another, for example speed for accuracy.
|
||||
|
||||
<details>
|
||||
<summary><b>Advanced tuning variables</b></summary>
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_PREFILL_CHUNK_SIZE` | NPU | Integer | `256` | Number of tokens that the NPU processes in one prefill step. A larger value can make long prompts faster, and makes the first load longer. Zero or a negative value means the default. Example: `GGML_OPENVINO_PREFILL_CHUNK_SIZE=512`. |
|
||||
| `GGML_OPENVINO_NPU_TOKEN_EMBD_QUANT` | NPU | String | `q8_0_c` | How a `Q6_K` `token_embd.weight` is stored for the NPU: `q8_0_c` (8-bit, one scale per row), `f16`, `q4_0_128` (4-bit, one scale per 128 weights) or `q4_0_c` (4-bit, one scale per row). The 4-bit layouts use less memory at some cost in accuracy; `f16` uses more memory. Keep the default unless memory is tight. |
|
||||
| `GGML_OPENVINO_GDN_DECOMPOSE` | all | Boolean | auto | Translate `GATED_DELTA_NET` (Qwen3.5) into MatMul operations instead of the OpenVINO `GatedDeltaNet` op. Unset: on for the Lunar Lake NPU, off elsewhere. `0` turns it off, `1` turns it on for any device. Try `1` for Qwen3.5 on the Panther Lake NPU. |
|
||||
| `GGML_OPENVINO_MANUAL_GQA_ATTN` | CPU, GPU | Boolean | on for GPU | Build grouped-query attention from separate MatMul operations. Unset: on for GPU, off for CPU and NPU. `1` turns it on, `0` turns it off. Try `1` on the CPU for faster decode. |
|
||||
| `GGML_OPENVINO_REQUANT_KQUANT` | CPU, GPU | String | `not set` | Requantize `Q6_K` and `Q5_K` weights to a 4-bit layout instead of `Q8_0_C`: faster decode for lower accuracy. One of `q4_asym64`, `q4_asym64_all` (also `Q4_K`), `q4_sym128`, `q4_sym128_all` (also `Q4_K`) or `native` (no requantization). On GPU with `Q4_K_M` files use `q4_asym64_all`, because `q4_asym64` does not compile there. |
|
||||
| `GGML_OPENVINO_EXACT_ZERO_POINT` | CPU, GPU | Boolean | `0` | Keep the exact zero point of asymmetric weights (`Q4_1`, `Q4_K`, `Q5_1`, `Q5_K`) instead of rounding it to an integer. This improves accuracy and lowers speed. Use it when accuracy matters more than speed. |
|
||||
|
||||
</details>
|
||||
|
||||
### Compiled model cache
|
||||
|
||||
The cache keeps compiled models on disk, so later starts of the same model load faster. Use the same model and the same settings in every run.
|
||||
|
||||
```bash
|
||||
# CPU or GPU: the first run compiles the model and writes the cache. Later runs read it.
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024
|
||||
|
||||
# CPU or GPU, later runs (optional): require the cached model and skip the weight upload and the compilation.
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1
|
||||
|
||||
# NPU: add the weightless bank.
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache
|
||||
export GGML_OPENVINO_NPU_WEIGHTLESS_BANK=1
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 512
|
||||
|
||||
# NPU, later runs (optional): use less memory. The run fails if a cached model is missing.
|
||||
export GGML_OPENVINO_SELF_CONTAINED_MMAP=1
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary><b>Cache variables</b></summary>
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY` | CPU, GPU | Boolean | `0` | Require an existing cached model and skip the weight upload and the compilation. Needs mmap model loading (the default on Linux and Windows) and a model that runs fully on OpenVINO. |
|
||||
| `GGML_OPENVINO_NPU_WEIGHTLESS_BANK` | NPU | Boolean | `0` | Requires `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR`. Write the prefill and decode models without weights, plus one shared weights file. Later runs read the weights from that file and keep one copy in memory. |
|
||||
| `GGML_OPENVINO_SELF_CONTAINED_MMAP` | NPU | Boolean | `0` | Later runs only. Requires `GGML_OPENVINO_NPU_WEIGHTLESS_BANK=1` and an existing cache. Use the memory-mapped GGUF file as the model buffer, which lowers the host memory peak. Fails if a cached model is missing; run once without it to build the cache. |
|
||||
| `GGML_OPENVINO_CACHE_DIR` | all | String | `not set` | Directory for the cache of the OpenVINO plugin itself. `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` is the recommended cache. |
|
||||
|
||||
- One directory can hold the cached models of different models and settings.
|
||||
- NPU: each prefill length enters the cache in a run without `GGML_OPENVINO_SELF_CONTAINED_MMAP`.
|
||||
- See [Compiled model cache](../../ggml/src/ggml-openvino/README.md#compiled-model-cache) for the workflow and the restrictions.
|
||||
|
||||
</details>
|
||||
|
||||
### Deprecated
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_STATEFUL_EXECUTION` | CPU, GPU | Boolean | `0` | **Deprecated**, to be removed. Keep the KV cache inside the OpenVINO model. Use the default, stateless execution (`0`). |
|
||||
|
||||
<details>
|
||||
<summary>Limits of stateful execution</summary>
|
||||
|
||||
- CPU and GPU only; it has no effect on the NPU.
|
||||
- One slot only (`-np 1`).
|
||||
- State save and restore, sequence rewind, context shift and recurrent rollback are not supported.
|
||||
- Qwen3.5 needs a graph that runs fully on OpenVINO, with model caching on.
|
||||
- A prompt that starts at position 0 resets all states.
|
||||
|
||||
</details>
|
||||
|
||||
### Debugging and development
|
||||
|
||||
Debug output, switches that turn a default optimization off, experimental options and older variable names are in the [developer documentation](../../ggml/src/ggml-openvino/dev_doc/OpenVINO-ENV-VARS.md).
|
||||
|
||||
### Example Usage
|
||||
|
||||
#### GPU Inference with Profiling
|
||||
|
||||
```bash
|
||||
# If the system has multiple GPUs, use GPU.0 or GPU.1 to explicitly target a specific GPU.
|
||||
|
||||
# Linux
|
||||
export GGML_OPENVINO_CACHE_DIR=/tmp/ov_cache
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache
|
||||
export GGML_OPENVINO_PROFILING=1
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
export GGML_OPENVINO_STATEFUL_EXECUTION=1
|
||||
|
||||
./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -n 50 "The story of AI is "
|
||||
```
|
||||
|
||||
# Windows Command Line
|
||||
set GGML_OPENVINO_CACHE_DIR=C:\tmp\ov_cache
|
||||
<details>
|
||||
<summary><b>Windows</b></summary>
|
||||
|
||||
```cmd
|
||||
REM Windows Command Prompt
|
||||
set GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=C:\tmp\ov_cache
|
||||
set GGML_OPENVINO_PROFILING=1
|
||||
set GGML_OPENVINO_DEVICE=GPU
|
||||
set GGML_OPENVINO_STATEFUL_EXECUTION=1
|
||||
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
|
||||
```
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
$env:GGML_OPENVINO_CACHE_DIR = "C:\tmp\ov_cache"
|
||||
$env:GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR = "C:\tmp\ov_cache"
|
||||
$env:GGML_OPENVINO_PROFILING = "1"
|
||||
$env:GGML_OPENVINO_DEVICE = "GPU"
|
||||
$env:GGML_OPENVINO_STATEFUL_EXECUTION = "1"
|
||||
|
||||
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
|
||||
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
## Known Limitations
|
||||
|
||||
**General (all devices)**
|
||||
|
||||
- Llama.cpp OpenVINO backend currently supports a subset of GGML ops and text-only models. Unsupported ops or unsupported op shapes/cases fail during OpenVINO translation.
|
||||
- Multimodal features (audio/image/video) are a work in progress.
|
||||
- Limited Embedding and Reranking model support.
|
||||
- Embedding models: reranking models are not validated.
|
||||
- Llama.cpp tool coverage across CPU/GPU/NPU is not uniform.
|
||||
|
||||
**Tool-specific**
|
||||
|
||||
- `llama-bench`: requires `-fa 1` (flash-attention).
|
||||
- `llama-cli --context-shift`: stateless only (`GGML_OPENVINO_STATEFUL_EXECUTION=0`). In stateful mode the KV cache is owned by the OpenVINO model and cannot be shifted externally.
|
||||
- `llama-server`: only one chat session/thread when `GGML_OPENVINO_STATEFUL_EXECUTION=1`.
|
||||
- Mamba-based models (e.g. Falcon-H1): `llama-perplexity` with several sequences per batch fails; set `-b` to the same value as `-c`. GPU accuracy is slightly lower than CPU.
|
||||
|
||||
**GPU-specific**
|
||||
|
||||
- `llama-server -np > 1`: concurrent requests are batched together, which may slightly reduce per-request throughput.
|
||||
- Multiple GPUs (`GGML_OPENVINO_DEVICE=GPU.0,GPU.1` with `-sm layer`):
|
||||
- `-sm tensor` is not supported.
|
||||
- Copies between GPUs go through host memory. There is no asynchronous copy and no peer-to-peer copy.
|
||||
- Only GPUs can be combined. A list with several NPUs, or with a GPU and the CPU or the NPU, is rejected.
|
||||
- MoE models use the GPU plugin's fused MoE op, which runs in f16 only:
|
||||
- granitemoe models (e.g. `granite-3.1-1b-a400m`) give wrong output.
|
||||
- **Workaround:** `GGML_OPENVINO_MOE_OP=0`. Output is correct, but prompt processing is much slower.
|
||||
- `GGML_OPENVINO_GPU_CONFIG="INFERENCE_PRECISION_HINT=f32"` fails to compile.
|
||||
- `llama-perplexity` with MoE models: set `-b` to the same value as `-c`, so that each batch holds one sequence.
|
||||
|
||||
**NPU-specific**
|
||||
|
||||
- Default context resolves to the model's training context (e.g. 131072 for Llama 3.2 1B), which can OOM or fail or degrade performance on NPU. Inspect the resolved value with `-lv 3`.
|
||||
- **Workaround:** Pass an explicit `-c <N>`, e.g. `-c 1024`.
|
||||
- **Workaround:** Pass an explicit `-c <N>`, e.g. `-c 512`.
|
||||
- NPU device uses a static graph with a fixed prefill chunk size (defaults to 256), configurable with `GGML_OPENVINO_PREFILL_CHUNK_SIZE`. Large prefill/batch settings may need tuning.
|
||||
- `llama-server -np > 1` (multiple parallel sequences) is not supported.
|
||||
- `llama-perplexity`: requires `-b 512` or smaller.
|
||||
- Flash attention must stay on, which is the default (`-fa auto`). With `-fa off`, the run fails on the NPU.
|
||||
- `llama-embedding` fails on the NPU.
|
||||
- Mamba-based models (e.g. Falcon-H1) fail on the NPU with `Failed to set tensor` (element type mismatch).
|
||||
- Lunar Lake NPU: decode slows down sharply when the context is large. The context size at which this starts depends on the model.
|
||||
- **Workaround:** Pass the smallest context that fits the use, e.g. `-c 512`, or `-c 256` for larger models.
|
||||
- Panther Lake NPU: the load of a Qwen3.5 model can end with a segmentation fault in the NPU compiler of OpenVINO 2026.4.1.
|
||||
- **Workaround:** Set `GGML_OPENVINO_NPU_CONFIG=COMPILATION_NUM_THREADS=1`.
|
||||
- Panther Lake NPU: `GGML_OPENVINO_NPU_TOKEN_EMBD_QUANT=f16` and `q4_0_128` can give wrong output. Keep the default.
|
||||
- Lunar Lake NPU: `granite-4.0-1b` gives wrong output. The activations of the model exceed the FP16 range, and the NPU computes in FP16. The model runs correctly on CPU and GPU.
|
||||
- Lunar Lake NPU: `gemma-4-12B` does not run. The first inference fails with `ZE_RESULT_ERROR_UNINITIALIZED` from the NPU driver. The model runs on CPU and GPU.
|
||||
- The first load of a model on the NPU compiles it for the device, which can take minutes for large models.
|
||||
- **Workaround:** Use the [compiled model cache](#compiled-model-cache).
|
||||
|
||||
> [!NOTE]
|
||||
> The OpenVINO backend is actively under development. Fixes and improvements are underway, and this document will continue to be updated.
|
||||
|
||||
@@ -15,6 +15,9 @@ Supported models:
|
||||
- [Fish Audio S2 Pro](#fish-audio-s2-pro)
|
||||
- [YuE2](#yue2) (songs)
|
||||
- [MiniMax Music 3](#minimax-music-3) (songs)
|
||||
- [Kokoro](#kokoro)
|
||||
- [tiny-voca](#tiny-voca)
|
||||
- [VoxCPM2](#voxcpm2)
|
||||
|
||||
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
|
||||
|
||||
@@ -23,6 +26,7 @@ The sections below list the params supported by each model, with `llama-tts` exa
|
||||
Available params:
|
||||
- `--tts-lang` can be `zh`, `en`, `de`, `it`, `pt`, `es`, `ja`, `ko`, `fr`, `ru` (default: `en`)
|
||||
- `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3)
|
||||
- `--tts-speaker-text` is the transcript of the speaker file. With it, the speech continues from the reference codes (the "ICL" mode of the reference), which clones the voice more closely. It needs a clip of 30 s or less and an mmproj converted with the speech tokenizer encoder, otherwise it falls back to the speaker embedding only
|
||||
|
||||
Example usage:
|
||||
|
||||
@@ -34,6 +38,16 @@ llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Voice cloning with the transcript of the reference:
|
||||
|
||||
```sh
|
||||
llama-tts -hf ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF \
|
||||
-p "Hello world" \
|
||||
--tts-speaker-file speaker.wav \
|
||||
--tts-speaker-text "The words spoken in speaker.wav." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
## Pocket TTS
|
||||
|
||||
Available params:
|
||||
@@ -352,3 +366,147 @@ python convert_hf_to_gguf.py MiniMax-Music3 --mmproj --outfile mmproj-minimax-mu
|
||||
```
|
||||
|
||||
The mmproj holds the depth decoder and the DiT layers (in the `--outtype` type), and the condition encoder, the small DiT projections and the DAC decoder, always in F32.
|
||||
|
||||
## Kokoro
|
||||
|
||||
Available params:
|
||||
- `--tts-voice` selects one of the voices shipped with the weights (default: `af_heart`); `llama-tts` lists them when the name is unknown. The first letter of a voice is its language: `a` American English, `b` British English, `e` Spanish, `f` French, `h` Hindi, `i` Italian, `p` Brazilian Portuguese, `j` Japanese, `z` Mandarin
|
||||
- `--tts-lang` can be `en-us` or `en-gb`; when it is not given, the voices named `b*` speak British English and those named `a*` American English
|
||||
- Note: `--tts-speaker-file` is not supported, the model has no voice cloning
|
||||
- Note: sampling params are not used, the model is not autoregressive. `--seed` only changes the noise of the vocoder source
|
||||
- `speed` (`llama-server` only) changes the speaking rate, e.g. `1.5`; `loudness` is not supported
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m kokoro.gguf \
|
||||
-mm mmproj-kokoro.gguf \
|
||||
-p "Hello world" \
|
||||
--tts-voice am_michael \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
The model speaks phonemes. Only English has a text to phonemes step (the model's `tts` chat template, with the pronunciation dictionary stored in the model):
|
||||
- the pronunciation of a word missing from the dictionary (mostly rare names) is predicted from its spelling, by a model that the converter learns from the dictionary. The reference uses espeak-ng for these words; the prediction is about as accurate, but it is not the same, so such a word can sound different from the reference
|
||||
- a text longer than 510 phonemes is split at sentence ends and spoken segment by segment
|
||||
|
||||
**Other languages, or your own phonemes:**
|
||||
|
||||
The prompt of a voice of another language must already be phonemes, in the symbols of the model (see `vocab` in its `config.json`). Run the text to phonemes step in your own code, for example with [espeak-ng](https://github.com/espeak-ng/espeak-ng), which the reference uses for Spanish, French, Hindi, Italian and Brazilian Portuguese:
|
||||
|
||||
```sh
|
||||
PHONEMES=$(echo "El gobierno anunció nuevas medidas." | espeak-ng -q --ipa -v es --stdin | tr '\n' ' ')
|
||||
llama-tts -m kokoro.gguf \
|
||||
-mm mmproj-kokoro.gguf \
|
||||
-p "$PHONEMES" \
|
||||
--tts-voice ef_dora \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Notes:
|
||||
- espeak-ng drops the punctuation, which the model uses for pauses: phonemize the text between the marks and put them back for better prosody
|
||||
- the reference also replaces a few sounds by single symbols (e.g. `tʃ` by `ʧ`), see `espeak.py` of [misaki](https://github.com/hexgrad/misaki)
|
||||
- with `llama-server`, send `"raw_prompt": true` to pass phonemes with any voice, including the English ones
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
Download `config.json`, `kokoro-v1_0.pth` and the `voices/*.pt` files you want from [Kokoro-82M](https://huggingface.co/hexgrad/Kokoro-82M), every voice in the directory is stored in the mmproj:
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/Kokoro-82M --outfile kokoro.gguf
|
||||
python convert_hf_to_gguf.py path/to/Kokoro-82M --mmproj --outfile mmproj-kokoro.gguf
|
||||
```
|
||||
|
||||
The pronunciation dictionary and the part-of-speech tagger of the `tts` template come from the `misaki` and `nltk` python packages. The converter also learns the spelling-to-sound model for unknown words from that dictionary, which adds about 15 seconds:
|
||||
|
||||
```sh
|
||||
pip install misaki nltk
|
||||
python -m nltk.downloader averaged_perceptron_tagger_eng
|
||||
```
|
||||
|
||||
Without them the converter writes no `tts` template, and the prompt must be a phoneme string (as produced by misaki).
|
||||
|
||||
## tiny-voca
|
||||
|
||||
[tiny-voca](https://huggingface.co/n4ze3m/tiny-voca): a 4.5M-parameter English TTS model. A phoneme encoder (the text model), then a duration predictor, two frame transformers and a source-filter synthesizer (in the mmproj).
|
||||
|
||||
Available params:
|
||||
- `--tts-voice` can be `M2` (default) or `F2`
|
||||
- Note: `--tts-speaker-file` is not supported, the model has no voice cloning
|
||||
- Note: sampling params are not used, the model is not autoregressive. `--seed` only changes the noise of the synthesizer
|
||||
- `speed` (`llama-server` only) changes the speaking rate, from `0.25` to `4`; `loudness` is not supported
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m tiny-voca.gguf \
|
||||
-mm mmproj-tiny-voca.gguf \
|
||||
-p "Hello! The meeting moved to 3:30 pm." \
|
||||
--tts-voice F2 \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
The model speaks phonemes. The text to phonemes step is the model's `tts` chat template, a port of [dizione](https://github.com/n4ze3m/dizione) (the reference one), with its pronunciation dictionary and tables stored in the model:
|
||||
- the choice of a pronunciation for each word: exceptions and names, the dictionary, function words and their sentence stress, words spelled alike and read differently ("I read it yesterday", "to read"), hyphenated words, capitals read letter by letter, mixed case ("iPhone"), stretched words ("soooo"), accents, chat spellings ("lol", "btw")
|
||||
- a word missing from the dictionary is built from a known word when it can be (endings, prefixes, compounds), else it goes through the unknown-word model of dizione (stored in the mmproj) and its conversion rules
|
||||
- a pronunciation can be given in the text, in CMUdict notation or as phonemes: `Say {HH AH0 L OW1} to Samir{s ə m ˈɪɹ}`
|
||||
- the text rules are ported too: numbers, money, times, dates, telephone numbers, versions, codes, units, fractions, ranges, math signs and comparisons, web addresses and file names, abbreviations, chat spellings
|
||||
|
||||
With `llama-server`, send `"raw_prompt": true` to pass phonemes directly, in the format printed by dizione: symbols separated by spaces, `<sp>` between words, a stress mark (`ˈ` or `ˌ`) before a stressed vowel, e.g. `h ə l ˈoʊ <sp> w ˈɜː l d .`
|
||||
|
||||
Each sentence is spoken as one utterance. The template splits the text as the reference does (before a capital or a figure, not after a title or an initial), ends each sentence with `<eos>`, and adds a full stop when the text ends without a mark. Phonemes without `<eos>` are cut after each `.`, `?`, `!` or `…`, and a sentence without a final mark gets a full stop.
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
Convert the root of the repository (it needs `config.json`, `tiny-voca.pt` and `phonemizer/lexicon.tsv`). The other tables of the `tts` template come from the `dizione` python package; without it the converter writes no `tts` template, and the prompt must be phonemes:
|
||||
|
||||
```sh
|
||||
pip install git+https://github.com/n4ze3m/dizione
|
||||
```
|
||||
|
||||
```sh
|
||||
python convert_hf_to_gguf.py path/to/tiny-voca --outfile tiny-voca.gguf
|
||||
python convert_hf_to_gguf.py path/to/tiny-voca --mmproj --outfile mmproj-tiny-voca.gguf
|
||||
```
|
||||
|
||||
## VoxCPM2
|
||||
|
||||
[VoxCPM2](https://huggingface.co/openbmb/VoxCPM2): a MiniCPM4 backbone (2B) makes one hidden state per frame, without an audio token. The mmproj turns it into a patch of 4 continuous latents: FSQ, an 8-layer residual LM (run with libllama, one context per sequence), then the LocDiT flow matching. The causal AudioVAE decodes the latents to 48 kHz audio.
|
||||
|
||||
Available params:
|
||||
- The language is the one of the text, `--tts-lang` is not used. A voice description in parentheses at the start of the text designs the voice, for example `(A young woman, gentle and warm)Hello!`
|
||||
- Voice cloning: `--tts-speaker-file` with the reference clip (resampled to 16 kHz, truncated to the first 30 s). `--tts-speaker-text` is not used, the transcript of the clip is not needed
|
||||
- `--temp` scales the noise of the flow matching (default 1.0), `--top-k` and `--top-p` are not used
|
||||
- The flow matching uses 10 Euler steps and classifier-free guidance with scale 2.0, like the reference. `llama-server` takes `flow_steps` (2 to 32) and `cfg_scale` to change them
|
||||
- Each frame is 4 latents, 7680 samples (160 ms). Generation stops on the stop head of the model, at least 4 frames, at most 6 frames per text token + 10 (the limit of the reference)
|
||||
- Not supported yet: the continuation mode of the reference (prompt audio with its transcript)
|
||||
- Not supported yet: the bad case retry of the reference (`retry_badcase`, a new seed when the audio reaches 6 frames per text token). Short prompts may then run long and repeat some words, try another seed
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m VoxCPM2.gguf \
|
||||
-mm mmproj-VoxCPM2.gguf \
|
||||
-p "(A calm male narrator)The quick brown fox jumps over the lazy dog." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Voice cloning:
|
||||
|
||||
```sh
|
||||
llama-tts -m VoxCPM2.gguf \
|
||||
-mm mmproj-VoxCPM2.gguf \
|
||||
--tts-speaker-file reference.wav \
|
||||
-p "Hello, this is a cloned voice." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The mmproj holds LocEnc, LocDiT and the residual LM layers (loaded with libllama as a Granite text model with RoPE off) in the `--outtype` type, and the AudioVAE (`audiovae.pth`, encoder for voice cloning, decoder for the audio) always in F32:
|
||||
|
||||
```sh
|
||||
hf download openbmb/VoxCPM2 --local-dir VoxCPM2
|
||||
python convert_hf_to_gguf.py VoxCPM2 --outfile VoxCPM2.gguf
|
||||
python convert_hf_to_gguf.py VoxCPM2 --mmproj --outfile mmproj-VoxCPM2.gguf
|
||||
```
|
||||
|
||||
+1
-1
@@ -90,7 +90,7 @@ Legend:
|
||||
| PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ |
|
||||
| POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
|
||||
+128
-128
@@ -1155,134 +1155,134 @@
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[1,8,1,3],nr23=[1,1],r=2,v=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -9,6 +9,8 @@ Jinja has no regex support, and exposing `std::regex` / Python `re` would not he
|
||||
|
||||
A third function, `tts_dict(name, key)`, gives the template access to dictionaries stored in the model (e.g. a pronunciation lexicon).
|
||||
|
||||
A fourth one, `tts_g2p(word)`, gives the pronunciation of a word by the letter-to-sound model of the mmproj, when the caller provides one (see below).
|
||||
|
||||
Implementations:
|
||||
- C++: [common/chat-tts-norm.cpp](../common/chat-tts-norm.cpp), injected into task templates by `common_chat_template_render()` in [common/chat.cpp](../common/chat.cpp). They are not available to chat templates.
|
||||
- Python reference: [scripts/chat_tts_norm.py](../scripts/chat_tts_norm.py), for use with `jinja2` (`env.globals["tts_norm_scan"] = chat_tts_norm.tts_norm_scan`)
|
||||
@@ -87,6 +89,17 @@ The lookup is exact: no case folding and no normalization, do them in the templa
|
||||
|
||||
The GGUF lists the names of the dictionaries in `tokenizer.tts_dicts`. Each one is stored as two arrays of strings of the same length: `tokenizer.tts_dict.<name>.keys`, sorted by the UTF-8 bytes of the key, and `tokenizer.tts_dict.<name>.values`. A value is a JSON document (string, number, list, object, ...) and is returned decoded. `GGUFWriter.add_tts_dicts(dicts)` writes all of this.
|
||||
|
||||
### `tts_g2p(word)`
|
||||
|
||||
Pronunciation of `word` by a letter-to-sound model, as phonemes separated by spaces, in the notation of the model (e.g. CMUdict with stress digits for tiny-voca), or `none`. The template converts it to the symbols of the model. It is `none` when the model has no letter-to-sound model, or when the caller gives none: `common_chat_get_tts_prompt()` takes it as an optional callback, `llama-tts` and `llama-server` pass `mtmd_gen_audio_g2p()`. A template must keep working without it, e.g. by spelling the word.
|
||||
|
||||
```jinja
|
||||
{%- set g = tts_g2p(word) -%}
|
||||
{%- if g is none -%}{{ spell(word) }}{%- else -%}{{ from_cmudict(word, g.split(' ')) }}{%- endif -%}
|
||||
```
|
||||
|
||||
There is no Python reference for it: a test passes its own function (`env.globals["tts_g2p"]`).
|
||||
|
||||
## Pattern syntax
|
||||
|
||||
Patterns operate on Unicode codepoints. Invalid UTF-8 bytes (C++ only) are seen as U+FFFD, one per byte, and preserved in the output.
|
||||
|
||||
@@ -8,10 +8,10 @@ extern "C" {
|
||||
|
||||
#define RPC_PROTO_MAJOR_VERSION 10
|
||||
#define RPC_PROTO_MINOR_VERSION 0
|
||||
#define RPC_PROTO_PATCH_VERSION 0
|
||||
#define RPC_PROTO_PATCH_VERSION 1
|
||||
|
||||
#ifdef __cplusplus
|
||||
static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
|
||||
static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
|
||||
#endif
|
||||
|
||||
#define GGML_RPC_MAX_SERVERS 16
|
||||
|
||||
+8
-6
@@ -429,6 +429,8 @@ extern "C" {
|
||||
|
||||
GGML_OP_GLU,
|
||||
|
||||
GGML_OP_SLEEP,
|
||||
|
||||
GGML_OP_COUNT,
|
||||
};
|
||||
|
||||
@@ -1314,12 +1316,6 @@ extern "C" {
|
||||
|
||||
// indirect matrix multiplication
|
||||
// an id of -1 skips the slot and zeroes the matching row of the result
|
||||
//
|
||||
// [TAG_MUL_MAT_ID_DUPLICATE_ID]
|
||||
// note: this op has undefined behaviour if a given `a` is referenced more than once for a given `b`
|
||||
// (i.e. duplicate experts per token are not guaranteed to be supported in all backends)
|
||||
// TODO: add backend support: https://github.com/ggml-org/llama.cpp/pull/29998#issuecomment-6153742533
|
||||
// the CPU backend skips negative ids, their rows of the result are zero
|
||||
GGML_API struct ggml_tensor * ggml_mul_mat_id(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * as,
|
||||
@@ -2599,6 +2595,12 @@ extern "C" {
|
||||
int n_tasks,
|
||||
void * userdata);
|
||||
|
||||
// Busy-wait operator for scheduling and backend testing.
|
||||
GGML_API struct ggml_tensor * ggml_sleep(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a, // wait after this tensor has executed, return value is simply a full view of this tensor
|
||||
int32_t us); // amount of time to wait in microseconds
|
||||
|
||||
// loss function
|
||||
|
||||
GGML_API struct ggml_tensor * ggml_cross_entropy_loss(
|
||||
|
||||
@@ -231,7 +231,7 @@ if (GGML_OPENMP_FETCH)
|
||||
message(FATAL_ERROR "GGML_OPENMP_FETCH currently requires Clang on Windows")
|
||||
endif()
|
||||
|
||||
set(GGML_OPENMP_LLVM_VERSION "20.1.8")
|
||||
set(GGML_OPENMP_LLVM_VERSION "22.1.8")
|
||||
string(REGEX MATCH "^[0-9]+" GGML_OPENMP_LLVM_VERSION_MAJOR "${GGML_OPENMP_LLVM_VERSION}")
|
||||
string(REGEX MATCH "^[0-9]+" GGML_OPENMP_COMPILER_VERSION_MAJOR "${CMAKE_C_COMPILER_VERSION}")
|
||||
if (NOT GGML_OPENMP_COMPILER_VERSION_MAJOR STREQUAL GGML_OPENMP_LLVM_VERSION_MAJOR)
|
||||
@@ -242,11 +242,11 @@ if (GGML_OPENMP_FETCH)
|
||||
if (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(amd64|x86_64)$")
|
||||
set(GGML_OPENMP_ARCH "x64")
|
||||
set(GGML_OPENMP_INSTALLER_SUFFIX "win64")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "3197846a2b19063687dd56e93e34cd941e3548d907f23a6131571321bdf9fe7b")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "16e5709785fef73c854646241c4a92c5cd574318d1b33c63330dd7721903e55c")
|
||||
elseif (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$")
|
||||
set(GGML_OPENMP_ARCH "arm64")
|
||||
set(GGML_OPENMP_INSTALLER_SUFFIX "woa64")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "7c4ac97eb2ae6b960ca5f9caf3ff6124c8d2a18cc07a7840a4d2ea15537bad8e")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "76f44ef1ba6eeb5a65904e9500f042f588fade49952778ce48f0374daa934396")
|
||||
else()
|
||||
message(FATAL_ERROR "GGML_OPENMP_FETCH does not support ${CMAKE_SYSTEM_PROCESSOR}")
|
||||
endif()
|
||||
|
||||
@@ -2584,6 +2584,7 @@ bool ggml_op_alloc_size_may_expand(enum ggml_op op) {
|
||||
case GGML_OP_ARGSORT:
|
||||
case GGML_OP_TOP_K:
|
||||
case GGML_OP_ARGMAX:
|
||||
case GGML_OP_CONV_2D:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -2501,6 +2501,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
|
||||
break;
|
||||
case GGML_OP_SET_ROWS:
|
||||
{
|
||||
if (op->src[0]->type == GGML_TYPE_BF16) {
|
||||
return false;
|
||||
}
|
||||
switch (op->type) {
|
||||
case GGML_TYPE_F32:
|
||||
case GGML_TYPE_F16:
|
||||
@@ -2553,6 +2556,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
|
||||
if (((const int32_t *) op->op_params)[15] != 0) {
|
||||
return false; // FIXME: support ggml_rope_set_offset
|
||||
}
|
||||
if (op->src[0]->type != GGML_TYPE_F32 && op->src[0]->type != GGML_TYPE_F16) {
|
||||
return false;
|
||||
}
|
||||
if (op->src[0]->ne[0] > 896) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -262,13 +262,22 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
v128_t v0_0 = wasm_v128_load(x0->qs);
|
||||
v128_t v0_0l = wasm_v128_and(v0_0, m4b);
|
||||
v128_t v0_0h = wasm_u8x16_shr(v0_0, 4);
|
||||
v128_t v0_0ls = wasm_i8x16_sub(v0_0l, s8b);
|
||||
v128_t v0_0hs = wasm_i8x16_sub(v0_0h, s8b);
|
||||
|
||||
// Load y0 vectors
|
||||
v128_t y0_l = wasm_v128_load(y0->qs);
|
||||
v128_t y0_h = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// unsigned nibbles (0..15) in the 7-bit operand, then subtract 8*sum(y)
|
||||
v128_t dp0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_l, v0_0l, wasm_i32x4_splat(0));
|
||||
dp0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_h, v0_0h, dp0);
|
||||
v128_t ys0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_l, s8b, wasm_i32x4_splat(0));
|
||||
ys0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_h, s8b, ys0);
|
||||
dp0 = wasm_i32x4_sub(dp0, ys0);
|
||||
# else
|
||||
v128_t v0_0ls = wasm_i8x16_sub(v0_0l, s8b);
|
||||
v128_t v0_0hs = wasm_i8x16_sub(v0_0h, s8b);
|
||||
|
||||
// Extend to i16x8 and compute dot products
|
||||
v128_t dx0l = wasm_i16x8_extend_low_i8x16(v0_0ls);
|
||||
v128_t dx0h = wasm_i16x8_extend_high_i8x16(v0_0ls);
|
||||
@@ -290,18 +299,27 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_dot_i16x8(dx0hh, dy0hh)
|
||||
)
|
||||
);
|
||||
# endif
|
||||
|
||||
// Load and process x1
|
||||
v128_t v0_1 = wasm_v128_load(x1->qs);
|
||||
v128_t v0_1l = wasm_v128_and(v0_1, m4b);
|
||||
v128_t v0_1h = wasm_u8x16_shr(v0_1, 4);
|
||||
v128_t v0_1ls = wasm_i8x16_sub(v0_1l, s8b);
|
||||
v128_t v0_1hs = wasm_i8x16_sub(v0_1h, s8b);
|
||||
|
||||
// Load y1 vectors
|
||||
v128_t y1_l = wasm_v128_load(y1->qs);
|
||||
v128_t y1_h = wasm_v128_load(y1->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// unsigned nibbles (0..15) in the 7-bit operand, then subtract 8*sum(y)
|
||||
v128_t dp1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_l, v0_1l, wasm_i32x4_splat(0));
|
||||
dp1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_h, v0_1h, dp1);
|
||||
v128_t ys1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_l, s8b, wasm_i32x4_splat(0));
|
||||
ys1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_h, s8b, ys1);
|
||||
dp1 = wasm_i32x4_sub(dp1, ys1);
|
||||
# else
|
||||
v128_t v0_1ls = wasm_i8x16_sub(v0_1l, s8b);
|
||||
v128_t v0_1hs = wasm_i8x16_sub(v0_1h, s8b);
|
||||
// Extend to i16x8 and compute dot products
|
||||
v128_t dx1l = wasm_i16x8_extend_low_i8x16(v0_1ls);
|
||||
v128_t dx1h = wasm_i16x8_extend_high_i8x16(v0_1ls);
|
||||
@@ -323,6 +341,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_dot_i16x8(dx1hh, dy1hh)
|
||||
)
|
||||
);
|
||||
# endif
|
||||
|
||||
// Accumulate results with scaling
|
||||
float scale0 = GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d);
|
||||
@@ -461,10 +480,18 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
// extract the 5th bit
|
||||
memcpy(&qh_, x0->qh, sizeof(qh_));
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// the high bit as is: unsigned 5-bit quants for the 7-bit operand
|
||||
tmp[0] = table_b2b_0[(qh_ >> 0) & 0xFF];
|
||||
tmp[1] = table_b2b_0[(qh_ >> 8) & 0xFF];
|
||||
tmp[2] = table_b2b_0[(qh_ >> 16) & 0xFF];
|
||||
tmp[3] = table_b2b_0[(qh_ >> 24) ];
|
||||
# else
|
||||
tmp[0] = table_b2b_1[(qh_ >> 0) & 0xFF];
|
||||
tmp[1] = table_b2b_1[(qh_ >> 8) & 0xFF];
|
||||
tmp[2] = table_b2b_1[(qh_ >> 16) & 0xFF];
|
||||
tmp[3] = table_b2b_1[(qh_ >> 24) ];
|
||||
# endif
|
||||
|
||||
const v128_t qhl = wasm_v128_load(tmp + 0);
|
||||
const v128_t qhh = wasm_v128_load(tmp + 2);
|
||||
@@ -475,14 +502,29 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
const v128_t v0l = wasm_v128_and (v0, m4b);
|
||||
const v128_t v0h = wasm_u8x16_shr(v0, 4);
|
||||
|
||||
// add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
|
||||
const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);
|
||||
const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);
|
||||
|
||||
// load y
|
||||
const v128_t v1l = wasm_v128_load(y0->qs);
|
||||
const v128_t v1h = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// unsigned 5-bit quants (0..31) in the 7-bit operand, then subtract 16*sum(y)
|
||||
const v128_t s16b = wasm_i8x16_splat(0x10);
|
||||
const v128_t v0lu = wasm_v128_or(v0l, qhl);
|
||||
const v128_t v0hu = wasm_v128_or(v0h, qhh);
|
||||
|
||||
v128_t dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1l, v0lu, wasm_i32x4_splat(0));
|
||||
dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1h, v0hu, dp);
|
||||
v128_t ys = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1l, s16b, wasm_i32x4_splat(0));
|
||||
ys = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1h, s16b, ys);
|
||||
dp = wasm_i32x4_sub(dp, ys);
|
||||
|
||||
sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(dp),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# else
|
||||
// add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
|
||||
const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);
|
||||
const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);
|
||||
|
||||
// int8x16 -> int16x8
|
||||
const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);
|
||||
const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);
|
||||
@@ -502,6 +544,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),
|
||||
wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# endif
|
||||
}
|
||||
|
||||
sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
|
||||
@@ -578,6 +621,13 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
const v128_t v1l = wasm_v128_load(y0->qs);
|
||||
const v128_t v1h = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1l, v0lf, wasm_i32x4_splat(0));
|
||||
dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1h, v0hf, dp);
|
||||
|
||||
sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(dp),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# else
|
||||
// int8x16 -> int16x8
|
||||
const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);
|
||||
const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);
|
||||
@@ -597,6 +647,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),
|
||||
wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# endif
|
||||
}
|
||||
|
||||
sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
|
||||
@@ -642,6 +693,17 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
const v128_t y0_0 = wasm_v128_load(y0->qs);
|
||||
const v128_t y0_1 = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// |x| (0..127) in the 7-bit operand and y*sign(x) in the other; relies on
|
||||
// q8_0 values being in -127..127, as ggml's quantizers produce
|
||||
const v128_t zero = wasm_i8x16_splat(0);
|
||||
const v128_t sx0 = wasm_i8x16_lt(x0_0, zero);
|
||||
const v128_t sx1 = wasm_i8x16_lt(x0_1, zero);
|
||||
const v128_t ys0 = wasm_i8x16_sub(wasm_v128_xor(y0_0, sx0), sx0);
|
||||
const v128_t ys1 = wasm_i8x16_sub(wasm_v128_xor(y0_1, sx1), sx1);
|
||||
v128_t sum_dots = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(ys0, wasm_i8x16_abs(x0_0), wasm_i32x4_splat(0));
|
||||
sum_dots = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(ys1, wasm_i8x16_abs(x0_1), sum_dots);
|
||||
# else
|
||||
// Extend 8-bit to 16-bit
|
||||
const v128_t x0_0l = wasm_i16x8_extend_low_i8x16(x0_0);
|
||||
const v128_t x0_0h = wasm_i16x8_extend_high_i8x16(x0_0);
|
||||
@@ -661,7 +723,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
|
||||
// Sum all dot products
|
||||
const v128_t sum_dots = wasm_i32x4_add(wasm_i32x4_add(dx0_0, dx0_1), wasm_i32x4_add(dx1_0, dx1_1));
|
||||
|
||||
# endif
|
||||
// Convert to float and accumulate
|
||||
const float scale = GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d);
|
||||
sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(sum_dots), wasm_f32x4_splat(scale)));
|
||||
@@ -750,6 +812,10 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
v128_t q2_shift_1 = wasm_u8x16_shr(q2_1, shift);
|
||||
v128_t q2_bits_1 = wasm_v128_and(q2_shift_1, wasm_i8x16_splat(0x03));
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t p_sum0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_0, q2_bits_0, wasm_i32x4_splat(0));
|
||||
v128_t p_sum1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_1, q2_bits_1, wasm_i32x4_splat(0));
|
||||
# else
|
||||
// Calculate dot products
|
||||
v128_t p0 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q8_0),
|
||||
@@ -768,10 +834,14 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q2_bits_1)
|
||||
);
|
||||
|
||||
v128_t p_sum0 = wasm_i32x4_add(p0, p1);
|
||||
v128_t p_sum1 = wasm_i32x4_add(p2, p3);
|
||||
# endif
|
||||
|
||||
// Accumulate scaled results
|
||||
v128_t scaled = wasm_i32x4_add(
|
||||
wasm_i32x4_mul(wasm_i32x4_add(p0, p1), wasm_i32x4_splat(d0)),
|
||||
wasm_i32x4_mul(wasm_i32x4_add(p2, p3), wasm_i32x4_splat(d1))
|
||||
wasm_i32x4_mul(p_sum0, wasm_i32x4_splat(d0)),
|
||||
wasm_i32x4_mul(p_sum1, wasm_i32x4_splat(d1))
|
||||
);
|
||||
|
||||
isum_vec = wasm_i32x4_add(isum_vec, scaled);
|
||||
@@ -817,7 +887,84 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
|
||||
const int nb = n / QK_K;
|
||||
|
||||
#if defined __wasm_simd128__
|
||||
#if defined(__wasm_relaxed_simd__)
|
||||
// Unsigned 3-bit quants (0..7) in the relaxed dot's 7-bit operand, q8 in
|
||||
// the other; the -4 offset is taken off with the q8 block sums.
|
||||
const v128_t m3 = wasm_i8x16_splat(0x03);
|
||||
const v128_t m1 = wasm_i8x16_splat(0x01);
|
||||
const v128_t zero = wasm_i32x4_splat(0);
|
||||
uint32_t auxs[4];
|
||||
|
||||
float sumf = 0;
|
||||
|
||||
for (int i = 0; i < nb; ++i) {
|
||||
const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
|
||||
|
||||
const uint8_t * GGML_RESTRICT q3 = x[i].qs;
|
||||
const int8_t * GGML_RESTRICT q8 = y[i].qs;
|
||||
|
||||
// Extract scales
|
||||
memcpy(auxs, x[i].scales, 12);
|
||||
uint32_t tmp = auxs[2];
|
||||
auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);
|
||||
auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);
|
||||
auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);
|
||||
auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);
|
||||
|
||||
const v128_t vsc = wasm_i8x16_sub(wasm_v128_load(auxs), wasm_i8x16_splat(32));
|
||||
const v128_t vsc_l = wasm_i16x8_extend_low_i8x16(vsc);
|
||||
const v128_t vsc_h = wasm_i16x8_extend_high_i8x16(vsc);
|
||||
|
||||
// sum(scale * bsum), for the -4 offset
|
||||
const v128_t vmins = wasm_i32x4_add(
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums), vsc_l),
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums + 8), vsc_h));
|
||||
|
||||
const v128_t hm0 = wasm_v128_load(x[i].hmask);
|
||||
const v128_t hm1 = wasm_v128_load(x[i].hmask + 16);
|
||||
|
||||
v128_t acc = zero;
|
||||
v128_t acc2 = zero;
|
||||
|
||||
for (int j = 0; j < QK_K/128; ++j) {
|
||||
const v128_t q3_0 = wasm_v128_load(q3);
|
||||
const v128_t q3_1 = wasm_v128_load(q3 + 16);
|
||||
q3 += 32;
|
||||
|
||||
const v128_t s32a = wasm_i32x4_extend_low_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
const v128_t s32b = wasm_i32x4_extend_high_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
const v128_t s32[2] = { s32a, s32b };
|
||||
|
||||
for (int g = 0; g < 4; ++g) {
|
||||
// 2 low bits at shift 2g, high bit 4j+g of hmask moved to bit 2
|
||||
// (16-bit shifts are safe on masked bytes)
|
||||
const int hb = 4*j + g;
|
||||
const v128_t u0 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(q3_0, 2*g), m3),
|
||||
wasm_i16x8_shl(wasm_v128_and(wasm_u16x8_shr(hm0, hb), m1), 2));
|
||||
const v128_t u1 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(q3_1, 2*g), m3),
|
||||
wasm_i16x8_shl(wasm_v128_and(wasm_u16x8_shr(hm1, hb), m1), 2));
|
||||
|
||||
const v128_t p0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8), u0, zero);
|
||||
const v128_t p1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 16), u1, zero);
|
||||
q8 += 32;
|
||||
|
||||
const v128_t sv = s32[g >> 1];
|
||||
const v128_t sc0 = (g & 1) ? wasm_i32x4_shuffle(sv, sv, 2, 2, 2, 2) : wasm_i32x4_shuffle(sv, sv, 0, 0, 0, 0);
|
||||
const v128_t sc1 = (g & 1) ? wasm_i32x4_shuffle(sv, sv, 3, 3, 3, 3) : wasm_i32x4_shuffle(sv, sv, 1, 1, 1, 1);
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p0, sc0));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p1, sc1));
|
||||
}
|
||||
}
|
||||
|
||||
acc = wasm_i32x4_sub(wasm_i32x4_add(acc, acc2), wasm_i32x4_shl(vmins, 2));
|
||||
const int32_t isum = wasm_i32x4_extract_lane(acc, 0) + wasm_i32x4_extract_lane(acc, 1) +
|
||||
wasm_i32x4_extract_lane(acc, 2) + wasm_i32x4_extract_lane(acc, 3);
|
||||
sumf += d * isum;
|
||||
}
|
||||
|
||||
*s = sumf;
|
||||
|
||||
#elif defined __wasm_simd128__
|
||||
int8_t aux8[QK_K];
|
||||
float sums[8] = {0};
|
||||
uint32_t auxs[4];
|
||||
@@ -986,6 +1133,14 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
q8 += 64;
|
||||
|
||||
// Low nibble products
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t vacc1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x0, q4l0, wasm_i32x4_splat(0)
|
||||
);
|
||||
vacc1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x1, q4l1, vacc1
|
||||
);
|
||||
# else
|
||||
v128_t vacc1 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q4l0),
|
||||
wasm_i16x8_extend_low_i8x16(q8x0)
|
||||
@@ -1002,8 +1157,17 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q4l1),
|
||||
wasm_i16x8_extend_high_i8x16(q8x1)
|
||||
));
|
||||
# endif
|
||||
|
||||
// High nibble products
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t vacc2 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x2, q4h0, wasm_i32x4_splat(0)
|
||||
);
|
||||
vacc2 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x3, q4h1, vacc2
|
||||
);
|
||||
# else
|
||||
v128_t vacc2 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q4h0),
|
||||
wasm_i16x8_extend_low_i8x16(q8x2)
|
||||
@@ -1020,6 +1184,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q4h1),
|
||||
wasm_i16x8_extend_high_i8x16(q8x3)
|
||||
));
|
||||
# endif
|
||||
|
||||
// Accumulate scaled results
|
||||
int32_t vacc1_sum = wasm_i32x4_extract_lane(vacc1, 0) + wasm_i32x4_extract_lane(vacc1, 1) +
|
||||
@@ -1127,6 +1292,11 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
v128_t q8_3 = wasm_v128_load(q8 + 48);
|
||||
q8 += 64;
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// Process low quants using relaxed SIMD
|
||||
v128_t sum_low = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_0, q5l_0, wasm_i32x4_splat(0));
|
||||
sum_low = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_1, q5l_1, sum_low);
|
||||
# else
|
||||
// Process low quants
|
||||
v128_t pl0 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q5l_0),
|
||||
@@ -1145,7 +1315,13 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q8_1)
|
||||
));
|
||||
v128_t sum_low = wasm_i32x4_add(pl0, pl1);
|
||||
# endif
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// Process high quants using relaxed SIMD
|
||||
v128_t sum_high = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_2, q5h_0, wasm_i32x4_splat(0));
|
||||
sum_high = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_3, q5h_1, sum_high);
|
||||
# else
|
||||
// Process high quants
|
||||
v128_t ph0 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q5h_0),
|
||||
@@ -1164,6 +1340,7 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q8_3)
|
||||
));
|
||||
v128_t sum_high = wasm_i32x4_add(ph0, ph1);
|
||||
# endif
|
||||
|
||||
// Accumulate with scale factors
|
||||
int32_t sl = wasm_i32x4_extract_lane(sum_low, 0) + wasm_i32x4_extract_lane(sum_low, 1) +
|
||||
@@ -1204,7 +1381,78 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
|
||||
const int nb = n / QK_K;
|
||||
|
||||
#if defined __wasm_simd128__
|
||||
#if defined(__wasm_relaxed_simd__)
|
||||
const v128_t m4 = wasm_i8x16_splat(0x0F);
|
||||
const v128_t m2 = wasm_i8x16_splat(0x03);
|
||||
const v128_t zero = wasm_i32x4_splat(0);
|
||||
|
||||
float sumf = 0;
|
||||
|
||||
for (int i = 0; i < nb; ++i) {
|
||||
const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
|
||||
const uint8_t * GGML_RESTRICT ql = x[i].ql;
|
||||
const uint8_t * GGML_RESTRICT qh = x[i].qh;
|
||||
const int8_t * GGML_RESTRICT q8 = y[i].qs;
|
||||
|
||||
const v128_t vsc = wasm_v128_load(x[i].scales);
|
||||
const v128_t vsc_l = wasm_i16x8_extend_low_i8x16(vsc);
|
||||
const v128_t vsc_h = wasm_i16x8_extend_high_i8x16(vsc);
|
||||
const v128_t vmins = wasm_i32x4_add(
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums), vsc_l),
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums + 8), vsc_h));
|
||||
|
||||
v128_t acc = zero;
|
||||
v128_t acc2 = zero;
|
||||
for (int j = 0; j < QK_K/128; ++j) {
|
||||
const v128_t ql0 = wasm_v128_load(ql);
|
||||
const v128_t ql1 = wasm_v128_load(ql + 16);
|
||||
const v128_t ql2 = wasm_v128_load(ql + 32);
|
||||
const v128_t ql3 = wasm_v128_load(ql + 48);
|
||||
const v128_t qh0 = wasm_v128_load(qh);
|
||||
const v128_t qh1 = wasm_v128_load(qh + 16);
|
||||
ql += 64;
|
||||
qh += 32;
|
||||
|
||||
const v128_t q6_0 = wasm_v128_or(wasm_v128_and(ql0, m4), wasm_i16x8_shl(wasm_v128_and(qh0, m2), 4));
|
||||
const v128_t q6_1 = wasm_v128_or(wasm_v128_and(ql1, m4), wasm_i16x8_shl(wasm_v128_and(qh1, m2), 4));
|
||||
const v128_t q6_2 = wasm_v128_or(wasm_v128_and(ql2, m4), wasm_i16x8_shl(wasm_v128_and(qh0, wasm_i8x16_splat(0x0C)), 2));
|
||||
const v128_t q6_3 = wasm_v128_or(wasm_v128_and(ql3, m4), wasm_i16x8_shl(wasm_v128_and(qh1, wasm_i8x16_splat(0x0C)), 2));
|
||||
const v128_t q6_4 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql0, 4), m4), wasm_v128_and(qh0, wasm_i8x16_splat(0x30)));
|
||||
const v128_t q6_5 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql1, 4), m4), wasm_v128_and(qh1, wasm_i8x16_splat(0x30)));
|
||||
const v128_t q6_6 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql2, 4), m4), wasm_u16x8_shr(wasm_v128_and(qh0, wasm_i8x16_splat((int8_t)0xC0)), 2));
|
||||
const v128_t q6_7 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql3, 4), m4), wasm_u16x8_shr(wasm_v128_and(qh1, wasm_i8x16_splat((int8_t)0xC0)), 2));
|
||||
|
||||
const v128_t p0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 0), q6_0, zero);
|
||||
const v128_t p1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 16), q6_1, zero);
|
||||
const v128_t p2 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 32), q6_2, zero);
|
||||
const v128_t p3 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 48), q6_3, zero);
|
||||
const v128_t p4 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 64), q6_4, zero);
|
||||
const v128_t p5 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 80), q6_5, zero);
|
||||
const v128_t p6 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 96), q6_6, zero);
|
||||
const v128_t p7 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 112), q6_7, zero);
|
||||
q8 += 128;
|
||||
|
||||
const v128_t s32a = wasm_i32x4_extend_low_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
const v128_t s32b = wasm_i32x4_extend_high_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p0, wasm_i32x4_shuffle(s32a, s32a, 0, 0, 0, 0)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p1, wasm_i32x4_shuffle(s32a, s32a, 1, 1, 1, 1)));
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p2, wasm_i32x4_shuffle(s32a, s32a, 2, 2, 2, 2)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p3, wasm_i32x4_shuffle(s32a, s32a, 3, 3, 3, 3)));
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p4, wasm_i32x4_shuffle(s32b, s32b, 0, 0, 0, 0)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p5, wasm_i32x4_shuffle(s32b, s32b, 1, 1, 1, 1)));
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p6, wasm_i32x4_shuffle(s32b, s32b, 2, 2, 2, 2)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p7, wasm_i32x4_shuffle(s32b, s32b, 3, 3, 3, 3)));
|
||||
}
|
||||
|
||||
acc = wasm_i32x4_sub(wasm_i32x4_add(acc, acc2), wasm_i32x4_shl(vmins, 5));
|
||||
const int32_t isum = wasm_i32x4_extract_lane(acc, 0) + wasm_i32x4_extract_lane(acc, 1) +
|
||||
wasm_i32x4_extract_lane(acc, 2) + wasm_i32x4_extract_lane(acc, 3);
|
||||
sumf += d * isum;
|
||||
}
|
||||
|
||||
*s = sumf;
|
||||
|
||||
#elif defined __wasm_simd128__
|
||||
int8_t aux8[QK_K] __attribute__((aligned(16)));
|
||||
int32_t aux32[8] __attribute__((aligned(16))) = {0};
|
||||
float sums[8] __attribute__((aligned(16))) = {0};
|
||||
|
||||
@@ -861,8 +861,8 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
__m128 acc_3 = _mm_setzero_ps();
|
||||
|
||||
for (; ib + 1 < nb; ib += 2) {
|
||||
_mm_prefetch(&x[ib] + sizeof(block_q4_0), _MM_HINT_T0);
|
||||
_mm_prefetch(&y[ib] + sizeof(block_q8_0), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&x[ib] + sizeof(block_q4_0)), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&y[ib] + sizeof(block_q8_0)), _MM_HINT_T0);
|
||||
|
||||
// Compute combined scale for the block 0 and 1
|
||||
const __m128 d_0_1 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d) );
|
||||
@@ -879,8 +879,8 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
bx_1 = _mm_sub_epi8(bx_1, off);
|
||||
const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);
|
||||
|
||||
_mm_prefetch(&x[ib] + 2 * sizeof(block_q4_0), _MM_HINT_T0);
|
||||
_mm_prefetch(&y[ib] + 2 * sizeof(block_q8_0), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&x[ib] + 2 * sizeof(block_q4_0)), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&y[ib] + 2 * sizeof(block_q8_0)), _MM_HINT_T0);
|
||||
|
||||
// Compute combined scale for the block 2 and 3
|
||||
const __m128 d_2_3 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) );
|
||||
|
||||
@@ -2034,6 +2034,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
|
||||
{
|
||||
ggml_compute_forward_fill(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_SLEEP:
|
||||
{
|
||||
ggml_compute_forward_sleep(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_FLASH_ATTN_EXT:
|
||||
{
|
||||
ggml_compute_forward_flash_attn_ext(params, tensor);
|
||||
@@ -2306,6 +2310,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
|
||||
case GGML_OP_REPEAT:
|
||||
case GGML_OP_REPEAT_BACK:
|
||||
case GGML_OP_LEAKY_RELU:
|
||||
case GGML_OP_SLEEP:
|
||||
{
|
||||
n_tasks = 1;
|
||||
} break;
|
||||
@@ -2378,10 +2383,8 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
|
||||
} break;
|
||||
case GGML_OP_GET_ROWS:
|
||||
{
|
||||
// FIXME: get_rows can use additional threads, but the cost of launching additional threads
|
||||
// decreases performance with GPU offloading
|
||||
//n_tasks = n_threads;
|
||||
n_tasks = 1;
|
||||
// waking the other threads costs more than a small lookup, e.g. when this is the only CPU node of an offloaded model
|
||||
n_tasks = ggml_nelements(node) >= 262144 ? n_threads : 1;
|
||||
} break;
|
||||
case GGML_OP_SCALE:
|
||||
case GGML_OP_SET:
|
||||
@@ -2930,7 +2933,8 @@ struct ggml_cplan ggml_graph_plan(
|
||||
} break;
|
||||
case GGML_OP_SET_ROWS:
|
||||
{
|
||||
if (node->src[0]->type == GGML_TYPE_F16 && node->type != GGML_TYPE_F16) {
|
||||
if ((node->src[0]->type == GGML_TYPE_F16 && node->type != GGML_TYPE_F16) ||
|
||||
(node->src[0]->type == GGML_TYPE_BF16 && node->type != GGML_TYPE_BF16)) {
|
||||
cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
|
||||
}
|
||||
} break;
|
||||
|
||||
+135
-52
@@ -2238,9 +2238,23 @@ void ggml_compute_forward_fill(const ggml_compute_params * params, ggml_tensor *
|
||||
}
|
||||
}
|
||||
|
||||
// ggml_compute_forward_sleep
|
||||
|
||||
void ggml_compute_forward_sleep(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
if (params->ith != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// same clock as ggml_time_us() callers; on macOS steady_clock is CLOCK_MONOTONIC_RAW, which can run faster
|
||||
const int64_t t_end = ggml_time_us() + ggml_get_op_params_i32(dst, 0);
|
||||
|
||||
while (ggml_time_us() < t_end) {}
|
||||
}
|
||||
|
||||
// ggml_compute_tri
|
||||
|
||||
static void ggml_compute_forward_tri_f32(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_tri_t(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
|
||||
const ggml_tri_type ttype = (ggml_tri_type) ggml_get_op_params_i32(dst, 0);
|
||||
@@ -2266,11 +2280,11 @@ static void ggml_compute_forward_tri_f32(const ggml_compute_params * params, ggm
|
||||
const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
|
||||
const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
|
||||
|
||||
const float * src_ptr = (const float *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
|
||||
float * dst_ptr = ( float *) (( char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);
|
||||
const T * src_ptr = (const T *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
|
||||
T * dst_ptr = ( T *) (( char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);
|
||||
|
||||
for (int i0 = 0; i0 < ne0; ++i0) {
|
||||
dst_ptr[i0] = bipred(i0, i01) ? src_ptr[i0] : 0.0f;
|
||||
dst_ptr[i0] = bipred(i0, i01) ? src_ptr[i0] : T(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2281,7 +2295,12 @@ void ggml_compute_forward_tri(const ggml_compute_params * params, ggml_tensor *
|
||||
switch (src0->type) {
|
||||
case GGML_TYPE_F32:
|
||||
{
|
||||
ggml_compute_forward_tri_f32(params, dst);
|
||||
ggml_compute_forward_tri_t<uint32_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_tri_t<uint16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -2708,7 +2727,8 @@ static void ggml_compute_forward_leaky_relu_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_leaky_relu_f16(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_leaky_relu_half(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -2728,13 +2748,16 @@ static void ggml_compute_forward_leaky_relu_f16(
|
||||
float negative_slope;
|
||||
memcpy(&negative_slope, dst->op_params, sizeof(float));
|
||||
|
||||
assert(dst->nb[0] == sizeof(ggml_fp16_t));
|
||||
assert(src0->nb[0] == sizeof(ggml_fp16_t));
|
||||
assert(dst->nb[0] == sizeof(T));
|
||||
assert(src0->nb[0] == sizeof(T));
|
||||
|
||||
for (int i = 0; i < n; i++) {
|
||||
ggml_vec_leaky_relu_f16(nc,
|
||||
(ggml_fp16_t *) ((char *) dst->data + i*( dst->nb[1])),
|
||||
(ggml_fp16_t *) ((char *) src0->data + i*(src0->nb[1])), negative_slope);
|
||||
T * y = (T *) ((char *) dst->data + i*( dst->nb[1]));
|
||||
const T * x = (const T *) ((const char *) src0->data + i*(src0->nb[1]));
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float v = type_conversion_table<T>::to_f32(x[k]);
|
||||
y[k] = type_conversion_table<T>::from_f32(((v > 0.f) ? v : 0.f) + negative_slope * ((v < 0.0f) ? v : 0.f));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2751,7 +2774,11 @@ void ggml_compute_forward_leaky_relu(
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
ggml_compute_forward_leaky_relu_f16(params, dst);
|
||||
ggml_compute_forward_leaky_relu_half<ggml_fp16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_leaky_relu_half<ggml_bf16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -3452,7 +3479,8 @@ static void ggml_compute_forward_swiglu_oai_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_swiglu_oai_f16(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_swiglu_oai_half(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -3492,9 +3520,9 @@ static void ggml_compute_forward_swiglu_oai_f16(
|
||||
const int ir1 = MIN(ir0 + dr, nr);
|
||||
|
||||
for (int i1 = ir0; i1 < ir1; i1++) {
|
||||
ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);
|
||||
ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);
|
||||
ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1]));
|
||||
T * src0_p = (T *) (src0_d + i1*src0_o);
|
||||
T * src1_p = (T *) (src1_d + i1*src1_o);
|
||||
T * dst_p = (T *) ((char *) dst->data + i1*(dst->nb[1]));
|
||||
|
||||
if (!src1) {
|
||||
src0_p += swapped ? nc : 0;
|
||||
@@ -3502,15 +3530,15 @@ static void ggml_compute_forward_swiglu_oai_f16(
|
||||
}
|
||||
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float x = std::min(GGML_CPU_FP16_TO_FP32(src0_p[k]), limit);
|
||||
const float y = std::clamp(GGML_CPU_FP16_TO_FP32(src1_p[k]), -limit, limit);
|
||||
const float x = std::min(type_conversion_table<T>::to_f32(src0_p[k]), limit);
|
||||
const float y = std::clamp(type_conversion_table<T>::to_f32(src1_p[k]), -limit, limit);
|
||||
const float out_glu = x / (1.f + expf(alpha * (-x)));
|
||||
dst_p[k] = GGML_CPU_FP32_TO_FP16(out_glu * (y + 1.f));
|
||||
dst_p[k] = type_conversion_table<T>::from_f32(out_glu * (y + 1.f));
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float x = GGML_CPU_FP16_TO_FP32(dst_p[k]);
|
||||
const float x = type_conversion_table<T>::to_f32(dst_p[k]);
|
||||
GGML_UNUSED(x);
|
||||
assert(!isnan(x));
|
||||
assert(!isinf(x));
|
||||
@@ -3532,7 +3560,11 @@ static void ggml_compute_forward_swiglu_oai(
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
ggml_compute_forward_swiglu_oai_f16(params, dst);
|
||||
ggml_compute_forward_swiglu_oai_half<ggml_fp16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_swiglu_oai_half<ggml_bf16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -3602,7 +3634,8 @@ static void ggml_compute_forward_swiglu_clamp_f32(const ggml_compute_params * pa
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_swiglu_clamp_half(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const ggml_tensor * src1 = dst->src[1];
|
||||
char * src0_d = (char *) src0->data;
|
||||
@@ -3635,9 +3668,9 @@ static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * pa
|
||||
const int ir1 = MIN(ir0 + dr, nr);
|
||||
|
||||
for (int i1 = ir0; i1 < ir1; i1++) {
|
||||
ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1 * src0_o);
|
||||
ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1 * src1_o);
|
||||
ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1 * (dst->nb[1]));
|
||||
T * src0_p = (T *) (src0_d + i1 * src0_o);
|
||||
T * src1_p = (T *) (src1_d + i1 * src1_o);
|
||||
T * dst_p = (T *) ((char *) dst->data + i1 * (dst->nb[1]));
|
||||
|
||||
if (!src1) {
|
||||
src0_p += swapped ? nc : 0;
|
||||
@@ -3645,14 +3678,14 @@ static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * pa
|
||||
}
|
||||
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float gate = std::min(GGML_FP16_TO_FP32(src0_p[k]), limit);
|
||||
const float up = std::clamp(GGML_FP16_TO_FP32(src1_p[k]), -limit, limit);
|
||||
dst_p[k] = GGML_FP32_TO_FP16(gate / (1.f + expf(-gate)) * up);
|
||||
const float gate = std::min(type_conversion_table<T>::to_f32(src0_p[k]), limit);
|
||||
const float up = std::clamp(type_conversion_table<T>::to_f32(src1_p[k]), -limit, limit);
|
||||
dst_p[k] = type_conversion_table<T>::from_f32(gate / (1.f + expf(-gate)) * up);
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float x = GGML_FP16_TO_FP32(dst_p[k]);
|
||||
const float x = type_conversion_table<T>::to_f32(dst_p[k]);
|
||||
GGML_UNUSED(x);
|
||||
assert(!isnan(x));
|
||||
assert(!isinf(x));
|
||||
@@ -3667,7 +3700,10 @@ static void ggml_compute_forward_swiglu_clamp(const ggml_compute_params * params
|
||||
ggml_compute_forward_swiglu_clamp_f32(params, dst);
|
||||
break;
|
||||
case GGML_TYPE_F16:
|
||||
ggml_compute_forward_swiglu_clamp_f16(params, dst);
|
||||
ggml_compute_forward_swiglu_clamp_half<ggml_fp16_t>(params, dst);
|
||||
break;
|
||||
case GGML_TYPE_BF16:
|
||||
ggml_compute_forward_swiglu_clamp_half<ggml_bf16_t>(params, dst);
|
||||
break;
|
||||
default:
|
||||
GGML_ABORT("fatal error");
|
||||
@@ -4953,7 +4989,8 @@ static void ggml_compute_forward_scale_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_scale_bf16(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_scale_half(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -4989,10 +5026,10 @@ static void ggml_compute_forward_scale_bf16(
|
||||
const int64_t i2 = (i / ne01) % ne02;
|
||||
const int64_t i3 = i / (ne01*ne02);
|
||||
|
||||
const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
|
||||
ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
const T * x = (const T *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
|
||||
T * y = (T *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
for (int k = 0; k < nc; k++) {
|
||||
y[k] = GGML_FP32_TO_BF16(s*GGML_BF16_TO_FP32(x[k]) + b);
|
||||
y[k] = type_conversion_table<T>::from_f32(s*type_conversion_table<T>::to_f32(x[k]) + b);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5008,9 +5045,13 @@ void ggml_compute_forward_scale(
|
||||
{
|
||||
ggml_compute_forward_scale_f32(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
ggml_compute_forward_scale_half<ggml_fp16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_scale_bf16(params, dst);
|
||||
ggml_compute_forward_scale_half<ggml_bf16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -5490,7 +5531,7 @@ static void ggml_compute_forward_set_rows_impl(
|
||||
assert(ne0 == nc);
|
||||
assert(ne2 == ne02);
|
||||
assert(ne3 == ne03);
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);
|
||||
assert(ne02 % ne11 == 0);
|
||||
assert(ne03 % ne12 == 0);
|
||||
|
||||
@@ -5537,6 +5578,20 @@ static void ggml_compute_forward_set_rows_impl(
|
||||
from_float(wdata,
|
||||
((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc);
|
||||
}
|
||||
} else if constexpr (std::is_same_v<src_t, ggml_bf16_t>) {
|
||||
if (dst->type == GGML_TYPE_BF16) {
|
||||
memcpy(
|
||||
((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3),
|
||||
((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03),
|
||||
rs);
|
||||
} else {
|
||||
float * wdata = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith;
|
||||
ggml_bf16_to_fp32_row(
|
||||
(const ggml_bf16_t *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03),
|
||||
wdata, nc);
|
||||
from_float(wdata,
|
||||
((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc);
|
||||
}
|
||||
} else {
|
||||
GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type));
|
||||
}
|
||||
@@ -5573,6 +5628,16 @@ void ggml_compute_forward_set_rows(
|
||||
GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type));
|
||||
}
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
if (src1->type == GGML_TYPE_I64) {
|
||||
ggml_compute_forward_set_rows_impl<ggml_bf16_t, int64_t>(params, dst);
|
||||
} else if (src1->type == GGML_TYPE_I32) {
|
||||
ggml_compute_forward_set_rows_impl<ggml_bf16_t, int32_t>(params, dst);
|
||||
} else {
|
||||
GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type));
|
||||
}
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type));
|
||||
@@ -5694,7 +5759,8 @@ void ggml_compute_forward_get_rows_back(
|
||||
|
||||
// ggml_compute_forward_diag
|
||||
|
||||
static void ggml_compute_forward_diag_f32(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_diag_t(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -5714,14 +5780,14 @@ static void ggml_compute_forward_diag_f32(
|
||||
GGML_ASSERT(ne02 == ne2);
|
||||
GGML_ASSERT(ne03 == ne3);
|
||||
|
||||
GGML_ASSERT(nb00 == sizeof(float));
|
||||
GGML_ASSERT(nb0 == sizeof(float));
|
||||
GGML_ASSERT(nb00 == sizeof(T));
|
||||
GGML_ASSERT(nb0 == sizeof(T));
|
||||
|
||||
for (int i3 = 0; i3 < ne3; i3++) {
|
||||
for (int i2 = 0; i2 < ne2; i2++) {
|
||||
for (int i1 = 0; i1 < ne1; i1++) {
|
||||
float * d = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
|
||||
float * s = (float *)((char *) src0->data + i3*nb03 + i2*nb02);
|
||||
T * d = (T *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
|
||||
T * s = (T *)((char *) src0->data + i3*nb03 + i2*nb02);
|
||||
for (int i0 = 0; i0 < i1; i0++) {
|
||||
d[i0] = 0;
|
||||
}
|
||||
@@ -5743,7 +5809,12 @@ void ggml_compute_forward_diag(
|
||||
switch (src0->type) {
|
||||
case GGML_TYPE_F32:
|
||||
{
|
||||
ggml_compute_forward_diag_f32(params, dst);
|
||||
ggml_compute_forward_diag_t<uint32_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_diag_t<uint16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -6026,7 +6097,8 @@ static void ggml_compute_forward_clamp_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_clamp_f16(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_clamp_half(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -6045,20 +6117,20 @@ static void ggml_compute_forward_clamp_f16(
|
||||
|
||||
GGML_TENSOR_UNARY_OP_LOCALS
|
||||
|
||||
GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));
|
||||
GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
|
||||
GGML_ASSERT( nb0 == sizeof(T));
|
||||
GGML_ASSERT(nb00 == sizeof(T));
|
||||
|
||||
for (int j = ith; j < n; j += nth) {
|
||||
const int64_t i1 = j % ne01;
|
||||
const int64_t i2 = (j / ne01) % ne02;
|
||||
const int64_t i3 = j / (ne01*ne02);
|
||||
|
||||
ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
|
||||
T * dst_ptr = (T *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
T * src0_ptr = (T *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
|
||||
|
||||
for (int i = 0; i < nc; i++) {
|
||||
float v = GGML_CPU_FP16_TO_FP32(src0_ptr[i]);
|
||||
dst_ptr[i] = GGML_CPU_FP32_TO_FP16(MAX(MIN(v, max), min));
|
||||
float v = type_conversion_table<T>::to_f32(src0_ptr[i]);
|
||||
dst_ptr[i] = type_conversion_table<T>::from_f32(MAX(MIN(v, max), min));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6076,9 +6148,12 @@ void ggml_compute_forward_clamp(
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
ggml_compute_forward_clamp_f16(params, dst);
|
||||
ggml_compute_forward_clamp_half<ggml_fp16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_clamp_half<ggml_bf16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_Q1_0:
|
||||
case GGML_TYPE_Q2_0:
|
||||
case GGML_TYPE_Q4_0:
|
||||
@@ -6251,7 +6326,7 @@ static void rotate_pairs(const int64_t n, const int64_t n_offset, const float *
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T> //float or ggml_fp16_t
|
||||
template<typename T> //float, ggml_fp16_t or ggml_bf16_t
|
||||
static void ggml_compute_forward_rope_flt(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst,
|
||||
@@ -6261,7 +6336,7 @@ static void ggml_compute_forward_rope_flt(
|
||||
const ggml_tensor * src1 = dst->src[1];
|
||||
const ggml_tensor * src2 = dst->src[2];
|
||||
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);
|
||||
GGML_ASSERT(src1->type == GGML_TYPE_I32);
|
||||
|
||||
float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
|
||||
@@ -6421,6 +6496,10 @@ void ggml_compute_forward_rope(
|
||||
{
|
||||
ggml_compute_forward_rope_flt<ggml_fp16_t>(params, dst, true);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_rope_flt<ggml_bf16_t>(params, dst, true);
|
||||
} break;
|
||||
case GGML_TYPE_F32:
|
||||
{
|
||||
ggml_compute_forward_rope_flt<float>(params, dst, true);
|
||||
@@ -6445,6 +6524,10 @@ void ggml_compute_forward_rope_back(
|
||||
{
|
||||
ggml_compute_forward_rope_flt<ggml_fp16_t>(params, dst, false);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_rope_flt<ggml_bf16_t>(params, dst, false);
|
||||
} break;
|
||||
case GGML_TYPE_F32:
|
||||
{
|
||||
ggml_compute_forward_rope_flt<float>(params, dst, false);
|
||||
|
||||
@@ -84,6 +84,7 @@ void ggml_compute_forward_top_k(const struct ggml_compute_params * params, struc
|
||||
void ggml_compute_forward_leaky_relu(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_fill(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_sleep(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_flash_attn_ext(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_flash_attn_back(
|
||||
const struct ggml_compute_params * params,
|
||||
|
||||
@@ -962,12 +962,6 @@ inline static void ggml_vec_relu_f16 (const int n, ggml_fp16_t * y, const ggml_f
|
||||
}
|
||||
}
|
||||
inline static void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }
|
||||
inline static void ggml_vec_leaky_relu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float ns) {
|
||||
for (int i = 0; i < n; ++i) {
|
||||
float v = GGML_CPU_FP16_TO_FP32(x[i]);
|
||||
y[i] = GGML_CPU_FP32_TO_FP16(((v > 0.f) ? v : 0.f) + ns * ((v < 0.0f) ? v : 0.f));
|
||||
}
|
||||
}
|
||||
inline static void ggml_vec_sigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = 1.f / (1.f + expf(-x[i])); }
|
||||
inline static void ggml_vec_sigmoid_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
|
||||
for (int i = 0; i < n; ++i) {
|
||||
|
||||
@@ -133,6 +133,8 @@ if (CUDAToolkit_FOUND)
|
||||
${GGML_SOURCES_CUDA}
|
||||
)
|
||||
|
||||
target_link_libraries(ggml-cuda PRIVATE ${CMAKE_DL_LIBS})
|
||||
|
||||
if (GGML_CUDA_GRAPHS)
|
||||
add_compile_definitions(GGML_CUDA_USE_GRAPHS)
|
||||
endif()
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
#include "batch-get.cuh"
|
||||
#include "batch-copy.cuh"
|
||||
#include "common.cuh"
|
||||
#include "ggml-cpp.h"
|
||||
|
||||
#include <climits>
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
struct ggml_cuda_batch_get_descriptor {
|
||||
struct ggml_cuda_batch_copy_descriptor {
|
||||
const unsigned char * src;
|
||||
unsigned char * dst;
|
||||
size_t size;
|
||||
};
|
||||
|
||||
static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_descriptor * copies, size_t count) {
|
||||
static __global__ void ggml_cuda_batch_copy_kernel(const ggml_cuda_batch_copy_descriptor * copies, size_t count) {
|
||||
// Blocks along y cooperate on each descriptor.
|
||||
const size_t index = blockIdx.x;
|
||||
if (index >= count) {
|
||||
@@ -50,11 +50,11 @@ static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_desc
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_cuda_batch_get_mapping {
|
||||
struct ggml_cuda_batch_copy_mapping {
|
||||
void * host = nullptr;
|
||||
unsigned char * device = nullptr;
|
||||
|
||||
ggml_cuda_batch_get_mapping(void * host, size_t size) {
|
||||
ggml_cuda_batch_copy_mapping(void * host, size_t size) {
|
||||
const cudaError_t err = cudaHostRegister(host, size, cudaHostRegisterMapped | cudaHostRegisterPortable);
|
||||
if (err != cudaSuccess) {
|
||||
(void) cudaGetLastError();
|
||||
@@ -70,10 +70,10 @@ struct ggml_cuda_batch_get_mapping {
|
||||
}
|
||||
}
|
||||
|
||||
ggml_cuda_batch_get_mapping(const ggml_cuda_batch_get_mapping &) = delete;
|
||||
ggml_cuda_batch_get_mapping & operator=(const ggml_cuda_batch_get_mapping &) = delete;
|
||||
ggml_cuda_batch_copy_mapping(const ggml_cuda_batch_copy_mapping &) = delete;
|
||||
ggml_cuda_batch_copy_mapping & operator=(const ggml_cuda_batch_copy_mapping &) = delete;
|
||||
|
||||
~ggml_cuda_batch_get_mapping() {
|
||||
~ggml_cuda_batch_copy_mapping() {
|
||||
if (host) {
|
||||
// Release only the registration acquired by this object.
|
||||
CUDA_CHECK(cudaHostUnregister(host));
|
||||
@@ -82,11 +82,12 @@ struct ggml_cuda_batch_get_mapping {
|
||||
};
|
||||
#endif
|
||||
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
static bool ggml_cuda_copy_tensor_n(int device, cudaMemcpyKind direction,
|
||||
const ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
GGML_UNUSED(device);
|
||||
GGML_UNUSED(direction);
|
||||
GGML_UNUSED(tensors);
|
||||
GGML_UNUSED(data);
|
||||
GGML_UNUSED(offsets);
|
||||
@@ -94,14 +95,17 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
GGML_UNUSED(n_tensors);
|
||||
return false;
|
||||
#else
|
||||
const bool to_host = direction == cudaMemcpyDeviceToHost;
|
||||
// Provisional dispatch values based on experiments.
|
||||
constexpr size_t min_batch_reads = 128;
|
||||
constexpr size_t max_gather_read_bytes = 128 * 1024;
|
||||
// Collect non-empty CPU destination ranges.
|
||||
constexpr size_t max_kernel_read_bytes = 128 * 1024;
|
||||
// Collect destination ranges; repeated sources are allowed.
|
||||
std::vector<std::pair<uintptr_t, uintptr_t>> ranges;
|
||||
ranges.reserve(n_tensors);
|
||||
size_t payload_bytes = 0;
|
||||
// Number of non-empty reads whose size is at most max_gather_read_bytes.
|
||||
uintptr_t begin = UINTPTR_MAX;
|
||||
uintptr_t end = 0;
|
||||
// Number of non-empty reads whose size is at most max_kernel_read_bytes.
|
||||
size_t n_reads_small = 0;
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
if (sizes[i] == 0) {
|
||||
@@ -111,15 +115,19 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
GGML_ASSERT(address <= UINTPTR_MAX - sizes[i]);
|
||||
GGML_ASSERT(payload_bytes <= SIZE_MAX - sizes[i]);
|
||||
payload_bytes += sizes[i];
|
||||
n_reads_small += sizes[i] <= max_gather_read_bytes;
|
||||
ranges.emplace_back(address, address + sizes[i]);
|
||||
n_reads_small += sizes[i] <= max_kernel_read_bytes;
|
||||
begin = std::min(begin, address);
|
||||
end = std::max(end, address + sizes[i]);
|
||||
const uintptr_t destination = to_host ? address : (uintptr_t) tensors[i]->data + offsets[i];
|
||||
GGML_ASSERT(destination <= UINTPTR_MAX - sizes[i]);
|
||||
ranges.emplace_back(destination, destination + sizes[i]);
|
||||
}
|
||||
if (ranges.empty()) {
|
||||
return true;
|
||||
}
|
||||
// Total number of non-empty reads.
|
||||
const size_t n_reads = ranges.size();
|
||||
if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_get_descriptor) || n_reads > INT_MAX) {
|
||||
if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_copy_descriptor) || n_reads > INT_MAX) {
|
||||
return false;
|
||||
}
|
||||
if (n_reads < 2 || (n_reads < min_batch_reads && n_reads_small == n_reads)) {
|
||||
@@ -127,72 +135,85 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
}
|
||||
std::sort(ranges.begin(), ranges.end());
|
||||
for (size_t i = 1; i < ranges.size(); ++i) {
|
||||
// Preserve the ordered copy path when CPU destinations overlap.
|
||||
// Preserve the ordered copy path when destinations overlap.
|
||||
if (ranges[i].first < ranges[i - 1].second) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Find the CPU span to register, including gaps between destinations.
|
||||
const uintptr_t begin = ranges.front().first;
|
||||
const uintptr_t end = ranges.back().second;
|
||||
if (end - begin - payload_bytes > payload_bytes) {
|
||||
// Limit gaps in the CPU span without underflow when sources overlap.
|
||||
if (end - begin > payload_bytes && end - begin - payload_bytes > payload_bytes) {
|
||||
return false;
|
||||
}
|
||||
// Register and map the existing CPU destination so the kernel can write to it directly.
|
||||
ggml_cuda_batch_get_mapping mapping((void *) begin, end - begin);
|
||||
// Register and map the CPU span for direct kernel access.
|
||||
ggml_cuda_batch_copy_mapping mapping((void *) begin, end - begin);
|
||||
if (!mapping.host) {
|
||||
return false;
|
||||
}
|
||||
// Reads selected for gather; zero means all reads use DMA.
|
||||
size_t n_reads_gather = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0;
|
||||
// Reads selected for the kernel; zero means all reads use DMA.
|
||||
size_t n_reads_kernel = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0;
|
||||
// Keep descriptor storage alive until the final stream synchronization.
|
||||
constexpr size_t max_batch_descriptors = 128 * 1024;
|
||||
const size_t bytes = std::min(n_reads_gather, max_batch_descriptors) * sizeof(ggml_cuda_batch_get_descriptor);
|
||||
ggml_backend_buffer_ptr descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr);
|
||||
const size_t bytes = std::min(n_reads_kernel, max_batch_descriptors) * sizeof(ggml_cuda_batch_copy_descriptor);
|
||||
ggml_backend_buffer_ptr descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr);
|
||||
if (!descriptors) {
|
||||
n_reads_gather = 0;
|
||||
n_reads_kernel = 0;
|
||||
}
|
||||
const auto host_type = ggml_backend_cuda_host_buffer_type();
|
||||
ggml_backend_buffer_ptr host_descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr);
|
||||
ggml_backend_buffer_ptr host_descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr);
|
||||
if (!host_descriptors || ggml_backend_buffer_get_type(host_descriptors.get()) != host_type) {
|
||||
n_reads_gather = 0;
|
||||
n_reads_kernel = 0;
|
||||
}
|
||||
if (n_reads_gather) {
|
||||
auto * copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get()));
|
||||
auto * device_copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(descriptors.get()));
|
||||
if (n_reads_kernel) {
|
||||
auto * copies = static_cast<ggml_cuda_batch_copy_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get()));
|
||||
auto * device_copies = static_cast<ggml_cuda_batch_copy_descriptor *>(ggml_backend_buffer_get_base(descriptors.get()));
|
||||
const int threads = 256;
|
||||
const size_t nsm = ggml_cuda_info().devices[device].nsm;
|
||||
size_t i = 0;
|
||||
for (size_t first = 0; first < n_reads_gather; first += max_batch_descriptors) {
|
||||
const size_t n_batch = std::min(n_reads_gather - first, max_batch_descriptors);
|
||||
for (size_t first = 0; first < n_reads_kernel; first += max_batch_descriptors) {
|
||||
const size_t n_batch = std::min(n_reads_kernel - first, max_batch_descriptors);
|
||||
size_t count = 0;
|
||||
for (; i < n_tensors && count < n_batch; ++i) {
|
||||
if (sizes[i] == 0 || sizes[i] > max_gather_read_bytes) {
|
||||
if (sizes[i] == 0 || sizes[i] > max_kernel_read_bytes) {
|
||||
continue;
|
||||
}
|
||||
copies[count++] = {(const unsigned char *) tensors[i]->data + offsets[i],
|
||||
mapping.device + ((uintptr_t) data[i] - begin), sizes[i]};
|
||||
auto * host = mapping.device + ((uintptr_t) data[i] - begin);
|
||||
auto * tensor = (unsigned char *) tensors[i]->data + offsets[i];
|
||||
copies[count++] = {to_host ? tensor : host, to_host ? host : tensor, sizes[i]};
|
||||
}
|
||||
const size_t blocks_per_copy = std::min<size_t>(UINT16_MAX, std::max<size_t>(1, (nsm + n_batch - 1) / n_batch));
|
||||
const dim3 blocks((unsigned int) n_batch, (unsigned int) blocks_per_copy);
|
||||
// Stream ordering keeps each kernel ahead of the next descriptor overwrite.
|
||||
CUDA_CHECK(cudaMemcpyAsync(device_copies, copies, n_batch * sizeof(copies[0]), cudaMemcpyHostToDevice, cudaStreamPerThread));
|
||||
ggml_cuda_batch_get_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch);
|
||||
ggml_cuda_batch_copy_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch);
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
if (first + n_batch < n_reads_gather) {
|
||||
if (first + n_batch < n_reads_kernel) {
|
||||
// Wait before refilling the pinned host descriptor buffer.
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Keep the destination pinned for DMA too; overlapping destinations already declined above.
|
||||
// Keep the CPU span pinned for DMA too; overlapping destinations already declined above.
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
if (sizes[i] && (!n_reads_gather || sizes[i] > max_gather_read_bytes)) {
|
||||
CUDA_CHECK(cudaMemcpyAsync(data[i], (const char *) tensors[i]->data + offsets[i], sizes[i], cudaMemcpyDeviceToHost, cudaStreamPerThread));
|
||||
if (sizes[i] && (!n_reads_kernel || sizes[i] > max_kernel_read_bytes)) {
|
||||
void * host = const_cast<void *>(data[i]);
|
||||
void * tensor = (char *) tensors[i]->data + offsets[i];
|
||||
CUDA_CHECK(cudaMemcpyAsync(to_host ? host : tensor, to_host ? tensor : host, sizes[i], direction, cudaStreamPerThread));
|
||||
}
|
||||
}
|
||||
// Wait for GPU writes before releasing the CPU mapping and descriptor buffers.
|
||||
// Wait for transfers before releasing the CPU mapping and descriptor buffers.
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
return ggml_cuda_copy_tensor_n(device, cudaMemcpyDeviceToHost, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
|
||||
bool ggml_cuda_set_tensor_n(int device,
|
||||
ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
return ggml_cuda_copy_tensor_n(device, cudaMemcpyHostToDevice, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
@@ -5,3 +5,7 @@
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors);
|
||||
|
||||
bool ggml_cuda_set_tensor_n(int device,
|
||||
ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors);
|
||||
@@ -59,7 +59,7 @@
|
||||
// While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, see
|
||||
// https://docs.nvidia.com/cutlass/media/docs/cpp/blackwell_functionality.html#blackwell-sm120-gemms
|
||||
#define GGML_CUDA_CC_BLACKWELL 1200
|
||||
#define GGML_CUDA_CC_DGX_SPARK 1210
|
||||
#define GGML_CUDA_CC_GB10 1210
|
||||
#define GGML_CUDA_CC_RUBIN 1300
|
||||
#define GGML_CUDA_CC_OFFSET_AMD 0x1000000
|
||||
#define GGML_CUDA_CC_OFFSET_MTHREADS 0x0100000
|
||||
@@ -452,6 +452,16 @@ static __device__ __forceinline__ ggml_cuda_thread_mask_t ggml_cuda_ballot(int p
|
||||
return __ballot_sync(mask, predicate);
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ int ggml_cuda_popc(ggml_cuda_thread_mask_t x) {
|
||||
static_assert(sizeof(ggml_cuda_thread_mask_t) == 4 || sizeof(ggml_cuda_thread_mask_t) == 8, "unsupported thread mask type");
|
||||
if constexpr (sizeof(ggml_cuda_thread_mask_t) == 4) {
|
||||
return __popc(x);
|
||||
} else {
|
||||
return __popcll(x);
|
||||
}
|
||||
}
|
||||
template<typename T> static __device__ __forceinline__ int ggml_cuda_popc(T x) = delete; // Makes implicit type casts result in an error.
|
||||
|
||||
static __device__ __forceinline__ int ggml_cuda_all(int predicate, ggml_cuda_thread_mask_t mask = GGML_CUDA_FULL_THREAD_MASK) {
|
||||
return __all_sync(mask, predicate);
|
||||
}
|
||||
@@ -1293,6 +1303,7 @@ struct ggml_cuda_device_info {
|
||||
size_t total_vram;
|
||||
int warp_size; // Number of threads in a dispatch
|
||||
bool supports_cooperative_launch; // whether cooperative launch is supported
|
||||
bool prefer_fp32_acc; // prefer BF16 inputs with FP32 accumulation for quantized cuBLAS
|
||||
int physical_device; // backing physical CUDA device for this (virtual) device
|
||||
int physical_share_count; // number of (virtual) devices sharing this device's physical GPU
|
||||
int virtual_index; // index of this (virtual) device among those sharing its physical GPU
|
||||
|
||||
@@ -128,7 +128,9 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
|
||||
const half * __restrict__ weight,
|
||||
float * __restrict__ output,
|
||||
const conv_params P,
|
||||
const int split_k) {
|
||||
const int split_k,
|
||||
const uint3 kw_fd,
|
||||
const uint3 kwkh_fd) {
|
||||
using namespace ggml_cuda_mma;
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
constexpr int nthreads = 4 * warp_size;
|
||||
@@ -202,7 +204,10 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
|
||||
}
|
||||
for (int k = load_row; k < BK; k += nthreads / (BN / 2)) {
|
||||
const int ki = k0 + k;
|
||||
const int ci = ki / (kw * kh), ky = ki / kw % kh, kx = ki % kw;
|
||||
const int ci = KW && KH ? ki / (KW * KH) : int(fastdiv(uint32_t(ki), kwkh_fd));
|
||||
const int r = ki - ci * kw * kh;
|
||||
const int ky = KW ? r / KW : int(fastdiv(uint32_t(r), kw_fd));
|
||||
const int kx = r - ky * kw;
|
||||
const int offset = ki < k_total ? (n * ic + ci) * ih * iw + ky * dy * iw + kx * dx : 0;
|
||||
half lo = __float2half(0.0f), hi = lo;
|
||||
if (ki < k_total && spatial < ow * oh) {
|
||||
@@ -314,12 +319,17 @@ static void conv2d_launch_implicit_gemm(const half * input,
|
||||
dim3 grid,
|
||||
dim3 block,
|
||||
cudaStream_t stream) {
|
||||
const uint3 kw_fd = init_fastdiv_values(uint64_t(params.KW));
|
||||
const uint3 kwkh_fd = init_fastdiv_values(uint64_t(params.KW * params.KH));
|
||||
if (params.KW == 3 && params.KH == 3) {
|
||||
conv2d_implicit_gemm_f16<3, 3, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k);
|
||||
conv2d_implicit_gemm_f16<3, 3, use_mma>
|
||||
<<<grid, block, 0, stream>>>(input, weight, output, params, split_k, kw_fd, kwkh_fd);
|
||||
} else if (params.KW == 1 && params.KH == 1) {
|
||||
conv2d_implicit_gemm_f16<1, 1, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k);
|
||||
conv2d_implicit_gemm_f16<1, 1, use_mma>
|
||||
<<<grid, block, 0, stream>>>(input, weight, output, params, split_k, kw_fd, kwkh_fd);
|
||||
} else {
|
||||
conv2d_implicit_gemm_f16<0, 0, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k);
|
||||
conv2d_implicit_gemm_f16<0, 0, use_mma>
|
||||
<<<grid, block, 0, stream>>>(input, weight, output, params, split_k, kw_fd, kwkh_fd);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -386,7 +396,8 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const int64_t padded_w = IW + 2 * int64_t(PD_X), padded_h = IH + 2 * int64_t(PD_Y);
|
||||
const bool padded_fits = padded_w > 0 && padded_w <= limit && padded_h > 0 && padded_h <= limit &&
|
||||
padded_w * padded_h <= limit && IC * B <= limit / (padded_w * padded_h);
|
||||
if (kernel->type == GGML_TYPE_F16 && (use_mma || use_blas || use_simt) && (use_blas || !small_conv) &&
|
||||
const bool kernel_ok = kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32;
|
||||
if (kernel_ok && (use_mma || use_blas || use_simt) && (use_blas || !small_conv) &&
|
||||
ggml_nelements(input) <= limit && ggml_nelements(kernel) <= limit && total <= limit && padded_fits &&
|
||||
PD_X >= 0 && PD_Y >= 0 && ST_X > 0 && ST_Y > 0 && DL_X > 0 && DL_Y > 0 &&
|
||||
(OW - 1) * ST_X + (KW - 1) * DL_X < padded_w && (OH - 1) * ST_Y + (KH - 1) * DL_Y < padded_h &&
|
||||
@@ -402,6 +413,13 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
conv2d_pad_f16<<<(padded_total + 255) / 256, 256, 0, st>>>(X_D, x_half.get(), int(IW), int(IH), pw, ph,
|
||||
PD_X, PD_Y, padded_total);
|
||||
}
|
||||
// F32 kernels take the same path: stage them once as F16, like the input, and still accumulate in F32.
|
||||
ggml_cuda_pool_alloc<half> k_half(ctx.pool());
|
||||
const half * K_H = (const half *) K_D;
|
||||
if (kernel->type == GGML_TYPE_F32) {
|
||||
K_H = k_half.alloc(ggml_nelements(kernel));
|
||||
ggml_get_to_fp16_cuda(GGML_TYPE_F32)(K_D, k_half.get(), ggml_nelements(kernel), st);
|
||||
}
|
||||
const conv_params padded_params = { pw, ph, OW, OH, KW, KH, ST_X, ST_Y, 0, 0, DL_X, DL_Y, IC, OC, B, total };
|
||||
if (use_blas) {
|
||||
const float alpha = 1.0f, beta = 0.0f;
|
||||
@@ -409,7 +427,7 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
cublasHandle_t cublas_h = ctx.cublas_handle();
|
||||
for (int n = 0; n < B; ++n) {
|
||||
CUBLAS_CHECK(cublasGemmEx(cublas_h, CUBLAS_OP_N, CUBLAS_OP_N, positions, int(OC), int(IC), &alpha,
|
||||
x_half.get() + int64_t(n) * IC * positions, CUDA_R_16F, positions, K_D,
|
||||
x_half.get() + int64_t(n) * IC * positions, CUDA_R_16F, positions, K_H,
|
||||
CUDA_R_16F, int(IC), &beta, Y_D + int64_t(n) * OC * positions, CUDA_R_32F,
|
||||
positions, CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT_TENSOR_OP));
|
||||
}
|
||||
@@ -426,11 +444,9 @@ void ggml_cuda_op_conv2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const dim3 block(device.warp_size, 4);
|
||||
const dim3 grid(unsigned((OW * OH + 63) / 64), unsigned((OC + 63) / 64), unsigned(B * split_k));
|
||||
if (use_mma) {
|
||||
conv2d_launch_implicit_gemm<true>(x_half.get(), (const half *) K_D, result, padded_params, split_k, grid,
|
||||
block, st);
|
||||
conv2d_launch_implicit_gemm<true>(x_half.get(), K_H, result, padded_params, split_k, grid, block, st);
|
||||
} else {
|
||||
conv2d_launch_implicit_gemm<false>(x_half.get(), (const half *) K_D, result, padded_params, split_k, grid,
|
||||
block, st);
|
||||
conv2d_launch_implicit_gemm<false>(x_half.get(), K_H, result, padded_params, split_k, grid, block, st);
|
||||
}
|
||||
if (split_k > 1) {
|
||||
conv2d_reduce_split_k<<<(total + 255) / 256, 256, 0, st>>>(result, Y_D, int(total), int(OC * OW * OH),
|
||||
|
||||
@@ -1206,7 +1206,7 @@ void launch_fattn(
|
||||
dim3 blocks_num;
|
||||
if (stream_k) {
|
||||
// Stream-K splits the work before the mask scan is applied, so skipped KV tiles make the blocks uneven.
|
||||
const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK && async_kv_preload && scan_mask;
|
||||
const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10 && async_kv_preload && scan_mask;
|
||||
|
||||
auto should_use_stream_k = [prefer_whole_tiles](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) {
|
||||
const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks;
|
||||
|
||||
@@ -17,15 +17,6 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
constexpr int values_per_lane = 8;
|
||||
using lane_mask_t = std::conditional_t<warp_size == 64, uint64_t, uint32_t>;
|
||||
constexpr uint64_t ballot_mask = ~(lane_mask_t)0;
|
||||
const auto popc = [](const lane_mask_t mask) {
|
||||
if constexpr (warp_size == 64) {
|
||||
return __popcll(mask);
|
||||
} else {
|
||||
return __popc(mask);
|
||||
}
|
||||
};
|
||||
const int tid = threadIdx.x;
|
||||
const int warp = tid / warp_size;
|
||||
const int lane = tid % warp_size;
|
||||
@@ -48,7 +39,7 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
__syncthreads();
|
||||
|
||||
for (int i0 = 0; i0 < ne30; i0 += blockDim.x*values_per_lane) {
|
||||
lane_mask_t selected_warp[values_per_lane];
|
||||
ggml_cuda_thread_mask_t selected_warp[values_per_lane];
|
||||
int warp_count = 0;
|
||||
#pragma unroll
|
||||
for (int item = 0; item < values_per_lane; ++item) {
|
||||
@@ -60,8 +51,8 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
selected |= (!oob || q < q1 - q0) && isfinite(__half2float(mask[q*s31 + i]));
|
||||
}
|
||||
}
|
||||
selected_warp[item] = (lane_mask_t)ggml_cuda_ballot((int) selected, ballot_mask);
|
||||
warp_count += popc(selected_warp[item]);
|
||||
selected_warp[item] = ggml_cuda_ballot((int) selected);
|
||||
warp_count += ggml_cuda_popc(selected_warp[item]);
|
||||
}
|
||||
|
||||
if (lane == 0) {
|
||||
@@ -81,16 +72,16 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const lane_mask_t lane_mask = lane == 0 ? 0 : ((lane_mask_t)1 << lane) - 1;
|
||||
const ggml_cuda_thread_mask_t lane_mask = lane == 0 ? 0 : ((ggml_cuda_thread_mask_t)1 << lane) - 1;
|
||||
int warp_item_offset = 0;
|
||||
#pragma unroll
|
||||
for (int item = 0; item < values_per_lane; ++item) {
|
||||
const int i = i0 + (warp*values_per_lane + item)*warp_size + lane;
|
||||
const int dst = row_count + warp_offsets[warp] + warp_item_offset + popc(selected_warp[item] & lane_mask);
|
||||
if ((selected_warp[item] & ((lane_mask_t)1 << lane)) && dst < n_kv_max) {
|
||||
const int dst = row_count + warp_offsets[warp] + warp_item_offset + ggml_cuda_popc(selected_warp[item] & lane_mask);
|
||||
if ((selected_warp[item] & ((ggml_cuda_thread_mask_t)1 << lane)) && dst < n_kv_max) {
|
||||
indices[dst] = i;
|
||||
}
|
||||
warp_item_offset += popc(selected_warp[item]);
|
||||
warp_item_offset += ggml_cuda_popc(selected_warp[item]);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
@@ -397,7 +388,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
|
||||
if (amd_mfma_available(cc) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 16)) {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
|
||||
} else if (gqa_ratio == 20 && GGML_CUDA_CC_IS_NVIDIA(cc)) { // GLM 4.7 Flash
|
||||
if (cc >= GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (cc >= GGML_CUDA_CC_GB10) {
|
||||
if (Q->ne[1] <= 8) {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
|
||||
break;
|
||||
|
||||
@@ -468,7 +468,7 @@ static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a
|
||||
// Enable only architectures benchmarked for this schedule. MMA support alone does not imply a speedup.
|
||||
if (!ampere_mma_available(info.cc) ||
|
||||
(info.cc != GGML_CUDA_CC_AMPERE_GA10X && info.cc != GGML_CUDA_CC_ADA_LOVELACE &&
|
||||
info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_DGX_SPARK)) {
|
||||
info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_GB10)) {
|
||||
return gdn_path::ar;
|
||||
}
|
||||
// Value splitting pays off when whole heads leave at least half the SMs idle.
|
||||
|
||||
+166
-14
@@ -10,7 +10,7 @@
|
||||
#include "ggml-cuda/arange.cuh"
|
||||
#include "ggml-cuda/argmax.cuh"
|
||||
#include "ggml-cuda/argsort.cuh"
|
||||
#include "ggml-cuda/batch-get.cuh"
|
||||
#include "ggml-cuda/batch-copy.cuh"
|
||||
#include "ggml-cuda/binbcast.cuh"
|
||||
#include "ggml-cuda/clamp.cuh"
|
||||
#include "ggml-cuda/col2im-1d.cuh"
|
||||
@@ -48,6 +48,7 @@
|
||||
#include "ggml-cuda/rope.cuh"
|
||||
#include "ggml-cuda/roll.cuh"
|
||||
#include "ggml-cuda/scale.cuh"
|
||||
#include "ggml-cuda/sleep.cuh"
|
||||
#include "ggml-cuda/snake.cuh"
|
||||
#include "ggml-cuda/softcap.cuh"
|
||||
#include "ggml-cuda/softmax.cuh"
|
||||
@@ -96,6 +97,11 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
#include "ggml-backend-dl.h"
|
||||
#include <nvml.h>
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");
|
||||
|
||||
#define GGML_LOG_WARN_ONCE(str) \
|
||||
@@ -309,6 +315,64 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
||||
}
|
||||
total_vram = 0;
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
#if defined(_WIN32)
|
||||
// NVML is installed by the driver; its NVSMI directory is not added to PATH on Windows.
|
||||
dl_handle_ptr library(LoadLibraryExW(L"nvml.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32));
|
||||
if (!library) {
|
||||
const DWORD size = GetEnvironmentVariableW(L"ProgramW6432", nullptr, 0);
|
||||
if (size > 0) {
|
||||
std::wstring path(size, L'\0');
|
||||
const DWORD length = GetEnvironmentVariableW(L"ProgramW6432", &path[0], size);
|
||||
if (length > 0 && length < size) {
|
||||
path.resize(length);
|
||||
path += L"\\NVIDIA Corporation\\NVSMI\\nvml.dll";
|
||||
if (fs::path(path).is_absolute()) {
|
||||
library.reset(LoadLibraryExW(path.c_str(), nullptr, LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
dl_handle_ptr library(dlopen("libnvidia-ml.so.1", RTLD_NOW | RTLD_LOCAL));
|
||||
#endif // defined(_WIN32)
|
||||
const auto get_symbol = [&](const char * name) {
|
||||
#if defined(_WIN32)
|
||||
return library ? GetProcAddress(library.get(), name) : nullptr;
|
||||
#else
|
||||
return library ? dlsym(library.get(), name) : nullptr;
|
||||
#endif // defined(_WIN32)
|
||||
};
|
||||
const auto init = reinterpret_cast<decltype(&nvmlInit_v2)>(get_symbol("nvmlInit_v2"));
|
||||
const auto shutdown = reinterpret_cast<decltype(&nvmlShutdown)>(get_symbol("nvmlShutdown"));
|
||||
const auto get_device = reinterpret_cast<decltype(&nvmlDeviceGetHandleByPciBusId_v2)>(get_symbol("nvmlDeviceGetHandleByPciBusId_v2"));
|
||||
const auto get_brand = reinterpret_cast<decltype(&nvmlDeviceGetBrand)>(get_symbol("nvmlDeviceGetBrand"));
|
||||
const bool nvml_initialized = init && shutdown && get_device && get_brand && init() == NVML_SUCCESS;
|
||||
|
||||
const auto should_use_fp32_acc = [&](const cudaDeviceProp & prop, const int cc) {
|
||||
if (cc == GGML_CUDA_CC_GB10) {
|
||||
return true;
|
||||
}
|
||||
if (!fast_bf16_hardware_available(cc) || !nvml_initialized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// CUDA and NVML device indices can differ, e.g. with CUDA_VISIBLE_DEVICES.
|
||||
char pci_bus_id[32];
|
||||
snprintf(pci_bus_id, sizeof(pci_bus_id), "%04x:%02x:%02x.0",
|
||||
(unsigned int) prop.pciDomainID, (unsigned int) prop.pciBusID, (unsigned int) prop.pciDeviceID);
|
||||
nvmlDevice_t device = nullptr;
|
||||
nvmlBrandType_t brand = NVML_BRAND_UNKNOWN;
|
||||
bool prefer_fp32_acc = false;
|
||||
if (get_device(pci_bus_id, &device) == NVML_SUCCESS && get_brand(device, &brand) == NVML_SUCCESS) {
|
||||
prefer_fp32_acc = brand == NVML_BRAND_QUADRO ||
|
||||
brand == NVML_BRAND_QUADRO_RTX ||
|
||||
brand == NVML_BRAND_NVIDIA_RTX;
|
||||
}
|
||||
return prefer_fp32_acc;
|
||||
};
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
std::vector<std::pair<int, std::string>> turing_devices_without_mma;
|
||||
for (int id = 0; id < info.device_count; ++id) {
|
||||
const int physical_id = info.devices[id].physical_device;
|
||||
@@ -379,6 +443,11 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
||||
#else
|
||||
info.devices[id].smpbo = prop.sharedMemPerBlockOptin;
|
||||
info.devices[id].cc = 100*prop.major + 10*prop.minor;
|
||||
if (info.devices[id].virtual_index == 0) {
|
||||
info.devices[id].prefer_fp32_acc = should_use_fp32_acc(prop, info.devices[id].cc);
|
||||
} else {
|
||||
info.devices[id].prefer_fp32_acc = info.devices[physical_id].prefer_fp32_acc;
|
||||
}
|
||||
GGML_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s, VRAM: %zu MiB\n",
|
||||
id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no",
|
||||
device_vram_mib);
|
||||
@@ -403,6 +472,13 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
||||
#endif // defined(GGML_USE_HIP)
|
||||
}
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
if (nvml_initialized) {
|
||||
(void) shutdown();
|
||||
}
|
||||
library.reset();
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
if (ggml_cuda_highest_compiled_arch(GGML_CUDA_CC_TURING) >= GGML_CUDA_CC_TURING && !turing_devices_without_mma.empty()) {
|
||||
GGML_LOG_INFO("The following devices will have suboptimal performance due to a lack of tensor cores:\n");
|
||||
for (size_t device_pos = 0; device_pos < turing_devices_without_mma.size(); device_pos++) {
|
||||
@@ -1041,6 +1117,11 @@ static void ggml_backend_cuda_buffer_set_tensor_n(ggml_backend_buffer_t buffer,
|
||||
|
||||
ggml_cuda_set_device(ctx->device);
|
||||
|
||||
if (!ggml_cuda_pinned_memory_disabled() &&
|
||||
ggml_cuda_set_tensor_n(ctx->device, tensors, data, offsets, sizes, n_tensors)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
@@ -1641,7 +1722,7 @@ static void * ggml_backend_cuda_comm_init_rank(ggml_backend_t backend, int rank,
|
||||
// NCCL_MAX_CTAS still overrides this.
|
||||
ncclConfig_t config = NCCL_CONFIG_INITIALIZER;
|
||||
const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc;
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10) {
|
||||
config.maxCTAs = 8;
|
||||
}
|
||||
|
||||
@@ -2047,6 +2128,9 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml
|
||||
ggml_type compute_type = src0->type;
|
||||
if (ggml_is_quantized(compute_type) || compute_type == GGML_TYPE_F8_E4M3) {
|
||||
compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32;
|
||||
if (ggml_cuda_info().devices[ctx.device].prefer_fp32_acc) {
|
||||
compute_type = GGML_TYPE_BF16;
|
||||
}
|
||||
} else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) {
|
||||
compute_type = GGML_TYPE_F32;
|
||||
} else if (compute_type == GGML_TYPE_BF16 && !fast_bf16_hardware_available(cc)) {
|
||||
@@ -2263,7 +2347,7 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) {
|
||||
src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32;
|
||||
|
||||
// fusion is not universally faster on Pascal
|
||||
if (cc <= GGML_CUDA_CC_PASCAL) {
|
||||
if (cc < GGML_CUDA_CC_VOLTA) {
|
||||
return false;
|
||||
}
|
||||
//we only support fusion for ncols_dst = 1
|
||||
@@ -2275,6 +2359,11 @@ static bool ggml_cuda_should_fuse_mul_mat_vec_q(const ggml_tensor * tensor) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// do not fuse the matmuls that few-row MMQ takes in ggml_cuda_mul_mat_id
|
||||
if (tensor->op == GGML_OP_MUL_MAT_ID && ggml_cuda_should_use_mmq_few(src0, src1, tensor->src[2], cc)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return use_mul_mat_vec_q;
|
||||
}
|
||||
|
||||
@@ -2426,13 +2515,13 @@ static bool ggml_cuda_should_fuse_mul_mat_q(const ggml_tensor * up, const ggml_t
|
||||
const ggml_tensor * y = up->src[1];
|
||||
const int cc = ggml_cuda_info().devices[device].cc;
|
||||
|
||||
// For MUL_MAT_ID: same kernel choice as ggml_cuda_mul_mat_id, with the nominal tiles of all experts (the per-expert token counts are only known on the GPU).
|
||||
// For MUL_MAT_ID: the MMVQ/MMQ choice of ggml_cuda_mul_mat_id, with the nominal tiles of all experts (the per-expert token counts are only known on the GPU).
|
||||
const bool is_id = up->op == GGML_OP_MUL_MAT_ID;
|
||||
const int64_t ncols = is_id ? y->ne[2] : y->ne[1];
|
||||
const int64_t nchannels = is_id ? x_up->ne[2] : y->ne[2]*y->ne[3];
|
||||
const bool use_mmvq = is_id ? ncols <= get_mmvq_mmid_max_batch(x_up->type, cc) : ggml_cuda_should_use_mmvq(x_up->type, cc, ncols);
|
||||
// do not fuse the matmuls that few-row MMQ takes in ggml_cuda_mul_mat
|
||||
if (!is_id && ggml_cuda_should_use_mmq_few(x_up, y, cc)) {
|
||||
// do not fuse the matmuls that few-row MMQ takes in ggml_cuda_mul_mat; for MUL_MAT_ID the fused MMQ keeps the expert gate and up
|
||||
if (!is_id && ggml_cuda_should_use_mmq_few(x_up, y, nullptr, cc)) {
|
||||
return false;
|
||||
}
|
||||
if (use_mmvq || !ggml_cuda_should_use_mmq(x_up->type, cc, ncols, /*n_experts =*/ is_id ? x_up->ne[2] : 0)) {
|
||||
@@ -2488,13 +2577,13 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
|
||||
return;
|
||||
}
|
||||
// from a per-type number of src1 columns up to 24, few-row MMQ is faster than MMVQ and MMQ
|
||||
const bool use_mmq_few = ggml_cuda_should_use_mmq_few(src0, src1, cc);
|
||||
const bool use_mmq_few = ggml_cuda_should_use_mmq_few(src0, src1, nullptr, cc);
|
||||
if (!use_mmq_few && ggml_cuda_should_use_mmvq(src0->type, cc, ne11)) {
|
||||
ggml_cuda_mul_mat_vec_q(ctx, src0, src1, nullptr, dst);
|
||||
return;
|
||||
}
|
||||
if (use_mmq_few) {
|
||||
ggml_cuda_mul_mat_q_few(ctx, src0, src1, dst);
|
||||
ggml_cuda_mul_mat_q_few(ctx, src0, src1, nullptr, dst);
|
||||
return;
|
||||
}
|
||||
if (ggml_cuda_should_use_mmq(src0->type, cc, ne11, /*n_experts =*/ 0)) {
|
||||
@@ -2531,6 +2620,10 @@ static bool ggml_cuda_mul_mat_id_needs_sync(const ggml_tensor * dst, const int c
|
||||
}
|
||||
}
|
||||
|
||||
if (ggml_cuda_should_use_mmq_few(src0, src1, dst->src[2], cc)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ggml_cuda_should_use_mmq(src0->type, cc, src1->ne[2], /*n_experts=*/src0->ne[2])) {
|
||||
return false;
|
||||
}
|
||||
@@ -2557,8 +2650,11 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
|
||||
|
||||
// [TAG_MUL_MAT_ID_CUDA_GRAPHS]
|
||||
if (src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
|
||||
// up to 24 tokens few-row MMQ is faster than MMQ, on DGX Spark also faster than MMVQ
|
||||
const bool use_mmq_few = ggml_cuda_should_use_mmq_few(src0, src1, ids, cc);
|
||||
|
||||
static_assert(MMVQ_MAX_BATCH_SIZE == MMVF_MAX_BATCH_SIZE);
|
||||
if (ne2 <= MMVQ_MAX_BATCH_SIZE) {
|
||||
if (!use_mmq_few && ne2 <= MMVQ_MAX_BATCH_SIZE) {
|
||||
if (ggml_is_quantized(src0->type)) {
|
||||
const int mmvq_mmid_max = get_mmvq_mmid_max_batch(src0->type, cc);
|
||||
if (ne2 <= mmvq_mmid_max) {
|
||||
@@ -2578,6 +2674,11 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
|
||||
}
|
||||
}
|
||||
|
||||
if (use_mmq_few) {
|
||||
ggml_cuda_mul_mat_q_few(ctx, src0, src1, ids, dst);
|
||||
return;
|
||||
}
|
||||
|
||||
if (ggml_cuda_should_use_mmq(src0->type, cc, ne12, /*n_experts=*/ne02)) {
|
||||
ggml_cuda_mul_mat_q(ctx, src0, src1, ids, dst);
|
||||
return;
|
||||
@@ -2629,7 +2730,8 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
|
||||
ids_from_sorted_host[i12*n_expert_used + iex] = ids_to_sorted_host.size();
|
||||
ids_to_sorted_host.push_back(i12*ne11 + iex % ne11);
|
||||
tokens_per_expert[i02]++;
|
||||
break;
|
||||
// TODO: removing the break here was done to make duplicate IDs work but this caused a ~10% performance regression, better solution needed.
|
||||
// break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3050,6 +3152,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
|
||||
case GGML_OP_FILL:
|
||||
ggml_cuda_op_fill(ctx, dst);
|
||||
break;
|
||||
case GGML_OP_SLEEP:
|
||||
ggml_cuda_op_sleep(ctx, dst);
|
||||
break;
|
||||
case GGML_OP_LIGHTNING_INDEXER:
|
||||
ggml_cuda_lightning_indexer(ctx, dst);
|
||||
break;
|
||||
@@ -3400,7 +3505,12 @@ static bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope,
|
||||
return false;
|
||||
}
|
||||
|
||||
if (set_rows->type != GGML_TYPE_F32 && set_rows->type != GGML_TYPE_F16) {
|
||||
if (set_rows->type != GGML_TYPE_F32 && set_rows->type != GGML_TYPE_F16 && set_rows->type != GGML_TYPE_BF16) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// BF16 rope fuses into an F16 or BF16 cache, other ropes not into a BF16 cache
|
||||
if (rope->type == GGML_TYPE_BF16 ? set_rows->type == GGML_TYPE_F32 : set_rows->type == GGML_TYPE_BF16) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -4152,6 +4262,26 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph,
|
||||
return false;
|
||||
}
|
||||
|
||||
if ((ops.size() == 2 || ops.size() == 3) && ops.begin()[0] == GGML_OP_NORM && ops.begin()[1] == GGML_OP_MUL) {
|
||||
const ggml_tensor * norm = cgraph->nodes[node_idx];
|
||||
const ggml_tensor * mul = cgraph->nodes[node_idx + 1];
|
||||
const ggml_tensor * weight = mul->src[0] == norm ? mul->src[1] : mul->src[0];
|
||||
if (norm->src[0]->type != GGML_TYPE_F32 || norm->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32 ||
|
||||
weight->type != GGML_TYPE_F32 || !ggml_is_vector(weight) || weight->ne[0] != norm->ne[0] ||
|
||||
!ggml_is_contiguous(weight)) {
|
||||
return false;
|
||||
}
|
||||
if (ops.size() == 3) {
|
||||
const ggml_tensor * add = cgraph->nodes[node_idx + 2];
|
||||
const ggml_tensor * bias = add->src[0] == mul ? add->src[1] : add->src[0];
|
||||
if (ops.begin()[2] != GGML_OP_ADD || add->type != GGML_TYPE_F32 || bias->type != GGML_TYPE_F32 ||
|
||||
!ggml_is_vector(bias) || bias->ne[0] != norm->ne[0] || !ggml_is_contiguous(bias)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
if ((ops.size() == 2 || ops.size() == 3) && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) {
|
||||
const ggml_tensor *rms_norm = cgraph->nodes[node_idx];
|
||||
const ggml_tensor *mul = cgraph->nodes[node_idx+1];
|
||||
@@ -5119,6 +5249,16 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
|
||||
return 2;
|
||||
}
|
||||
|
||||
if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) {
|
||||
ggml_cuda_op_norm(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]);
|
||||
return 2;
|
||||
}
|
||||
|
||||
if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_NORM, GGML_OP_MUL }, {})) {
|
||||
ggml_cuda_op_norm(*cuda_ctx, node, cgraph->nodes[i + 1]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) {
|
||||
ggml_cuda_op_rms_norm_fused_add(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]);
|
||||
return 2;
|
||||
@@ -6376,7 +6516,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || op->type == GGML_TYPE_F8_E4M3 ||
|
||||
op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 ||
|
||||
op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) &&
|
||||
op->src[0]->type == GGML_TYPE_F32
|
||||
(op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16)
|
||||
) || (
|
||||
op->type == GGML_TYPE_F16 && op->src[0]->type == GGML_TYPE_F16
|
||||
)
|
||||
@@ -6547,7 +6687,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type &&
|
||||
ggml_is_contiguous_rows(op->src[0]);
|
||||
case GGML_OP_CLAMP:
|
||||
return ggml_is_contiguous_rows(op->src[0]);
|
||||
return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
|
||||
case GGML_OP_ADD:
|
||||
case GGML_OP_SUB:
|
||||
case GGML_OP_MUL:
|
||||
@@ -6597,7 +6737,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
return false;
|
||||
case GGML_OP_ROPE:
|
||||
case GGML_OP_ROPE_BACK: {
|
||||
return op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && ggml_is_contiguous_2(op->src[0]);
|
||||
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) &&
|
||||
op->type == op->src[0]->type &&
|
||||
op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && ggml_is_contiguous_2(op->src[0]);
|
||||
}
|
||||
case GGML_OP_IM2COL:
|
||||
case GGML_OP_IM2COL_3D:
|
||||
@@ -6647,7 +6789,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
case GGML_OP_PAD_REFLECT_1D:
|
||||
case GGML_OP_ARANGE:
|
||||
case GGML_OP_TIMESTEP_EMBEDDING:
|
||||
return true;
|
||||
case GGML_OP_LEAKY_RELU:
|
||||
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
|
||||
case GGML_OP_RWKV_WKV6:
|
||||
case GGML_OP_GATED_LINEAR_ATTN:
|
||||
case GGML_OP_RWKV_WKV7:
|
||||
@@ -6673,13 +6817,21 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
case GGML_OP_FILL:
|
||||
case GGML_OP_CUMSUM:
|
||||
case GGML_OP_TRI:
|
||||
return true;
|
||||
case GGML_OP_DIAG:
|
||||
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
|
||||
case GGML_OP_SOLVE_TRI:
|
||||
return true;
|
||||
case GGML_OP_LIGHTNING_INDEXER:
|
||||
return ggml_cuda_lightning_indexer_supported(dev_ctx->device, op);
|
||||
case GGML_OP_TOP_K_MASK:
|
||||
return op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32;
|
||||
case GGML_OP_SLEEP:
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
return false;
|
||||
#else
|
||||
return true;
|
||||
#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -43,7 +43,6 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
const int64_t s13 = src1->nb[3] / ts_src1;
|
||||
const int64_t s3 = dst->nb[3] / ts_dst;
|
||||
|
||||
const int64_t ids_s0 = ids ? ids->nb[0] / ggml_type_size(ids->type) : 0;
|
||||
const int64_t ids_s1 = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;
|
||||
|
||||
mmf_ids_data ids_info{};
|
||||
@@ -70,7 +69,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
nchannels_y = ids->ne[0];
|
||||
}
|
||||
|
||||
if (ids && ncols_dst > 16) {
|
||||
if (ids) {
|
||||
const int64_t n_expert_used = ids->ne[0];
|
||||
const int64_t n_experts = ne02;
|
||||
const int64_t n_tokens = ne12;
|
||||
@@ -110,7 +109,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
constexpr int vals_per_T = 1;
|
||||
mul_mat_f_switch_rows_per_block<float>(
|
||||
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
|
||||
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
|
||||
} break;
|
||||
case GGML_TYPE_F16: {
|
||||
@@ -118,7 +117,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
constexpr int vals_per_T = 2;
|
||||
mul_mat_f_switch_rows_per_block<half2>(
|
||||
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
|
||||
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
|
||||
} break;
|
||||
case GGML_TYPE_BF16: {
|
||||
@@ -126,7 +125,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
constexpr int vals_per_T = 2;
|
||||
mul_mat_f_switch_rows_per_block<nv_bfloat162>(
|
||||
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
|
||||
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
|
||||
} break;
|
||||
default:
|
||||
|
||||
+69
-176
@@ -47,12 +47,11 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
|
||||
bool ggml_cuda_should_use_mmf(enum ggml_type type, int cc, int warp_size, const int64_t * scr0_ne, const size_t * src0_nb, const int src1_ncols, bool mul_mat_id);
|
||||
|
||||
template <typename T, int rows_per_block, int cols_per_block, int nwarps, bool has_ids>
|
||||
template <typename T, int rows_per_block, int cols_per_block, int nwarps>
|
||||
__launch_bounds__(ggml_cuda_get_physical_warp_size()*nwarps, 1)
|
||||
static __global__ void mul_mat_f(
|
||||
const T * __restrict__ x, const float * __restrict__ y, const int32_t * __restrict__ ids, float * __restrict__ dst,
|
||||
const int ncols, const int ncols_dst_total, const int nchannels_dst, const int stride_row, const int stride_col_y, const int stride_col_dst,
|
||||
const int stride_col_id, const int stride_row_id,
|
||||
const T * __restrict__ x, const float * __restrict__ y, float * __restrict__ dst,
|
||||
const int ncols, const int stride_row, const int stride_col_y, const int stride_col_dst,
|
||||
const int channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
|
||||
const int sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst) {
|
||||
// TODO: handle this in a consistent and simpler way after AMD MFMA support has been added
|
||||
@@ -90,85 +89,26 @@ static __global__ void mul_mat_f(
|
||||
constexpr int ntA = rows_per_block / tile_A::I;
|
||||
constexpr int ntB = (cols_per_block + tile_B::I - 1) / tile_B::I;
|
||||
|
||||
const int row0 = blockIdx.x * rows_per_block;
|
||||
const int row0 = blockIdx.x * rows_per_block;
|
||||
|
||||
int expert_idx = 0;
|
||||
[[maybe_unused]] int col_base = 0;
|
||||
|
||||
const int channel_dst = has_ids ? 0 : blockIdx.y;
|
||||
|
||||
if constexpr (has_ids) {
|
||||
// experts + tiles of ncols_dst are packed in the y dimension
|
||||
int col_tiles = (ncols_dst_total + cols_per_block - 1) / cols_per_block;
|
||||
const int nchannels_x = gridDim.y / col_tiles;
|
||||
const int tile_idx = blockIdx.y / nchannels_x;
|
||||
expert_idx = blockIdx.y - tile_idx * nchannels_x;
|
||||
col_base = tile_idx * cols_per_block;
|
||||
}
|
||||
|
||||
const int channel_x = has_ids ? expert_idx : (channel_dst / channel_ratio);
|
||||
const int channel_dst = blockIdx.y;
|
||||
const int channel_x = channel_dst / channel_ratio;
|
||||
const int channel_y = channel_dst;
|
||||
const int sample_dst = blockIdx.z;
|
||||
const int sample_x = sample_dst / sample_ratio;
|
||||
const int sample_y = sample_dst;
|
||||
|
||||
x += int64_t(sample_x) *stride_sample_x + channel_x *stride_channel_x + row0*stride_row ;
|
||||
y += int64_t(sample_y) *stride_sample_y + (has_ids ? 0 : channel_y *stride_channel_y);
|
||||
dst += int64_t(sample_dst)*stride_sample_dst + (has_ids ? 0 : channel_dst*stride_channel_dst);
|
||||
|
||||
if constexpr (has_ids) {
|
||||
constexpr int y_stride_scale = std::is_same_v<T, float> ? 1 : 2;
|
||||
const int64_t col_offset = col_base;
|
||||
y += col_offset * stride_col_y * y_stride_scale;
|
||||
dst += col_offset * stride_col_dst;
|
||||
ids += col_offset * stride_row_id;
|
||||
}
|
||||
y += int64_t(sample_y) *stride_sample_y + channel_y *stride_channel_y;
|
||||
dst += int64_t(sample_dst)*stride_sample_dst + channel_dst*stride_channel_dst;
|
||||
|
||||
[[maybe_unused]] const float2 * y2 = (const float2 *) y;
|
||||
|
||||
extern __shared__ char data_mmv[];
|
||||
|
||||
char * shmem_base = data_mmv;
|
||||
[[maybe_unused]] int * slot_map = (int *) shmem_base;
|
||||
char * compute_base = has_ids ? (shmem_base + GGML_PAD(cols_per_block, 16) * sizeof(int)) : shmem_base;
|
||||
|
||||
tile_C C[ntA][ntB];
|
||||
|
||||
T * tile_xy = (T *) compute_base + threadIdx.y*(tile_A::I * tile_k_padded);
|
||||
|
||||
if constexpr (has_ids) {
|
||||
int found = 0;
|
||||
|
||||
for (int j0 = 0; j0 < cols_per_block; j0 += nwarps) {
|
||||
const int j = j0 + threadIdx.y;
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
slot_map[j] = -1;
|
||||
}
|
||||
ggml_cuda_syncwarp();
|
||||
|
||||
if (col_base + j >= ncols_dst_total) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const int32_t * __restrict__ id_row = ids + j*stride_row_id;
|
||||
|
||||
for (int k = threadIdx.x; k < nchannels_dst; k += warp_size) {
|
||||
int match = id_row[k*stride_col_id] == expert_idx;
|
||||
|
||||
if (match) {
|
||||
slot_map[j] = k;
|
||||
found = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!__syncthreads_or(found)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
T * tile_xy = (T *) data_mmv + threadIdx.y*(tile_A::I * tile_k_padded);
|
||||
|
||||
for (int col = threadIdx.y*warp_size + threadIdx.x; col < ncols; col += nwarps*warp_size) {
|
||||
tile_A A[ntA][warp_size / tile_A::J];
|
||||
@@ -194,26 +134,15 @@ static __global__ void mul_mat_f(
|
||||
for (int j0 = 0; j0 < tile_B::I; ++j0) {
|
||||
const int j = j0 + itB*tile_B::I;
|
||||
|
||||
if constexpr (!has_ids) {
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = j < cols_per_block ? y[j*stride_col_y + col] : 0.0f;
|
||||
} else {
|
||||
const bool valid = j < cols_per_block && (col_base + j) < ncols_dst_total && slot_map[j] >= 0;
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = valid ? y[slot_map[j]*stride_channel_y + j*stride_col_y + col] : 0.0f;
|
||||
}
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = j < cols_per_block ? y[j*stride_col_y + col] : 0.0f;
|
||||
}
|
||||
} else if constexpr (std::is_same_v<T, half2> || std::is_same_v<T, nv_bfloat162>) {
|
||||
#pragma unroll
|
||||
for (int j0 = 0; j0 < tile_B::I; ++j0) {
|
||||
const int j = j0 + itB*tile_B::I;
|
||||
|
||||
if constexpr (!has_ids) {
|
||||
const float2 tmp = j < cols_per_block ? y2[j*stride_col_y + col] : make_float2(0.0f, 0.0f);
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
|
||||
} else {
|
||||
const bool valid = j < cols_per_block && (col_base + j) < ncols_dst_total && slot_map[j] >= 0;
|
||||
float2 tmp = valid ? *(const float2*) &y[slot_map[j]*stride_channel_y + 2*(j*stride_col_y + col)] : make_float2(0.0f, 0.0f);
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
|
||||
}
|
||||
const float2 tmp = j < cols_per_block ? y2[j*stride_col_y + col] : make_float2(0.0f, 0.0f);
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
|
||||
}
|
||||
} else {
|
||||
static_assert(std::is_same_v<T, void>, "unsupported type");
|
||||
@@ -231,7 +160,7 @@ static __global__ void mul_mat_f(
|
||||
}
|
||||
}
|
||||
|
||||
float * buf_iw = (float *) compute_base;
|
||||
float * buf_iw = (float *) data_mmv;
|
||||
constexpr int kiw = nwarps*rows_per_block + mmf_get_padding();
|
||||
|
||||
if (nwarps > 1) {
|
||||
@@ -278,26 +207,15 @@ static __global__ void mul_mat_f(
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (!has_ids) {
|
||||
#pragma unroll
|
||||
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
|
||||
dst[j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
|
||||
}
|
||||
} else {
|
||||
const int slot = (j < cols_per_block) ? slot_map[j] : -1;
|
||||
if (slot >= 0 && (col_base + j) < ncols_dst_total) {
|
||||
#pragma unroll
|
||||
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
|
||||
dst[slot*stride_channel_dst + j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
|
||||
}
|
||||
}
|
||||
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
|
||||
dst[j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
GGML_UNUSED_VARS(x, y, ids, dst,
|
||||
ncols, ncols_dst_total, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id,
|
||||
GGML_UNUSED_VARS(x, y, dst,
|
||||
ncols, stride_row, stride_col_y, stride_col_dst,
|
||||
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
|
||||
NO_DEVICE_CODE;
|
||||
@@ -588,22 +506,21 @@ static __global__ void mul_mat_f_ids(
|
||||
|
||||
template<typename T, int rows_per_block, int cols_per_block, int nwarps>
|
||||
static inline void mul_mat_f_switch_ids(
|
||||
const T * x, const float * y, const int32_t * ids, float * dst,
|
||||
const T * x, const float * y, float * dst,
|
||||
const int64_t ncols_x, const int64_t ncols_dst, const int64_t nchannels_dst,
|
||||
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
|
||||
const int64_t stride_col_id, const int64_t stride_row_id,
|
||||
const int64_t channel_ratio, const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst,
|
||||
const int64_t sample_ratio, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
|
||||
const dim3 & block_nums, const dim3 & block_dims, const int nbytes_shared_total, cudaStream_t stream,
|
||||
const mmf_ids_data * ids_data) {
|
||||
|
||||
const bool has_ids_data = ids_data && ids_data->ids_src_compact;
|
||||
|
||||
// Use the compact-ids kernel only for larger tiles; for small ncols_dst (< 16)
|
||||
// we prefer the normal mul_mat_f path with has_ids=true.
|
||||
if (has_ids_data && ncols_dst > 16) {
|
||||
if (has_ids_data) {
|
||||
if (ids_data->max_tiles == 0) {
|
||||
return;
|
||||
}
|
||||
GGML_ASSERT(block_nums.z == 1);
|
||||
dim3 block_nums_ids(block_nums.x, ids_data->max_tiles, 1);
|
||||
|
||||
const uint3 sis1_fd = ids_data->sis1 > 0 ? init_fastdiv_values((uint32_t) ids_data->sis1) : make_uint3(0, 0, 1);
|
||||
@@ -615,19 +532,10 @@ static inline void mul_mat_f_switch_ids(
|
||||
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
sis1_fd, nch_fd);
|
||||
} else if (ids) {
|
||||
const int64_t col_tiles = (ncols_dst + cols_per_block - 1) / cols_per_block;
|
||||
dim3 block_nums_ids = block_nums;
|
||||
block_nums_ids.y *= col_tiles;
|
||||
|
||||
mul_mat_f<T, rows_per_block, cols_per_block, nwarps, true><<<block_nums_ids, block_dims, nbytes_shared_total, stream>>>
|
||||
(x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
|
||||
} else {
|
||||
mul_mat_f<T, rows_per_block, cols_per_block, nwarps, false><<<block_nums, block_dims, nbytes_shared_total, stream>>>
|
||||
(x, y, ids, dst, ncols_x, cols_per_block, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
mul_mat_f<T, rows_per_block, cols_per_block, nwarps><<<block_nums, block_dims, nbytes_shared_total, stream>>>
|
||||
(x, y, dst, ncols_x, stride_row, stride_col_y, stride_col_dst,
|
||||
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
|
||||
}
|
||||
}
|
||||
@@ -637,7 +545,6 @@ void mul_mat_f_cuda(
|
||||
const T * x, const float * y, const int32_t * ids, float * dst,
|
||||
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
|
||||
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
|
||||
const int64_t stride_col_id, const int64_t stride_row_id,
|
||||
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
|
||||
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
|
||||
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
|
||||
@@ -673,70 +580,59 @@ void mul_mat_f_cuda(
|
||||
const int nbytes_shared_iter = nwarps_best * (volta_mma_available(cc) ? tile_A_32::I : tile_A_16::I) * (warp_size + mmf_get_padding(cc)) * 4;
|
||||
const int nbytes_cols_per_block_pad = (amd_wmma_available(cc) || amd_mfma_available(cc)) ? tile_B_16::I : tile_B_8::I;
|
||||
const int nbytes_shared_combine = GGML_PAD(cols_per_block, nbytes_cols_per_block_pad) * (nwarps_best*rows_per_block + mmf_get_padding(cc)) * 4;
|
||||
const int nbytes_shared = std::max(nbytes_shared_iter, nbytes_shared_combine);
|
||||
const int nbytes_slotmap = ids ? GGML_PAD(cols_per_block, 16) * sizeof(int) : 0;
|
||||
const int nbytes_shared_total = nbytes_shared + nbytes_slotmap;
|
||||
const int64_t grid_y = ids ? nchannels_x : nchannels_dst;
|
||||
const int nbytes_shared_total = std::max(nbytes_shared_iter, nbytes_shared_combine);
|
||||
|
||||
const dim3 block_nums(nrows_x/rows_per_block, grid_y, nsamples_dst);
|
||||
const dim3 block_nums(nrows_x/rows_per_block, nchannels_dst, nsamples_dst);
|
||||
const dim3 block_dims(warp_size, nwarps_best, 1);
|
||||
|
||||
switch (nwarps_best) {
|
||||
case 1: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 1>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 2: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 2>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 3: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 3>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 4: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 4>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 5: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 5>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 6: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 6>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 7: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 7>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 8: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 8>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
default:
|
||||
GGML_ABORT("fatal error");
|
||||
@@ -750,95 +646,94 @@ static void mul_mat_f_switch_cols_per_block(
|
||||
const T * x, const float * y, const int32_t * ids, float * dst,
|
||||
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
|
||||
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
|
||||
const int64_t stride_col_id, const int stride_row_id,
|
||||
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
|
||||
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
|
||||
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
|
||||
cudaStream_t stream, const mmf_ids_data * ids_data) {
|
||||
|
||||
const int ncols_case = (ids && ncols_dst > 16) ? 16 : ncols_dst;
|
||||
const int ncols_case = ids ? 16 : ncols_dst;
|
||||
|
||||
GGML_ASSERT(ids || ncols_dst <= 16);
|
||||
|
||||
switch (ncols_case) {
|
||||
case 1: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 1>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 2: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 2>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 3: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 3>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 4: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 4>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 5: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 5>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 6: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 6>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 7: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 7>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 8: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 8>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 9: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 9>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 10: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 10>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 11: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 11>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 12: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 12>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 13: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 13>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 14: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 14>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 15: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 15>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 16: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 16>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
default:
|
||||
@@ -851,7 +746,6 @@ static void mul_mat_f_switch_rows_per_block(
|
||||
const int rows_per_block, const T * x, const float * y, const int32_t * ids, float * dst,
|
||||
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
|
||||
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
|
||||
const int64_t stride_col_id, const int stride_row_id,
|
||||
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
|
||||
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
|
||||
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
|
||||
@@ -860,13 +754,13 @@ static void mul_mat_f_switch_rows_per_block(
|
||||
case MMF_ROWS_PER_BLOCK: {
|
||||
mul_mat_f_switch_cols_per_block<T, MMF_ROWS_PER_BLOCK>(
|
||||
x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case MMF_ROWS_PER_BLOCK_CDNA: {
|
||||
mul_mat_f_switch_cols_per_block<T, MMF_ROWS_PER_BLOCK_CDNA>(
|
||||
x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
default:
|
||||
@@ -878,7 +772,6 @@ static void mul_mat_f_switch_rows_per_block(
|
||||
template void mul_mat_f_cuda<T, nrows_dst, ncols_dst>( \
|
||||
const T * x, const float * y, const int32_t * ids, float * dst, \
|
||||
const int64_t ncols_x, const int64_t nrows_x, int64_t ncols_dst_total, const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst, \
|
||||
const int64_t stride_col_id, const int64_t stride_row_id, \
|
||||
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst, \
|
||||
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,\
|
||||
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst, \
|
||||
|
||||
+159
-60
@@ -24,6 +24,50 @@ template <int n> struct mm_ids_pow2 { static constexpr int value = 2*mm_ids_pow2
|
||||
template <> struct mm_ids_pow2<1> { static constexpr int value = 1; };
|
||||
template <> struct mm_ids_pow2<0> { static constexpr int value = 1; };
|
||||
|
||||
template <bool increment_nex_prev, bool write_to_store>
|
||||
static __device__ __forceinline__ void check_experts(
|
||||
mm_ids_helper_store * store, int & nex_prev, int & it_compact,
|
||||
const int expert, const int it, const int iex, const int expert_found) {
|
||||
if (increment_nex_prev) {
|
||||
nex_prev += expert_found < expert;
|
||||
}
|
||||
|
||||
if (write_to_store) {
|
||||
const ggml_cuda_thread_mask_t iex_used = ggml_cuda_ballot(expert_found == expert);
|
||||
const ggml_cuda_thread_mask_t mask = (ggml_cuda_thread_mask_t(1) << threadIdx.x) - 1;
|
||||
const int it_compact_offset = ggml_cuda_popc(iex_used & mask);
|
||||
|
||||
if (expert_found == expert) {
|
||||
store[it_compact + it_compact_offset] = mm_ids_helper_store(it, iex);
|
||||
}
|
||||
|
||||
it_compact += ggml_cuda_popc(iex_used);
|
||||
}
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ void dump_store(
|
||||
int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst,
|
||||
const int n_expert_used, const int nchannels_y, const int sis1, const bool write_inverse,
|
||||
const mm_ids_helper_store * __restrict__ store, const int nex_prev, int & n_dumped, const int it_compact) {
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
|
||||
for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {
|
||||
const mm_ids_helper_store store_it = store[itc];
|
||||
const int it = store_it.it();
|
||||
const int iex_used = store_it.iex_used();
|
||||
ids_dst[nex_prev + n_dumped + itc] = it*n_expert_used + iex_used;
|
||||
// ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup
|
||||
if (write_inverse) {
|
||||
ids_src1[it*n_expert_used + iex_used] = nex_prev + n_dumped + itc;
|
||||
} else {
|
||||
ids_src1[nex_prev + n_dumped + itc] = it*sis1 + iex_used % nchannels_y;
|
||||
}
|
||||
}
|
||||
n_dumped += it_compact;
|
||||
}
|
||||
|
||||
constexpr int n_unroll = 4; // Manually unroll this many loop iterations for better performance.
|
||||
|
||||
// Helper function for mul_mat_id, converts ids to a more convenient format.
|
||||
// ids_src1 describes how to permute the flattened column indices of src1 in order to get a compact src1 tensor sorted by expert.
|
||||
// ids_dst describes the same mapping but for the dst tensor.
|
||||
@@ -36,85 +80,138 @@ static __global__ void mm_ids_helper(
|
||||
const int32_t * __restrict__ ids,
|
||||
int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec,
|
||||
const int n_tokens, const int n_expert_used_var, const int nchannels_y, const uint3 tile_width,
|
||||
const int si1, const int sis1, const bool write_inverse) {
|
||||
const int si1, const int sis1, const bool write_inverse, const int ne_store) {
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
const int n_expert_used = n_expert_used_template == 0 ? n_expert_used_var : n_expert_used_template;
|
||||
const int expert = blockIdx.x;
|
||||
|
||||
const int expert = blockIdx.x; // The expert this CUDA block handles.
|
||||
|
||||
extern __shared__ char data_mm_ids_helper[];
|
||||
mm_ids_helper_store * store = (mm_ids_helper_store *) data_mm_ids_helper;
|
||||
|
||||
int nex_prev = 0; // Number of columns for experts with a lower index.
|
||||
int n_dumped = 0; // Number of data values that have been dumped to output buffers.
|
||||
int it_compact = 0; // Running index for the compact slice of this expert.
|
||||
|
||||
// Step 1: Iterate over n_tokens, increment nex_prev to determine the output indices, fill up store.
|
||||
// Step 1.5: If the store does not have enough space to hold all values, iterate over the rest of the tokens just to increment nex_prev.
|
||||
// Step 2: Dump the stored values using nex_prev to get the correct output indices.
|
||||
// Step 3: If we were not able to process all values, resume at the index up to which we were able to dump the output and re-do.
|
||||
|
||||
if constexpr (n_expert_used_template == 0) {
|
||||
// Generic implementation:
|
||||
for (int it = 0; it < n_tokens; ++it) {
|
||||
int iex_used = -1; // The index at which the expert is used, if any.
|
||||
for (int iex = threadIdx.x; iex < n_expert_used; iex += warp_size) {
|
||||
const int expert_used = ids[it*si1 + iex];
|
||||
nex_prev += expert_used < expert;
|
||||
if (expert_used == expert) {
|
||||
iex_used = iex;
|
||||
int it = 0;
|
||||
for (; it < n_tokens && it_compact + max(warp_size, n_expert_used) <= ne_store; ++it) {
|
||||
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
|
||||
const int iex = iex0 + threadIdx.x;
|
||||
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = true;
|
||||
constexpr bool write_to_store = true;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
}
|
||||
}
|
||||
int it_dumped = it;
|
||||
for (; it < n_tokens; ++it) {
|
||||
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
|
||||
const int iex = iex0 + threadIdx.x;
|
||||
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = true;
|
||||
constexpr bool write_to_store = false;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
}
|
||||
}
|
||||
|
||||
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
it_compact = 0;
|
||||
|
||||
for (int it = it_dumped; it < n_tokens; ++it) {
|
||||
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
|
||||
const int iex = iex0 + threadIdx.x;
|
||||
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = false;
|
||||
constexpr bool write_to_store = true;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
if (it_compact + max(warp_size, n_expert_used) >= ne_store) {
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
it_compact = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (iex_used != -1) {
|
||||
store[it_compact] = mm_ids_helper_store(it, iex_used);
|
||||
}
|
||||
|
||||
if (warp_reduce_any<warp_size>(iex_used != -1)) {
|
||||
it_compact++;
|
||||
}
|
||||
}
|
||||
if (it_compact > 0) {
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
}
|
||||
} else {
|
||||
// Implementation optimized for specific numbers of experts used:
|
||||
// a warp holds a whole number of token slots, so the slot count is padded to a power of 2
|
||||
constexpr int neu_padded = mm_ids_pow2<n_expert_used_template>::value;
|
||||
static_assert(neu_padded <= warp_size && warp_size % neu_padded == 0, "bad n_expert_used");
|
||||
for (int it0 = 0; it0 < n_tokens; it0 += warp_size/neu_padded) {
|
||||
const int it = it0 + threadIdx.x / neu_padded;
|
||||
const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any.
|
||||
|
||||
const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any.
|
||||
const int expert_used = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
|
||||
ids[it*si1 + iex] : INT_MAX;
|
||||
const int iex_used = expert_used == expert ? iex : -1;
|
||||
nex_prev += expert_used < expert;
|
||||
|
||||
// Whether the threads at this token position have used the expert:
|
||||
const int it_compact_add_self = warp_reduce_any<neu_padded>(iex_used != -1);
|
||||
|
||||
// Do a scan over threads at lower token positions in warp to get the correct index for writing data:
|
||||
int it_compact_add_lower = 0;
|
||||
int it0 = 0;
|
||||
for (; it0 < n_tokens && it_compact + n_unroll*warp_size <= ne_store; it0 += n_unroll*warp_size/neu_padded) {
|
||||
#pragma unroll
|
||||
for (int offset = neu_padded; offset < warp_size; offset += neu_padded) {
|
||||
const int tmp = ggml_cuda_shfl_up(it_compact_add_self, offset, warp_size);
|
||||
if (threadIdx.x >= static_cast<unsigned int>(offset)) {
|
||||
it_compact_add_lower += tmp;
|
||||
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
|
||||
const int it = it0 + it1 + threadIdx.x / neu_padded;
|
||||
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
|
||||
ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = true;
|
||||
constexpr bool write_to_store = true;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
}
|
||||
}
|
||||
int it0_dumped = it0;
|
||||
for (; it0 < n_tokens; it0 += n_unroll*warp_size/neu_padded) {
|
||||
#pragma unroll
|
||||
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
|
||||
const int it = it0 + it1 + threadIdx.x / neu_padded;
|
||||
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
|
||||
ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = true;
|
||||
constexpr bool write_to_store = false;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
}
|
||||
}
|
||||
|
||||
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
it_compact = 0;
|
||||
|
||||
for (it0 = it0_dumped; it0 < n_tokens; it0 += n_unroll*warp_size/neu_padded) {
|
||||
#pragma unroll
|
||||
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
|
||||
const int it = it0 + it1 + threadIdx.x / neu_padded;
|
||||
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
|
||||
ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = false;
|
||||
constexpr bool write_to_store = true;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
if (it_compact + warp_size >= ne_store) {
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
it_compact = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (iex_used != -1) {
|
||||
store[it_compact + it_compact_add_lower] = mm_ids_helper_store(it, iex_used);
|
||||
}
|
||||
|
||||
// The thread with the highest index in the warp always has the sum over the whole warp, use it to increment all threads:
|
||||
it_compact += ggml_cuda_shfl(it_compact_add_lower + it_compact_add_self, warp_size - 1, warp_size);
|
||||
}
|
||||
}
|
||||
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
|
||||
ggml_cuda_syncwarp();
|
||||
|
||||
for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {
|
||||
const mm_ids_helper_store store_it = store[itc];
|
||||
const int it = store_it.it();
|
||||
const int iex_used = store_it.iex_used();
|
||||
ids_dst[nex_prev + itc] = it*n_expert_used + iex_used;
|
||||
// ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup
|
||||
if (write_inverse) {
|
||||
ids_src1[it*n_expert_used + iex_used] = nex_prev + itc;
|
||||
} else {
|
||||
ids_src1[nex_prev + itc] = it*sis1 + iex_used % nchannels_y;
|
||||
if (it_compact > 0) {
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,7 +220,7 @@ static __global__ void mm_ids_helper(
|
||||
}
|
||||
|
||||
int col_low = nex_prev;
|
||||
int col_high = nex_prev + min(int(tile_width.z), it_compact);
|
||||
int col_high = nex_prev + min(int(tile_width.z), n_dumped);
|
||||
|
||||
// The while loop covers all tiles at the seams containing the column with % tile_width == 0.
|
||||
while (col_high > fastdiv(col_low + tile_width.z - 1, tile_width) * tile_width.z) {
|
||||
@@ -131,7 +228,7 @@ static __global__ void mm_ids_helper(
|
||||
= {expert, col_low, col_high, /*padding =*/ 0};
|
||||
|
||||
col_low = col_high;
|
||||
col_high = min(col_high + int(tile_width.z), nex_prev + it_compact);
|
||||
col_high = min(col_high + int(tile_width.z), nex_prev + n_dumped);
|
||||
}
|
||||
|
||||
// Finally, at most one extra tile_spec between the seams per expert.
|
||||
@@ -157,10 +254,12 @@ static void launch_mm_ids_helper(
|
||||
|
||||
const dim3 num_blocks(n_experts, 1, 1);
|
||||
const dim3 block_size(warp_size, 1, 1);
|
||||
const size_t nbytes_shared = n_tokens*sizeof(mm_ids_helper_store);
|
||||
GGML_ASSERT(nbytes_shared <= smpbo);
|
||||
const size_t ne_shared_pad = n_expert_used_template == 0 ? std::max(n_expert_used_var, warp_size) : n_unroll*warp_size;
|
||||
const size_t nbytes_store_core = std::min(smpbo - ne_shared_pad * sizeof(mm_ids_helper_store), n_tokens * sizeof(mm_ids_helper_store));
|
||||
const size_t nbytes_shared = nbytes_store_core + ne_shared_pad * sizeof(mm_ids_helper_store);
|
||||
const size_t ne_store = nbytes_shared / sizeof(mm_ids_helper_store);
|
||||
mm_ids_helper<n_expert_used_template><<<num_blocks, block_size, nbytes_shared, stream>>>
|
||||
(ids, ids_src1, ids_dst, tile_spec, n_tokens, n_expert_used_var, nchannels_y, tile_width_fd, si1, sis1, write_inverse);
|
||||
(ids, ids_src1, ids_dst, tile_spec, n_tokens, n_expert_used_var, nchannels_y, tile_width_fd, si1, sis1, write_inverse, ne_store);
|
||||
}
|
||||
|
||||
void ggml_cuda_launch_mm_ids_helper(
|
||||
|
||||
@@ -1,326 +1,326 @@
|
||||
static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_gcn(ggml_type type, int J, bool fallback) {
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_1, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_1, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q8_0, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q2_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q2_K, 512, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q3_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q3_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q3_K, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q3_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_K, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_K, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 3, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 3, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q5_K, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 56, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 56, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q6_K, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, true, false);
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
// CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
// CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
// CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
// CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
// CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
// CASE(GGML_TYPE_IQ2_XS, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, true);
|
||||
// CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, true, false);
|
||||
// CASE(GGML_TYPE_IQ2_S, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
// CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
// CASE(GGML_TYPE_IQ3_XXS, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
// CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
// CASE(GGML_TYPE_IQ3_S, 512, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false);
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_MXFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_MXFP4, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, true, false);
|
||||
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_NVFP4, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_NVFP4, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_NVFP4, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, true);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_NVFP4, 512, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
|
||||
CASE(GGML_TYPE_NVFP4, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, true, false);
|
||||
|
||||
return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true);
|
||||
}
|
||||
|
||||
@@ -2,86 +2,86 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
@@ -162,14 +162,14 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
@@ -198,14 +198,14 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
@@ -222,26 +222,26 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
+137
-70
@@ -1,6 +1,7 @@
|
||||
#include "mmq-few.cuh"
|
||||
#include "cp-async.cuh"
|
||||
#include "mma.cuh"
|
||||
#include "mmvq.cuh"
|
||||
#include "quantize.cuh"
|
||||
#include "vecdotq.cuh"
|
||||
|
||||
@@ -13,6 +14,7 @@ using namespace ggml_cuda_mma;
|
||||
|
||||
// Few-row MMQ: int8 MMA mat-mul for 2-24 src1 columns. A CUDA block has 16 src0 rows (32 for 2 column groups on large matrices).
|
||||
// Its warps split K in units of 256 values and copy raw src0 rows into their own cp.async ring, src1 is quantized to the MMA fragment layout.
|
||||
// MUL_MAT_ID: one column group per expert and CUDA block, for short rows the warps split the rows instead of K (64 rows per block).
|
||||
|
||||
static constexpr int MMQ_FEW_MAX_NCG = 3; // max. number of column groups
|
||||
static constexpr int MMQ_FEW_NROWS = 16; // src0 rows of one MMA tile
|
||||
@@ -80,6 +82,11 @@ static constexpr __host__ __device__ bool mmq_few_tall_supported(const ggml_type
|
||||
type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ4_NL || type == GGML_TYPE_IQ4_XS;
|
||||
}
|
||||
|
||||
// MUL_MAT_ID with fewer than 2 units of 256 values per warp: the warps split the rows, each warp has its own 16 rows.
|
||||
static constexpr __host__ __device__ bool mmq_few_ksplit(const bool has_ids, const int nunits) {
|
||||
return !has_ids || nunits >= 2*MMQ_FEW_NWARPS;
|
||||
}
|
||||
|
||||
#if defined(CP_ASYNC_AVAILABLE)
|
||||
|
||||
// The src1 fragments of a lane for one unit of 256 values, see quantize.cuh.
|
||||
@@ -484,17 +491,19 @@ static __device__ __forceinline__ void mmq_few_vec_dot(const char * const (&x)[2
|
||||
|
||||
#endif // defined(CP_ASYNC_AVAILABLE)
|
||||
|
||||
template <ggml_type type, int ncg, int nmt>
|
||||
// has_ids: MUL_MAT_ID, blockIdx.y is a pair of quantize_mmq_few_id_q8_1, its expert is the src0 channel and dst dims 1 and 2 are slot and token.
|
||||
template <ggml_type type, int ncg, int nmt, bool has_ids>
|
||||
__launch_bounds__(MMQ_FEW_NWARPS*ggml_cuda_get_physical_warp_size(), 1)
|
||||
static __global__ void mul_mat_q_few(
|
||||
const char * x, const int4 * y_ptr, float * dst,
|
||||
const int nunits, const int nrows_x, const int ncols_dst, const int64_t stride_row_x, const int stride_col_dst,
|
||||
const int channel_ratio, const int64_t stride_channel_x, const int stride_channel_dst,
|
||||
const int sample_ratio, const int64_t stride_sample_x, const int stride_sample_dst, const bool x_is_constant) {
|
||||
const int sample_ratio, const int64_t stride_sample_x, const int stride_sample_dst, const bool x_is_constant, const int * pairs) {
|
||||
#if defined(CP_ASYNC_AVAILABLE)
|
||||
constexpr int nwarps = MMQ_FEW_NWARPS;
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
static_assert(warp_size == WARP_SIZE, "unsupported warp size");
|
||||
static_assert(!has_ids || (ncg == 1 && nmt == 1), "unsupported MUL_MAT_ID config");
|
||||
|
||||
constexpr int unit_size = mmq_few_unit_size(type);
|
||||
constexpr int nrows = nmt*MMQ_FEW_NROWS;
|
||||
@@ -510,13 +519,30 @@ static __global__ void mul_mat_q_few(
|
||||
extern __shared__ char data_mmq_few[];
|
||||
char * ring = data_mmq_few + threadIdx.y*nstages*nbytes_stage;
|
||||
|
||||
const int row0 = blockIdx.x*nrows;
|
||||
const bool ksplit = mmq_few_ksplit(has_ids, nunits);
|
||||
const int row0_blk = blockIdx.x*(ksplit ? nrows : nwarps*nrows);
|
||||
const int row0 = ksplit ? row0_blk : row0_blk + threadIdx.y*nrows;
|
||||
const int iu_begin = ksplit ? nunits_stage*threadIdx.y : 0;
|
||||
const int iu_step = ksplit ? nunits_stage*nwarps : nunits_stage;
|
||||
|
||||
const int channel_dst = blockIdx.y;
|
||||
const int sample_dst = blockIdx.z;
|
||||
const int nchannels = gridDim.y;
|
||||
|
||||
x += int64_t(sample_dst/sample_ratio)*stride_sample_x + int64_t(channel_dst/channel_ratio)*stride_channel_x;
|
||||
dst += int64_t(sample_dst)*stride_sample_dst + int64_t(channel_dst)*stride_channel_dst;
|
||||
int ncols = ncols_dst;
|
||||
[[maybe_unused]] const int * pair = nullptr;
|
||||
if constexpr (has_ids) {
|
||||
ggml_cuda_pdl_sync(); // the pairs come from the preceding kernel
|
||||
pair = pairs + channel_dst*MMQ_FEW_PAIR_INTS;
|
||||
ncols = pair[1];
|
||||
if (ncols == 0) {
|
||||
return;
|
||||
}
|
||||
x += int64_t(pair[0])*stride_channel_x;
|
||||
} else {
|
||||
x += int64_t(sample_dst/sample_ratio)*stride_sample_x + int64_t(channel_dst/channel_ratio)*stride_channel_x;
|
||||
dst += int64_t(sample_dst)*stride_sample_dst + int64_t(channel_dst)*stride_channel_dst;
|
||||
}
|
||||
const int4 * GGML_CUDA_RESTRICT y = y_ptr + (int64_t(sample_dst)*nchannels + channel_dst)*nunits*ncg*MMQ_FEW_Q8_1_INT4;
|
||||
|
||||
// No bounds check for the units, the gate makes nunits a multiple of nunits_stage.
|
||||
@@ -558,28 +584,26 @@ static __global__ void mul_mat_q_few(
|
||||
ggml_cuda_pdl_sync();
|
||||
}
|
||||
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
// DGX Spark DRAM serves the short row pieces of the stage copies slowly, so the rows of the block are first prefetched into L2 in address order.
|
||||
{
|
||||
const int tid = threadIdx.y*warp_size + threadIdx.x;
|
||||
const int nrows_blk = min(nrows, nrows_x - row0);
|
||||
const int nrows_blk = min(ksplit ? nrows : nwarps*nrows, nrows_x - row0_blk);
|
||||
const int64_t row_bytes = int64_t(nunits)*unit_size;
|
||||
const bool contig = stride_row_x == row_bytes;
|
||||
for (int ir = 0; ir < (contig ? 1 : nrows_blk); ++ir) {
|
||||
const uintptr_t p0 = (uintptr_t) (x + int64_t(row0 + ir)*stride_row_x);
|
||||
const uintptr_t p0 = (uintptr_t) (x + int64_t(row0_blk + ir)*stride_row_x);
|
||||
const uintptr_t p1 = p0 + (contig ? nrows_blk*row_bytes : row_bytes);
|
||||
for (uintptr_t p = (p0 & ~uintptr_t(127)) + 128*tid; p < p1; p += 128*nwarps*warp_size) {
|
||||
ggml_cuda_prefetch_l2((const void *) p);
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
|
||||
const int iu_step = nunits_stage*nwarps;
|
||||
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
|
||||
#pragma unroll
|
||||
for (int stage = 0; stage < nstages - 1; ++stage) {
|
||||
const int iu0 = nunits_stage*threadIdx.y + stage*iu_step;
|
||||
const int iu0 = iu_begin + stage*iu_step;
|
||||
if (iu0 < nunits) {
|
||||
load_stage(stage, iu0);
|
||||
}
|
||||
@@ -598,16 +622,16 @@ static __global__ void mul_mat_q_few(
|
||||
|
||||
// The next src1 fragments are loaded right after the MMAs that use the current ones, an earlier load is slower.
|
||||
mmq_few_y_frag yf[nunits_stage][ncg];
|
||||
if (nunits_stage*threadIdx.y < nunits) {
|
||||
if (iu_begin < nunits) {
|
||||
#pragma unroll
|
||||
for (int k = 0; k < nunits_stage; ++k) {
|
||||
load_y(yf[k], nunits_stage*threadIdx.y + k);
|
||||
load_y(yf[k], iu_begin + k);
|
||||
}
|
||||
}
|
||||
|
||||
float sum[nmt][ncg][4] = {{{0.0f}}};
|
||||
|
||||
for (int iter = 0, iu0 = nunits_stage*threadIdx.y; iu0 < nunits; ++iter, iu0 += iu_step) {
|
||||
for (int iter = 0, iu0 = iu_begin; iu0 < nunits; ++iter, iu0 += iu_step) {
|
||||
const int iu_load = iu0 + (nstages - 1)*iu_step;
|
||||
if (iu_load < nunits) {
|
||||
load_stage((iter + nstages - 1) % nstages, iu_load);
|
||||
@@ -643,26 +667,28 @@ static __global__ void mul_mat_q_few(
|
||||
ggml_cuda_pdl_lc();
|
||||
|
||||
// reduce the sums over the warps as one flat array
|
||||
constexpr int nsum = nmt*ncg*4;
|
||||
float * sum_flat = &sum[0][0][0];
|
||||
if (ksplit) {
|
||||
constexpr int nsum = nmt*ncg*4;
|
||||
float * sum_flat = &sum[0][0][0];
|
||||
|
||||
float * buf_sum = (float *) data_mmq_few;
|
||||
__syncthreads();
|
||||
if (threadIdx.y > 0) {
|
||||
float * buf_sum = (float *) data_mmq_few;
|
||||
__syncthreads();
|
||||
if (threadIdx.y > 0) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < nsum; ++i) {
|
||||
buf_sum[((threadIdx.y - 1)*nsum + i)*warp_size + threadIdx.x] = sum_flat[i];
|
||||
for (int i = 0; i < nsum; ++i) {
|
||||
buf_sum[((threadIdx.y - 1)*nsum + i)*warp_size + threadIdx.x] = sum_flat[i];
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (threadIdx.y > 0) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (threadIdx.y > 0) {
|
||||
return;
|
||||
}
|
||||
#pragma unroll
|
||||
for (int w = 0; w < nwarps - 1; ++w) {
|
||||
for (int w = 0; w < nwarps - 1; ++w) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < nsum; ++i) {
|
||||
sum_flat[i] += buf_sum[(w*nsum + i)*warp_size + threadIdx.x];
|
||||
for (int i = 0; i < nsum; ++i) {
|
||||
sum_flat[i] += buf_sum[(w*nsum + i)*warp_size + threadIdx.x];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -674,8 +700,13 @@ static __global__ void mul_mat_q_few(
|
||||
for (int l = 0; l < 4; ++l) {
|
||||
const int i = row0 + MMQ_FEW_NROWS*m + tile<16, 8, int>::get_i(l);
|
||||
const int j = c*MMQ_FEW_NCOLS + tile<16, 8, int>::get_j(l);
|
||||
if (i < nrows_x && j < ncols_dst) {
|
||||
dst[int64_t(j)*stride_col_dst + i] = sum[m][c][l];
|
||||
if (i < nrows_x && j < ncols) {
|
||||
if constexpr (has_ids) {
|
||||
const int e = pair[2 + j]; // token*n_expert_used + slot
|
||||
dst[int64_t(e / ncols_dst)*stride_channel_dst + int64_t(e % ncols_dst)*stride_col_dst + i] = sum[m][c][l];
|
||||
} else {
|
||||
dst[int64_t(j)*stride_col_dst + i] = sum[m][c][l];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -683,7 +714,7 @@ static __global__ void mul_mat_q_few(
|
||||
#else
|
||||
GGML_UNUSED_VARS(x, y_ptr, dst, nunits, nrows_x, ncols_dst, stride_row_x, stride_col_dst,
|
||||
channel_ratio, stride_channel_x, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_dst, x_is_constant);
|
||||
sample_ratio, stride_sample_x, stride_sample_dst, x_is_constant, pairs);
|
||||
NO_DEVICE_CODE;
|
||||
#endif // defined(CP_ASYNC_AVAILABLE)
|
||||
}
|
||||
@@ -695,9 +726,10 @@ static bool mmq_few_is_constant(const ggml_tensor * src0) {
|
||||
!(x_root->flags & GGML_TENSOR_FLAG_PARAM);
|
||||
}
|
||||
|
||||
template <ggml_type type, int ncg, int nmt>
|
||||
// MUL_MAT_ID: grid y is the pairs, the kernel takes the expert from the pair and does not use the channel and sample ratios.
|
||||
template <ggml_type type, int ncg, int nmt, bool has_ids = false>
|
||||
static void launch_mul_mat_q_few(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
|
||||
const int4 * src1_q, cudaStream_t stream) {
|
||||
const int4 * src1_q, cudaStream_t stream, const int * pairs = nullptr, const int npairs_max = 0) {
|
||||
GGML_TENSOR_BINARY_OP_LOCALS;
|
||||
|
||||
const int nunits = ne00 / QK_K;
|
||||
@@ -705,18 +737,19 @@ static void launch_mul_mat_q_few(const ggml_tensor * src0, const ggml_tensor * s
|
||||
|
||||
const int warp_size = ggml_cuda_info().devices[ggml_cuda_get_device()].warp_size;
|
||||
const int nbytes_shared = mmq_few_nbytes_shared(type, ncg, nmt);
|
||||
CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q_few<type, ncg, nmt>), nbytes_shared);
|
||||
CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q_few<type, ncg, nmt, has_ids>), nbytes_shared);
|
||||
|
||||
const bool x_is_constant = mmq_few_is_constant(src0);
|
||||
|
||||
const dim3 block_nums((ne01 + nmt*MMQ_FEW_NROWS - 1) / (nmt*MMQ_FEW_NROWS), ne12, ne13);
|
||||
const int nrows_blk = (mmq_few_ksplit(has_ids, nunits) ? 1 : MMQ_FEW_NWARPS)*nmt*MMQ_FEW_NROWS;
|
||||
const dim3 block_nums((ne01 + nrows_blk - 1) / nrows_blk, has_ids ? npairs_max : ne12, ne13);
|
||||
const dim3 block_dims(warp_size, MMQ_FEW_NWARPS, 1);
|
||||
const ggml_cuda_kernel_launch_params params(block_nums, block_dims, nbytes_shared, stream);
|
||||
ggml_cuda_kernel_launch(mul_mat_q_few<type, ncg, nmt>, params,
|
||||
ggml_cuda_kernel_launch(mul_mat_q_few<type, ncg, nmt, has_ids>, params,
|
||||
(const char *) src0->data, src1_q, (float *) dst->data,
|
||||
nunits, (int) ne01, (int) ne1, (int64_t) nb01, (int) (nb1/sizeof(float)),
|
||||
(int) (ne12/ne02), (int64_t) nb02, (int) (nb2/sizeof(float)),
|
||||
(int) (ne13/ne03), (int64_t) nb03, (int) (nb3/sizeof(float)), x_is_constant);
|
||||
(int) (ne13/ne03), (int64_t) nb03, (int) (nb3/sizeof(float)), x_is_constant, pairs);
|
||||
}
|
||||
|
||||
static int mmq_few_ncg(const int64_t ncols) {
|
||||
@@ -743,7 +776,11 @@ static void launch_mul_mat_q_few_tall(const ggml_tensor * src0, const ggml_tenso
|
||||
|
||||
template <ggml_type type>
|
||||
static void launch_mul_mat_q_few_switch_ncg(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
|
||||
const int4 * src1_q, const int ncg, cudaStream_t stream) {
|
||||
const int4 * src1_q, const int * pairs, const int npairs_max, const int ncg, cudaStream_t stream) {
|
||||
if (pairs) {
|
||||
launch_mul_mat_q_few<type, 1, 1, true>(src0, src1, dst, src1_q, stream, pairs, npairs_max);
|
||||
return;
|
||||
}
|
||||
switch (ncg) {
|
||||
case 1:
|
||||
launch_mul_mat_q_few<type, 1, 1>(src0, src1, dst, src1_q, stream);
|
||||
@@ -759,7 +796,7 @@ static void launch_mul_mat_q_few_switch_ncg(const ggml_tensor * src0, const ggml
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
||||
void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {
|
||||
GGML_TENSOR_BINARY_OP_LOCALS;
|
||||
|
||||
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
||||
@@ -767,51 +804,73 @@ void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor
|
||||
GGML_ASSERT(nb00 == ggml_type_size(src0->type));
|
||||
GGML_ASSERT(nb10 == sizeof(float));
|
||||
GGML_ASSERT(nb0 == sizeof(float));
|
||||
GGML_ASSERT(ne11 <= MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS);
|
||||
GGML_ASSERT(ne00 % QK_K == 0);
|
||||
GGML_ASSERT(ne12 % ne02 == 0 && ne13 % ne03 == 0);
|
||||
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
const int nunits = ne00 / QK_K;
|
||||
const int ncg = mmq_few_ncg(ne11);
|
||||
const int ncg = ids ? 1 : mmq_few_ncg(ne11);
|
||||
|
||||
ggml_cuda_pool_alloc<int4> src1_q(ctx.pool(), int64_t(ne12)*ne13*nunits*ncg*MMQ_FEW_Q8_1_INT4);
|
||||
quantize_mmq_few_q8_1_cuda((const float *) src1->data, src1_q.get(), mmq_few_ds_layout(src0->type), ne10, nb11/sizeof(float), nb12/sizeof(float), nb13/sizeof(float),
|
||||
ne11, ne12, ne13, stream);
|
||||
ggml_cuda_pool_alloc<int> pairs(ctx.pool());
|
||||
ggml_cuda_pool_alloc<int4> src1_q(ctx.pool());
|
||||
int npairs_max = 0;
|
||||
if (ids) {
|
||||
GGML_ASSERT(ids->type == GGML_TYPE_I32);
|
||||
GGML_ASSERT(ids->nb[0] == ggml_type_size(ids->type));
|
||||
GGML_ASSERT(ne1 == ids->ne[0] && ne13 == 1 && ne3 == 1);
|
||||
|
||||
const int64_t n_expert_used = ids->ne[0];
|
||||
const int64_t n_tokens = ids->ne[1];
|
||||
const int64_t nentries = n_tokens*n_expert_used;
|
||||
const int64_t nused = std::min(ne02, nentries);
|
||||
// an expert with c > 0 entries (at most one per token) has ceil(c/MMQ_FEW_NCOLS) pairs
|
||||
npairs_max = (int) std::min(nused + (nentries - nused)/MMQ_FEW_NCOLS, ne02*((n_tokens + MMQ_FEW_NCOLS - 1) / MMQ_FEW_NCOLS));
|
||||
|
||||
pairs.alloc(int64_t(npairs_max)*MMQ_FEW_PAIR_INTS);
|
||||
src1_q.alloc(int64_t(npairs_max)*nunits*MMQ_FEW_Q8_1_INT4);
|
||||
quantize_mmq_few_id_q8_1_cuda((const float *) src1->data, (const int32_t *) ids->data, src1_q.get(), pairs.get(), mmq_few_ds_layout(src0->type),
|
||||
ne00, nb11/sizeof(float), nb12/sizeof(float), ne11, ne02, n_expert_used, n_tokens, ids->nb[1]/sizeof(int32_t), npairs_max, stream);
|
||||
} else {
|
||||
GGML_ASSERT(ne11 <= MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS);
|
||||
GGML_ASSERT(ne12 % ne02 == 0 && ne13 % ne03 == 0);
|
||||
|
||||
src1_q.alloc(int64_t(ne12)*ne13*nunits*ncg*MMQ_FEW_Q8_1_INT4);
|
||||
quantize_mmq_few_q8_1_cuda((const float *) src1->data, src1_q.get(), mmq_few_ds_layout(src0->type), ne10, nb11/sizeof(float), nb12/sizeof(float), nb13/sizeof(float),
|
||||
ne11, ne12, ne13, stream);
|
||||
}
|
||||
|
||||
switch (src0->type) {
|
||||
case GGML_TYPE_Q8_0:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q8_0>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q8_0>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q3_K:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q3_K>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q3_K>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q4_K:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q4_K>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q4_K>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q5_K:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q5_K>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q5_K>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_Q6_K:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q6_K>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_Q6_K>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_IQ3_S:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ3_S>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ3_S>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_IQ4_NL:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_NL>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_NL>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
case GGML_TYPE_IQ4_XS:
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_XS>(src0, src1, dst, src1_q.get(), ncg, stream);
|
||||
launch_mul_mat_q_few_switch_ncg<GGML_TYPE_IQ4_XS>(src0, src1, dst, src1_q.get(), pairs.get(), npairs_max, ncg, stream);
|
||||
break;
|
||||
default:
|
||||
GGML_ABORT("unsupported type for few-row MMQ");
|
||||
}
|
||||
}
|
||||
|
||||
// Min. number of src1 columns for the few-row MMQ, 0 if it is not used for this type, GPU and row length.
|
||||
static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const int64_t ne00) {
|
||||
// Min. number of src1 columns (MUL_MAT_ID: tokens) for the few-row MMQ, 0 if it is not used for this type, GPU and row length.
|
||||
static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const int64_t ne00, const bool has_ids) {
|
||||
#ifdef GGML_CUDA_FORCE_CUBLAS
|
||||
return 0;
|
||||
#endif // GGML_CUDA_FORCE_CUBLAS
|
||||
@@ -821,16 +880,19 @@ static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const
|
||||
return 0;
|
||||
}
|
||||
// on DGX Spark the kernel needs its L2 prefetch, which is only in the DGX Spark device code
|
||||
if (cc == GGML_CUDA_CC_DGX_SPARK && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (cc == GGML_CUDA_CC_GB10 && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_GB10) {
|
||||
return 0;
|
||||
}
|
||||
// with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there
|
||||
if (mmq_few_unit_size(type) == 0 || ne00 < 8*QK_K) {
|
||||
// with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there (MUL_MAT_ID splits the rows instead)
|
||||
if (mmq_few_unit_size(type) == 0 || (!has_ids && ne00 < 8*QK_K)) {
|
||||
return 0;
|
||||
}
|
||||
if (cc == GGML_CUDA_CC_DGX_SPARK) { // tuned on DGX Spark GB10
|
||||
if (cc == GGML_CUDA_CC_GB10) { // tuned on DGX Spark GB10, also for MUL_MAT_ID
|
||||
return type == GGML_TYPE_IQ4_XS || type == GGML_TYPE_IQ4_NL ? 8 : 2;
|
||||
}
|
||||
if (has_ids) { // after MMVQ, which is faster up to its max. batch size
|
||||
return get_mmvq_mmid_max_batch(type, cc) + 1;
|
||||
}
|
||||
switch (type) { // all GPUs except DGX Spark, tuned on RTX 5090
|
||||
case GGML_TYPE_Q3_K: return 3;
|
||||
case GGML_TYPE_Q5_K: return 3;
|
||||
@@ -844,14 +906,19 @@ static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const
|
||||
}
|
||||
}
|
||||
|
||||
bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const int cc) {
|
||||
const int min_ncols = ggml_cuda_mmq_few_min_ncols(src0->type, cc, src0->ne[0]);
|
||||
if (min_ncols == 0 || src1->ne[1] < min_ncols || src1->ne[1] > MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS) {
|
||||
bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, const int cc) {
|
||||
const int64_t ncols = ids ? ids->ne[1] : src1->ne[1];
|
||||
const int min_ncols = ggml_cuda_mmq_few_min_ncols(src0->type, cc, src0->ne[0], ids != nullptr);
|
||||
if (min_ncols == 0 || ncols < min_ncols || ncols > MMQ_FEW_MAX_NCG*MMQ_FEW_NCOLS) {
|
||||
return false;
|
||||
}
|
||||
// MUL_MAT_ID: the pairs are built in shared memory
|
||||
if (ids && (2*src0->ne[2] + 1)*sizeof(int) > ggml_cuda_info().devices[ggml_cuda_get_device()].smpb) {
|
||||
return false;
|
||||
}
|
||||
// the row must hold a whole number of ring stages of the launched kernel
|
||||
const int ncg = mmq_few_ncg(src1->ne[1]);
|
||||
const int nmt = mmq_few_nmt(src0->type, ncg, src0->ne[1]);
|
||||
const int ncg = ids ? 1 : mmq_few_ncg(ncols);
|
||||
const int nmt = ids ? 1 : mmq_few_nmt(src0->type, ncg, src0->ne[1]);
|
||||
const int nunits_stage = mmq_few_nunits_stage(src0->type, ncg, nmt);
|
||||
if (src0->ne[0] % (nunits_stage*QK_K) != 0) {
|
||||
return false;
|
||||
@@ -863,13 +930,13 @@ bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor *
|
||||
|
||||
#else // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
||||
GGML_UNUSED_VARS(ctx, src0, src1, dst);
|
||||
void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst) {
|
||||
GGML_UNUSED_VARS(ctx, src0, src1, ids, dst);
|
||||
GGML_ABORT("few-row MMQ is not supported on HIP or MUSA");
|
||||
}
|
||||
|
||||
bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const int cc) {
|
||||
GGML_UNUSED_VARS(src0, src1, cc);
|
||||
bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, const int cc) {
|
||||
GGML_UNUSED_VARS(src0, src1, ids, cc);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
#include "common.cuh"
|
||||
|
||||
bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, int cc);
|
||||
// ids != nullptr: MUL_MAT_ID
|
||||
bool ggml_cuda_should_use_mmq_few(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, int cc);
|
||||
|
||||
void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst);
|
||||
void ggml_cuda_mul_mat_q_few(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * ids, ggml_tensor * dst);
|
||||
|
||||
@@ -460,7 +460,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
|
||||
#endif //GGML_CUDA_FORCE_MMQ
|
||||
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc)) {
|
||||
return !fp16_mma_hardware_available(cc) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;
|
||||
return !fp16_mma_hardware_available(cc) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE || n_experts > 0;
|
||||
}
|
||||
|
||||
if (amd_mfma_available(cc)) {
|
||||
|
||||
+54
-13
@@ -1017,8 +1017,9 @@ static __device__ __forceinline__ void mul_mat_q_process_tile(
|
||||
|
||||
|
||||
// The mul_mat_q kernel implements "stream-k" work partitioning as described in https://arxiv.org/abs/2301.03598
|
||||
// force_tiling lets the host use the native tiled path for configs that normally use Stream-K.
|
||||
|
||||
template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion>
|
||||
template <ggml_type type, int J, bool fallback, ggml_prec prec_src1, bool has_fusion, bool force_tiling = false>
|
||||
__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1), ggml_cuda_mmq_get_occupancy(type, J, fallback, prec_src1))
|
||||
static __global__ void mul_mat_q(
|
||||
const char * __restrict__ x, const char * __restrict__ x_gate, const int * __restrict__ y, const int32_t * __restrict__ ids_dst,
|
||||
@@ -1072,7 +1073,7 @@ static __global__ void mul_mat_q(
|
||||
__syncthreads();
|
||||
|
||||
// The fused kernel needs the whole K range of a tile in one block, so it always uses tiling.
|
||||
if constexpr (has_fusion || !ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) {
|
||||
if constexpr (has_fusion || force_tiling || !ggml_cuda_mmq_get_stream_k(type, J, fallback, prec_src1)) {
|
||||
// The fused kernel writes the GLU of I/2 up rows and I/2 gate rows per tile.
|
||||
constexpr int I_dst = has_fusion ? I/2 : I;
|
||||
|
||||
@@ -1093,7 +1094,7 @@ static __global__ void mul_mat_q(
|
||||
int col_high = min(col_low + J, ncols_dst);
|
||||
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
||||
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
||||
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
||||
int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
|
||||
|
||||
if (ids_dst) {
|
||||
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
|
||||
@@ -1175,7 +1176,7 @@ static __global__ void mul_mat_q(
|
||||
int col_high = min(col_low + J, ncols_dst);
|
||||
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
||||
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
||||
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
||||
int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
|
||||
|
||||
if (ids_dst) {
|
||||
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
|
||||
@@ -1256,7 +1257,7 @@ static __global__ void mul_mat_q(
|
||||
int col_high = min(col_low + J, ncols_dst);
|
||||
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
||||
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
||||
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
||||
int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
|
||||
|
||||
if (ids_dst) {
|
||||
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
|
||||
@@ -1307,14 +1308,25 @@ static __global__ void mul_mat_q(
|
||||
}
|
||||
|
||||
template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
|
||||
__launch_bounds__(ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1)/2, 1)
|
||||
static constexpr __device__ int mul_mat_q_stream_k_fixup_nthreads() {
|
||||
#ifdef GCN
|
||||
// With NVIDIA warp32, the half-sized fixup launch has nthreads/64 warps.
|
||||
// GCN wave64 needs the full MMQ thread count to keep the same nthreads/64 wavefronts.
|
||||
return ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1);
|
||||
#else
|
||||
return ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / 2;
|
||||
#endif // GCN
|
||||
}
|
||||
|
||||
template <ggml_type type, int J, bool fallback, ggml_prec prec_src1>
|
||||
__launch_bounds__((mul_mat_q_stream_k_fixup_nthreads<type, J, fallback, prec_src1>()), 1)
|
||||
static __global__ void mul_mat_q_stream_k_fixup(
|
||||
const int32_t * __restrict__ ids_dst, const ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec, float * __restrict__ dst,
|
||||
float * __restrict__ tmp_last_tile, const uint3 blocks_per_ne00, const int nrows_x, const int ncols_dst,
|
||||
const int stride_col_dst, const uint3 nchannels_y, const int stride_channel_dst, const uint3 nsamples_y,
|
||||
const int stride_sample_dst, const uint3 ntj) {
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
constexpr int nwarps = (ggml_cuda_mmq_get_nthreads(type, J, fallback, prec_src1) / 2) / warp_size;
|
||||
constexpr int nwarps = mul_mat_q_stream_k_fixup_nthreads<type, J, fallback, prec_src1>() / warp_size;
|
||||
constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, prec_src1);
|
||||
constexpr int qk = ggml_cuda_type_traits<type>::qk;
|
||||
constexpr int ITER_K = ggml_cuda_mmq_get_K_vram(type, J, fallback, prec_src1);
|
||||
@@ -1391,7 +1403,7 @@ static __global__ void mul_mat_q_stream_k_fixup(
|
||||
const int it = tmp2.x;
|
||||
|
||||
if (!ids_dst) {
|
||||
const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst + it*I;
|
||||
const int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst + it*I;
|
||||
dst += offset_dst;
|
||||
|
||||
const int i_max = nrows_x - it*I - 1;
|
||||
@@ -1520,16 +1532,42 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
|
||||
return;
|
||||
}
|
||||
|
||||
// For the stream-k kernel it is possible to run it with tiling by setting the number of CUDA blocks equal to the number of tiles.
|
||||
// This is worthwhile if the efficiency of tiling is high and skipping the fixup kernel is more important.
|
||||
// Stream-K adds scheduling and fixup overhead when regular tiling already provides enough parallel work.
|
||||
const int ntiles_dst = args.ntj * nti * ntzw;
|
||||
const int tiles_nwaves = (ntiles_dst + nsm - 1) / nsm;
|
||||
const int tiles_efficiency_percent = 100 * ntiles_dst / (nsm*tiles_nwaves);
|
||||
const int stream_k_blocks = GGML_CUDA_CC_IS_GCN(cc) ? nsm * config.occupancy : nsm;
|
||||
|
||||
const int k_chunks = (args.ncols_x + config.K_vram - 1) / config.K_vram;
|
||||
// Use tiling when Stream-K is unlikely to improve performance:
|
||||
// With at least 2 full waves of tiles, tiling already saturates the device and Stream-K only adds overhead.
|
||||
// With few K chunks per tile, the extra parallelism of Stream-K is not needed.
|
||||
const bool force_tiling = GGML_CUDA_CC_IS_GCN(cc) &&
|
||||
(tiles_efficiency_percent >= 90 || ntiles_dst >= 2*stream_k_blocks ||
|
||||
k_chunks <= config.occupancy * tiles_nwaves);
|
||||
|
||||
if (force_tiling) {
|
||||
GGML_ASSERT(nti <= 65535);
|
||||
CUDA_SET_SHARED_MEMORY_LIMIT((mul_mat_q<type, J, fallback, prec_src1, has_fusion, true>), nbytes_shared);
|
||||
mul_mat_q<type, J, fallback, prec_src1, has_fusion, true><<<block_nums_xy_tiling, block_dims, nbytes_shared, stream>>>
|
||||
(args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.tile_spec, args.dst, nullptr,
|
||||
args.y_scale, args.fusion.glu_op, args.fusion.glu_limit, (const float *) args.fusion.x_scale, (const float *) args.fusion.gate_scale,
|
||||
blocks_per_ne00_fd, args.nrows_x, args.ncols_dst, args.stride_row_x, args.ncols_y, args.nrows_dst,
|
||||
channel_ratio_fd, nchannels_y_fd, args.stride_channel_x, args.stride_channel_y, args.stride_channel_dst,
|
||||
sample_ratio_fd, nsamples_y_fd, args.stride_sample_x, args.stride_sample_y, args.stride_sample_dst,
|
||||
ntj_fd);
|
||||
return;
|
||||
}
|
||||
|
||||
GGML_ASSERT(ntiles_dst * blocks_per_ne00_fd.z < (1 << 30)); // Assert that variable kbc will not overflow.
|
||||
|
||||
const int nwork = ntiles_dst * ((args.ncols_x + config.K_vram - 1) / config.K_vram); // Maximum number of CUDA blocks that can do useful work.
|
||||
const dim3 block_nums_stream_k(GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : std::min(nsm, nwork), 1, 1);
|
||||
const int nwork = ntiles_dst * k_chunks; // Maximum number of CUDA blocks that can do useful work.
|
||||
int nblocks_stream_k = GGML_CUDA_CC_IS_NVIDIA(cc) && tiles_efficiency_percent >= 90 ? ntiles_dst : std::min(stream_k_blocks, nwork);
|
||||
if (GGML_CUDA_CC_IS_GCN(cc) && ntiles_dst < nsm) {
|
||||
// Use a 512-thread budget per CU when the tiles do not fill the device.
|
||||
nblocks_stream_k = std::min(nwork, nsm * std::max(1, 512 / config.nthreads));
|
||||
}
|
||||
const dim3 block_nums_stream_k(nblocks_stream_k, 1, 1);
|
||||
|
||||
const bool fixup_needed = ntiles_dst % block_nums_stream_k.x != 0;
|
||||
|
||||
@@ -1540,7 +1578,10 @@ static void launch_mul_mat_q(ggml_backend_cuda_context & ctx, const mmq_args & a
|
||||
}
|
||||
|
||||
const dim3 block_nums_fixup(block_nums_stream_k.x, config.I/warp_size, 1);
|
||||
const dim3 block_dims_fixup(block_dims.x, block_dims.y/2, block_dims.z);
|
||||
|
||||
// GCN wave64 already halves the warp count relative to NVIDIA warp32, so halving nwarps again would halve fixup parallelism.
|
||||
const int nwarps_fixup = GGML_CUDA_CC_IS_GCN(cc) ? nwarps : nwarps/2;
|
||||
const dim3 block_dims_fixup(block_dims.x, nwarps_fixup, block_dims.z);
|
||||
|
||||
mul_mat_q<type, J, fallback, prec_src1, has_fusion><<<block_nums_stream_k, block_dims, nbytes_shared, stream>>>
|
||||
(args.x, (const char *) args.fusion.gate, args.y, args.ids_dst, args.tile_spec, args.dst, tmp_fixup.ptr,
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
// only enabled on DGX Spark, where it is a gain on every type below. On the higher-bandwidth parts the kernel
|
||||
// has little exposed latency left to hide and the extra requests cost more than they save.
|
||||
// For perf data, see https://github.com/ggml-org/llama.cpp/pull/26705#issuecomment-5569335031
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
// returns true only for those quants that benefit from prefetch and false otherwise
|
||||
static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) {
|
||||
switch (type) {
|
||||
@@ -110,7 +110,7 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
|
||||
return MMVQ_PARAMETERS_GCN;
|
||||
#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA && __CUDA_ARCH__ < GGML_CUDA_CC_AMPERE
|
||||
return MMVQ_PARAMETERS_TURING;
|
||||
#elif __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#elif __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
return MMVQ_PARAMETERS_GB10;
|
||||
#else
|
||||
return MMVQ_PARAMETERS_GENERIC;
|
||||
@@ -133,7 +133,7 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA && ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_AMPERE) {
|
||||
return MMVQ_PARAMETERS_TURING;
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_GB10) {
|
||||
return MMVQ_PARAMETERS_GB10;
|
||||
}
|
||||
return MMVQ_PARAMETERS_GENERIC;
|
||||
@@ -342,7 +342,7 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
|
||||
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
||||
}
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10) {
|
||||
switch (type) { // tuned on DGX Spark GB10
|
||||
case GGML_TYPE_Q2_K:
|
||||
return ne11 <= 6;
|
||||
@@ -751,7 +751,7 @@ static __global__ void mul_mat_vec_q(
|
||||
// x block quant index when casting the quants to int
|
||||
const int kqs = vdr * (tid % (qi/vdr));
|
||||
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
// start the next iterations' weight loads early
|
||||
if constexpr (mmvq_should_prefetch(type)) {
|
||||
// loop iterations, not blocks; q3_K and q6_K keep the distance in blocks they had with vdr 1
|
||||
|
||||
+58
-14
@@ -2,10 +2,18 @@
|
||||
#include "convert.cuh"
|
||||
#include <cstdint>
|
||||
|
||||
template <int block_size>
|
||||
static __global__ void norm_f32(
|
||||
const float * x, float * dst, const int ncols, const int nchannels, const int nsamples,
|
||||
const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps) {
|
||||
template <int block_size, bool do_multiply = false, bool do_add = false>
|
||||
static __global__ void norm_f32(const float * x,
|
||||
float * dst,
|
||||
const int ncols,
|
||||
const int nchannels,
|
||||
const int nsamples,
|
||||
const int64_t stride_row,
|
||||
const int64_t stride_channel,
|
||||
const int64_t stride_sample,
|
||||
const float eps,
|
||||
const float * mul,
|
||||
const float * add) {
|
||||
const int nrows = gridDim.x;
|
||||
const int row = blockIdx.x;
|
||||
const int tid = threadIdx.x;
|
||||
@@ -36,7 +44,14 @@ static __global__ void norm_f32(
|
||||
const float inv_std = rsqrtf(var + eps);
|
||||
|
||||
for (int col = tid; col < ncols; col += block_size) {
|
||||
dstc[col] = (xc[col] - mean) * inv_std;
|
||||
float v = (xc[col] - mean) * inv_std;
|
||||
if constexpr (do_multiply) {
|
||||
v *= mul[col];
|
||||
}
|
||||
if constexpr (do_add) {
|
||||
v += add[col];
|
||||
}
|
||||
dstc[col] = v;
|
||||
}
|
||||
|
||||
if constexpr (block_size > WARP_SIZE) {
|
||||
@@ -384,16 +399,33 @@ static __global__ void l2_norm_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void norm_f32_cuda(
|
||||
const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples,
|
||||
const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) {
|
||||
template <bool do_multiply = false, bool do_add = false>
|
||||
static void norm_f32_cuda(const float * x,
|
||||
float * dst,
|
||||
const int ncols,
|
||||
const int nrows,
|
||||
const int nchannels,
|
||||
const int nsamples,
|
||||
const int64_t stride_row,
|
||||
const int64_t stride_channel,
|
||||
const int64_t stride_sample,
|
||||
const float eps,
|
||||
cudaStream_t stream,
|
||||
const float * mul = nullptr,
|
||||
const float * add = nullptr) {
|
||||
const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX));
|
||||
if (ncols < 1024) {
|
||||
if (do_multiply && ncols <= 2048) {
|
||||
const dim3 block_dims(256, 1, 1);
|
||||
norm_f32<256, do_multiply, do_add><<<blocks_num, block_dims, 32 * sizeof(float2), stream>>>(
|
||||
x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, add);
|
||||
} else if (!do_multiply && ncols < 1024) {
|
||||
const dim3 block_dims(WARP_SIZE, 1, 1);
|
||||
norm_f32<WARP_SIZE><<<blocks_num, block_dims, 0, stream>>>(x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps);
|
||||
norm_f32<WARP_SIZE, do_multiply, do_add><<<blocks_num, block_dims, 0, stream>>>(
|
||||
x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, add);
|
||||
} else {
|
||||
const dim3 block_dims(1024, 1, 1);
|
||||
norm_f32<1024><<<blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float2): 0, stream>>>(x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps);
|
||||
norm_f32<1024, do_multiply, do_add><<<blocks_num, block_dims, 32 * sizeof(float2), stream>>>(
|
||||
x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, add);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -591,10 +623,10 @@ static void l2_norm_f32_cuda(
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul, ggml_tensor * add) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const float * src0_d = (const float *) src0->data;
|
||||
float * dst_d = (float *) dst->data;
|
||||
float * dst_d = (float *) (add ? add : mul ? mul : dst)->data;
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||
@@ -612,7 +644,19 @@ void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const int64_t s02 = nb02 / ts0;
|
||||
const int64_t s03 = nb03 / ts0;
|
||||
|
||||
norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);
|
||||
if (mul) {
|
||||
const ggml_tensor * weight = mul->src[0] == dst ? mul->src[1] : mul->src[0];
|
||||
const float * weight_d = (const float *) weight->data;
|
||||
if (add) {
|
||||
const ggml_tensor * bias = add->src[0] == mul ? add->src[1] : add->src[0];
|
||||
norm_f32_cuda<true, true>(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream, weight_d,
|
||||
(const float *) bias->data);
|
||||
} else {
|
||||
norm_f32_cuda<true, false>(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream, weight_d);
|
||||
}
|
||||
} else {
|
||||
norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#include "common.cuh"
|
||||
|
||||
void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx,
|
||||
ggml_tensor * dst,
|
||||
ggml_tensor * mul = nullptr,
|
||||
ggml_tensor * add = nullptr);
|
||||
|
||||
void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
|
||||
|
||||
@@ -49,6 +49,36 @@ static __global__ void pool1d_nchw_kernel(
|
||||
o_ptr[cur_ow] = res;
|
||||
}
|
||||
|
||||
// a window covering the whole row gives one output per row: reduce it with a block instead of one thread
|
||||
template <ggml_op_pool op>
|
||||
static __global__ void pool1d_row_kernel(const int iw, const float * src, float * dst) {
|
||||
static_assert(op == GGML_OP_POOL_MAX || op == GGML_OP_POOL_AVG, "unsupported pool op");
|
||||
|
||||
const float * row = src + (int64_t) blockIdx.x*iw;
|
||||
|
||||
float res = op == GGML_OP_POOL_MAX ? -FLT_MAX : 0.0f;
|
||||
for (int i = threadIdx.x; i < iw; i += blockDim.x) {
|
||||
res = op == GGML_OP_POOL_MAX ? max(res, row[i]) : res + row[i];
|
||||
}
|
||||
|
||||
__shared__ float buf[WARP_SIZE];
|
||||
res = block_reduce<op == GGML_OP_POOL_MAX ? block_reduce_method::MAX : block_reduce_method::SUM>(res, buf);
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
dst[blockIdx.x] = op == GGML_OP_POOL_MAX ? res : res/iw;
|
||||
}
|
||||
}
|
||||
|
||||
static void pool1d_row_f32_cuda(const int iw, const int64_t nr, const float * src, float * dst, const enum ggml_op_pool op, cudaStream_t stream) {
|
||||
const int nsm = ggml_cuda_info().devices[ggml_cuda_get_device()].nsm;
|
||||
const dim3 block_dims(nr/nsm < 2 ? 512 : (iw < 1024 ? 32 : 128));
|
||||
if (op == GGML_OP_POOL_MAX) {
|
||||
pool1d_row_kernel<GGML_OP_POOL_MAX><<<nr, block_dims, 0, stream>>>(iw, src, dst);
|
||||
} else {
|
||||
pool1d_row_kernel<GGML_OP_POOL_AVG><<<nr, block_dims, 0, stream>>>(iw, src, dst);
|
||||
}
|
||||
}
|
||||
|
||||
static void pool1d_nchw_kernel_f32_f32_cuda(
|
||||
const int iw, const int ow,
|
||||
const int kw, const int sw, const int pw,
|
||||
@@ -79,6 +109,12 @@ void ggml_cuda_op_pool1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const int64_t OW = dst->ne[0];
|
||||
const int64_t nr = ggml_nrows(src0);
|
||||
|
||||
// short rows: one thread per row is faster
|
||||
if (OW == 1 && p0 == 0 && k0 == IW && IW >= CUDA_POOL1D_ROW_MIN_IW && (op == GGML_OP_POOL_MAX || op == GGML_OP_POOL_AVG)) {
|
||||
pool1d_row_f32_cuda(IW, nr, src0_d, dst_d, op, stream);
|
||||
return;
|
||||
}
|
||||
|
||||
const int parallel_elements = (int)(nr * OW);
|
||||
|
||||
pool1d_nchw_kernel_f32_f32_cuda(IW, OW, k0, s0, p0, parallel_elements, src0_d, dst_d, op, stream);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "common.cuh"
|
||||
|
||||
#define CUDA_POOL1D_BLOCK_SIZE 256
|
||||
#define CUDA_POOL1D_ROW_MIN_IW 256
|
||||
|
||||
void ggml_cuda_op_pool1d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
|
||||
+149
-27
@@ -600,6 +600,40 @@ void quantize_mmq_q8_1_cuda(
|
||||
}
|
||||
}
|
||||
|
||||
// Stores the 4 values xi of a thread, values [iu, iu + 4) of column g in a unit of 256 values, in the column group yg of the unit, see quantize.cuh for the layout.
|
||||
template <mmq_q8_1_ds_layout ds_layout>
|
||||
static __device__ __forceinline__ void quantize_mmq_few_store(const float4 xi, int4 * yg, const int iu, const int g) {
|
||||
// the 32 values of a sub-block are held by 8 threads
|
||||
float amax = fmaxf(fmaxf(fabsf(xi.x), fabsf(xi.y)), fmaxf(fabsf(xi.z), fabsf(xi.w)));
|
||||
amax = warp_reduce_max<8>(amax);
|
||||
|
||||
const float d_inv = 127.0f / amax;
|
||||
char4 q;
|
||||
q.x = roundf(xi.x*d_inv);
|
||||
q.y = roundf(xi.y*d_inv);
|
||||
q.z = roundf(xi.z*d_inv);
|
||||
q.w = roundf(xi.w*d_inv);
|
||||
const float d = 1.0f / d_inv;
|
||||
|
||||
// as in quantize_mmq_q8_1: the sum of the quantized values; the int warp_reduce_sum reduces the whole warp on Ampere+
|
||||
[[maybe_unused]] float sum = 0.0f;
|
||||
if constexpr (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) {
|
||||
sum = d*warp_reduce_sum<8>(float(q.x + q.y + q.z + q.w));
|
||||
}
|
||||
|
||||
const int s = iu / QK8_1; // sub-block
|
||||
const int k = iu % QK8_1; // first value in sub-block
|
||||
((int *) (yg + (s/2)*WARP_SIZE + 4*g + (k % 16)/4))[2*(s % 2) + k/16] = *(const int *) &q;
|
||||
if (k == 0) {
|
||||
int4 * ds = yg + 4*WARP_SIZE + 4*(g/2) + s/2;
|
||||
if constexpr (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {
|
||||
((float *) ds)[2*(s % 2) + g % 2] = d;
|
||||
} else {
|
||||
((half2 *) ds)[2*(s % 2) + g % 2] = make_half2(d, sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Each thread quantizes 4 values of one column, see quantize.cuh for the layout.
|
||||
template <mmq_q8_1_ds_layout ds_layout>
|
||||
static __global__ void quantize_mmq_few_q8_1(
|
||||
@@ -619,37 +653,99 @@ static __global__ void quantize_mmq_few_q8_1(
|
||||
const int64_t i3 = blockIdx.z / ne2;
|
||||
const float4 xi = (int) blockIdx.x < ne1 ? *(const float4 *) (x + i3*s03 + i2*s02 + blockIdx.x*s01 + i0) : make_float4(0.0f, 0.0f, 0.0f, 0.0f);
|
||||
|
||||
// the 32 values of a sub-block are held by 8 threads
|
||||
float amax = fmaxf(fmaxf(fabsf(xi.x), fabsf(xi.y)), fmaxf(fabsf(xi.z), fabsf(xi.w)));
|
||||
amax = warp_reduce_max<8>(amax);
|
||||
|
||||
const float d_inv = 127.0f / amax;
|
||||
char4 q;
|
||||
q.x = roundf(xi.x*d_inv);
|
||||
q.y = roundf(xi.y*d_inv);
|
||||
q.z = roundf(xi.z*d_inv);
|
||||
q.w = roundf(xi.w*d_inv);
|
||||
const float d = 1.0f / d_inv;
|
||||
|
||||
// as in quantize_mmq_q8_1: the sum of the quantized values; the int warp_reduce_sum reduces the whole warp on Ampere+
|
||||
[[maybe_unused]] float sum = 0.0f;
|
||||
if constexpr (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) {
|
||||
sum = d*warp_reduce_sum<8>(float(q.x + q.y + q.z + q.w));
|
||||
}
|
||||
|
||||
const int s = i0 % QK_K / QK8_1; // sub-block
|
||||
const int k = i0 % QK8_1; // first value in sub-block
|
||||
const int g = blockIdx.x % MMQ_FEW_NCOLS; // column in group
|
||||
int4 * yg = vy + ((blockIdx.z*(ne00/QK_K) + i0/QK_K)*(gridDim.x/MMQ_FEW_NCOLS) + blockIdx.x/MMQ_FEW_NCOLS)*MMQ_FEW_Q8_1_INT4;
|
||||
((int *) (yg + (s/2)*WARP_SIZE + 4*g + (k % 16)/4))[2*(s % 2) + k/16] = *(const int *) &q;
|
||||
if (k == 0) {
|
||||
int4 * ds = yg + 4*WARP_SIZE + 4*(g/2) + s/2;
|
||||
if constexpr (ds_layout == MMQ_Q8_1_DS_LAYOUT_D4) {
|
||||
((float *) ds)[2*(s % 2) + g % 2] = d;
|
||||
} else {
|
||||
((half2 *) ds)[2*(s % 2) + g % 2] = make_half2(d, sum);
|
||||
quantize_mmq_few_store<ds_layout>(xi, yg, i0 % QK_K, g);
|
||||
}
|
||||
|
||||
// MUL_MAT_ID: block x is the (token, slot) entry, ids are as in mm_ids_helper. Each block finds the pair of its entry in shared memory, block (0, 0) also writes all pairs.
|
||||
// Entry rank r = number of earlier tokens with the same expert, pair = first pair of the expert + r/MMQ_FEW_NCOLS, column = r % MMQ_FEW_NCOLS.
|
||||
template <mmq_q8_1_ds_layout ds_layout>
|
||||
static __global__ void quantize_mmq_few_id_q8_1(
|
||||
const float * x_ptr, const int32_t * ids, void * vy_ptr, int * pairs, const int64_t ne00, const int64_t s11, const int64_t s12,
|
||||
const int ne11, const int n_experts, const int n_expert_used, const int64_t si1, const int npairs_max) {
|
||||
// per expert: bit t set if token t uses it and the index of its first pair; then the total number of pairs
|
||||
extern __shared__ unsigned int data_mmq_few_id[];
|
||||
unsigned int * tokens = data_mmq_few_id;
|
||||
int * pair0 = (int *) (tokens + n_experts);
|
||||
int & npairs = pair0[n_experts];
|
||||
|
||||
ggml_cuda_pdl_lc();
|
||||
const float * GGML_CUDA_RESTRICT x = x_ptr;
|
||||
int4 * GGML_CUDA_RESTRICT vy = (int4 *) vy_ptr;
|
||||
ggml_cuda_pdl_sync();
|
||||
|
||||
const int nentries = gridDim.x;
|
||||
auto expert_of = [&](const int i) {
|
||||
const int e = ids[(i / n_expert_used)*si1 + i % n_expert_used];
|
||||
return e >= 0 && e < n_experts ? e : -1; // other ids skip the slot
|
||||
};
|
||||
|
||||
for (int e = threadIdx.x; e < n_experts; e += blockDim.x) {
|
||||
tokens[e] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
for (int i = threadIdx.x; i < nentries; i += blockDim.x) {
|
||||
const int e = expert_of(i);
|
||||
if (e >= 0) {
|
||||
atomicOr(&tokens[e], 1u << (i / n_expert_used));
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (threadIdx.x < WARP_SIZE) {
|
||||
const int per_lane = (n_experts + WARP_SIZE - 1) / WARP_SIZE;
|
||||
const int e0 = min(int(threadIdx.x)*per_lane, n_experts);
|
||||
const int e1 = min(e0 + per_lane, n_experts);
|
||||
int n = 0;
|
||||
for (int e = e0; e < e1; ++e) {
|
||||
n += (__popc(tokens[e]) + MMQ_FEW_NCOLS - 1) / MMQ_FEW_NCOLS;
|
||||
}
|
||||
int p = warp_prefix_inclusive_sum<int>(n) - n;
|
||||
for (int e = e0; e < e1; ++e) {
|
||||
pair0[e] = p;
|
||||
p += (__popc(tokens[e]) + MMQ_FEW_NCOLS - 1) / MMQ_FEW_NCOLS;
|
||||
}
|
||||
if (threadIdx.x == WARP_SIZE - 1) {
|
||||
npairs = p;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
auto rank = [&](const int i, const int e) {
|
||||
return __popc(tokens[e] & ((1u << (i / n_expert_used)) - 1u));
|
||||
};
|
||||
|
||||
if (blockIdx.x == 0 && blockIdx.y == 0) {
|
||||
for (int i = threadIdx.x; i < nentries; i += blockDim.x) {
|
||||
const int e = expert_of(i);
|
||||
if (e < 0) {
|
||||
continue;
|
||||
}
|
||||
const int r = rank(i, e);
|
||||
int * pair = pairs + (pair0[e] + r/MMQ_FEW_NCOLS)*MMQ_FEW_PAIR_INTS;
|
||||
pair[2 + r % MMQ_FEW_NCOLS] = i;
|
||||
if (r % MMQ_FEW_NCOLS == 0) {
|
||||
pair[0] = e;
|
||||
pair[1] = min(__popc(tokens[e]) - r, MMQ_FEW_NCOLS);
|
||||
}
|
||||
}
|
||||
for (int p = npairs + threadIdx.x; p < npairs_max; p += blockDim.x) {
|
||||
pairs[p*MMQ_FEW_PAIR_INTS + 1] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const int e = expert_of(blockIdx.x);
|
||||
const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4;
|
||||
if (e < 0 || i0 >= ne00) {
|
||||
return;
|
||||
}
|
||||
const int it = blockIdx.x / n_expert_used;
|
||||
const int is = blockIdx.x % n_expert_used;
|
||||
const float4 xi = *(const float4 *) (x + it*s12 + (is % ne11)*s11 + i0);
|
||||
|
||||
const int r = rank(blockIdx.x, e);
|
||||
int4 * yg = vy + (int64_t(pair0[e] + r/MMQ_FEW_NCOLS)*(ne00/QK_K) + i0/QK_K)*MMQ_FEW_Q8_1_INT4;
|
||||
quantize_mmq_few_store<ds_layout>(xi, yg, i0 % QK_K, r % MMQ_FEW_NCOLS);
|
||||
}
|
||||
|
||||
void quantize_mmq_few_q8_1_cuda(
|
||||
@@ -675,6 +771,32 @@ void quantize_mmq_few_q8_1_cuda(
|
||||
}
|
||||
}
|
||||
|
||||
void quantize_mmq_few_id_q8_1_cuda(
|
||||
const float * x, const int32_t * ids, void * vy, int * pairs, const mmq_q8_1_ds_layout ds_layout, const int64_t ne00, const int64_t s11, const int64_t s12,
|
||||
const int64_t ne11, const int64_t n_experts, const int64_t n_expert_used, const int64_t n_tokens, const int64_t si1, const int64_t npairs_max, cudaStream_t stream) {
|
||||
GGML_ASSERT(ne00 % QK_K == 0);
|
||||
GGML_ASSERT(n_tokens <= 32); // one bit per token
|
||||
|
||||
const int64_t block_num_y = (ne00 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ);
|
||||
const size_t nbytes_shared = (2*n_experts + 1)*sizeof(int);
|
||||
GGML_ASSERT(nbytes_shared <= ggml_cuda_info().devices[ggml_cuda_get_device()].smpb);
|
||||
const dim3 num_blocks(n_tokens*n_expert_used, block_num_y, 1);
|
||||
const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1);
|
||||
const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(num_blocks, block_size, nbytes_shared, stream);
|
||||
switch (ds_layout) {
|
||||
case MMQ_Q8_1_DS_LAYOUT_D4:
|
||||
ggml_cuda_kernel_launch(quantize_mmq_few_id_q8_1<MMQ_Q8_1_DS_LAYOUT_D4>, launch_params,
|
||||
x, ids, vy, pairs, ne00, s11, s12, (int) ne11, (int) n_experts, (int) n_expert_used, si1, (int) npairs_max);
|
||||
break;
|
||||
case MMQ_Q8_1_DS_LAYOUT_DS4:
|
||||
ggml_cuda_kernel_launch(quantize_mmq_few_id_q8_1<MMQ_Q8_1_DS_LAYOUT_DS4>, launch_params,
|
||||
x, ids, vy, pairs, ne00, s11, s12, (int) ne11, (int) n_experts, (int) n_expert_used, si1, (int) npairs_max);
|
||||
break;
|
||||
default:
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
}
|
||||
|
||||
// scatter=true reuses the quant kernel: grid over tokens, ids = inverse map (token slot -> compact row)
|
||||
void quantize_scatter_mmq_q8_1_cuda(
|
||||
const float * x, const int32_t * ids_src1_inv, void * vy, const ggml_type type_src0,
|
||||
|
||||
@@ -32,6 +32,14 @@ void quantize_mmq_few_q8_1_cuda(
|
||||
const float * x, void * vy, mmq_q8_1_ds_layout ds_layout, int64_t ne00, int64_t s01, int64_t s02, int64_t s03,
|
||||
int64_t ne1, int64_t ne2, int64_t ne3, cudaStream_t stream);
|
||||
|
||||
// MUL_MAT_ID: a pair is an expert and one column group of up to MMQ_FEW_NCOLS of its (token, slot) entries. The src1 of pair p is channel p above, its columns past the entries are not written.
|
||||
// A pair is MMQ_FEW_PAIR_INTS ints: expert, number of entries (0 for an unused pair), then the entries token*n_expert_used + slot.
|
||||
static constexpr int MMQ_FEW_PAIR_INTS = 2 + MMQ_FEW_NCOLS;
|
||||
|
||||
void quantize_mmq_few_id_q8_1_cuda(
|
||||
const float * x, const int32_t * ids, void * vy, int * pairs, mmq_q8_1_ds_layout ds_layout, int64_t ne00, int64_t s11, int64_t s12,
|
||||
int64_t ne11, int64_t n_experts, int64_t n_expert_used, int64_t n_tokens, int64_t si1, int64_t npairs_max, cudaStream_t stream);
|
||||
|
||||
void quantize_mmq_fp4_cuda(const float * x,
|
||||
const int32_t * ids,
|
||||
void * vy,
|
||||
|
||||
+50
-13
@@ -92,13 +92,16 @@ static __global__ void rope_norm(const T * x,
|
||||
} else if constexpr (std::is_same_v<half, D>) {
|
||||
half2 v = make_half2(x0, x1);
|
||||
ggml_cuda_memcpy_1<4>(dst + idst, &v);
|
||||
} else if constexpr (std::is_same_v<nv_bfloat16, D>) {
|
||||
const __align__(4) nv_bfloat16 v[2] = { ggml_cuda_cast<nv_bfloat16>(x0), ggml_cuda_cast<nv_bfloat16>(x1) };
|
||||
ggml_cuda_memcpy_1<sizeof(v)>(dst + idst, v);
|
||||
}
|
||||
};
|
||||
if (i0 < n_offs || i0 >= n_offs + n_dims) {
|
||||
if (inplace) {
|
||||
return;
|
||||
}
|
||||
store_coaelsced(x[ix + 0], x[ix + 1]);
|
||||
store_coaelsced(ggml_cuda_cast<float>(x[ix + 0]), ggml_cuda_cast<float>(x[ix + 1]));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -113,8 +116,8 @@ static __global__ void rope_norm(const T * x,
|
||||
|
||||
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta);
|
||||
|
||||
const float x0 = x[ix + 0];
|
||||
const float x1 = x[ix + 1];
|
||||
const float x0 = ggml_cuda_cast<float>(x[ix + 0]);
|
||||
const float x1 = ggml_cuda_cast<float>(x[ix + 1]);
|
||||
|
||||
store_coaelsced(x0 * cos_theta - x1 * sin_theta, x0 * sin_theta + x1 * cos_theta);
|
||||
}
|
||||
@@ -189,8 +192,8 @@ static __global__ void rope_neox(const T * x,
|
||||
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta);
|
||||
|
||||
// idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
|
||||
const float x0 = x[ix + n_offs/2 + 0];
|
||||
const float x1 = x[ix + n_offs/2 + n_dims/2];
|
||||
const float x0 = ggml_cuda_cast<float>(x[ix + n_offs/2 + 0]);
|
||||
const float x1 = ggml_cuda_cast<float>(x[ix + n_offs/2 + n_dims/2]);
|
||||
|
||||
dst[idst + n_offs/2 + 0] = ggml_cuda_cast<D>(x0 * cos_theta - x1 * sin_theta);
|
||||
dst[idst + n_offs/2 + n_dims / 2] = ggml_cuda_cast<D>(x0 * sin_theta + x1 * cos_theta);
|
||||
@@ -293,8 +296,8 @@ static __global__ void rope_multi(const T * x,
|
||||
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta);
|
||||
|
||||
// idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
|
||||
const float x0 = x[ix + n_offs/2 + 0];
|
||||
const float x1 = x[ix + n_offs/2 + n_dims/2];
|
||||
const float x0 = ggml_cuda_cast<float>(x[ix + n_offs/2 + 0]);
|
||||
const float x1 = ggml_cuda_cast<float>(x[ix + n_offs/2 + n_dims/2]);
|
||||
|
||||
dst[idst + n_offs/2 + 0] = ggml_cuda_cast<D>(x0*cos_theta - x1*sin_theta);
|
||||
dst[idst + n_offs/2 + n_dims/2] = ggml_cuda_cast<D>(x0*sin_theta + x1*cos_theta);
|
||||
@@ -357,11 +360,11 @@ static __global__ void rope_vision(const T * x,
|
||||
|
||||
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta);
|
||||
|
||||
const float x0 = x[ix + 0];
|
||||
const float x1 = x[ix + n_dims];
|
||||
const float x0 = ggml_cuda_cast<float>(x[ix + 0]);
|
||||
const float x1 = ggml_cuda_cast<float>(x[ix + n_dims]);
|
||||
|
||||
dst[idst + 0] = x0*cos_theta - x1*sin_theta;
|
||||
dst[idst + n_dims] = x0*sin_theta + x1*cos_theta;
|
||||
dst[idst + 0] = ggml_cuda_cast<T>(x0*cos_theta - x1*sin_theta);
|
||||
dst[idst + n_dims] = ggml_cuda_cast<T>(x0*sin_theta + x1*cos_theta);
|
||||
}
|
||||
|
||||
template <bool forward, typename T, typename D>
|
||||
@@ -569,8 +572,8 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
|
||||
}
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
|
||||
GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);
|
||||
GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_BF16);
|
||||
// When not fused, src0 and dst types must match
|
||||
// When fused (ROPE+VIEW+SET_ROWS), src0 may be F32 and dst may be F16
|
||||
GGML_ASSERT(src0->type == dst->type || (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16));
|
||||
@@ -655,6 +658,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
||||
set_rows_stride, inplace, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) {
|
||||
rope_neox_cuda<forward, nv_bfloat16, nv_bfloat16>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
||||
set_rows_stride, inplace, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) {
|
||||
rope_neox_cuda<forward, nv_bfloat16, half>((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
||||
set_rows_stride, inplace, stream);
|
||||
} else {
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
@@ -674,6 +687,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
|
||||
row_indices, set_rows_stride, inplace, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) {
|
||||
rope_multi_cuda<forward, nv_bfloat16, nv_bfloat16>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
|
||||
row_indices, set_rows_stride, inplace, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) {
|
||||
rope_multi_cuda<forward, nv_bfloat16, half>((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
|
||||
row_indices, set_rows_stride, inplace, stream);
|
||||
} else {
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
@@ -686,6 +709,10 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
|
||||
rope_vision_cuda<forward>((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,
|
||||
s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,
|
||||
corr_dims, freq_factors, sections, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16) {
|
||||
rope_vision_cuda<forward>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,
|
||||
s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,
|
||||
corr_dims, freq_factors, sections, stream);
|
||||
} else {
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
@@ -705,6 +732,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
||||
set_rows_stride, inplace, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) {
|
||||
rope_norm_cuda<forward, nv_bfloat16, nv_bfloat16>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
||||
set_rows_stride, inplace, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) {
|
||||
rope_norm_cuda<forward, nv_bfloat16, half>((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
||||
set_rows_stride, inplace, stream);
|
||||
} else {
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
typedef void (*set_rows_kernel_t)(const char * src, char * dst);
|
||||
|
||||
// Generic quantized set_rows kernel template
|
||||
template <typename idx_t, typename block_type, int qk, void (*quantize_func)(const float *, block_type *)>
|
||||
static __global__ void k_set_rows_quant(const float * __restrict__ src0,
|
||||
template <typename src_t, typename idx_t, typename block_type, int qk, void (*quantize_func)(const float *, block_type *)>
|
||||
static __global__ void k_set_rows_quant(const src_t * __restrict__ src0,
|
||||
const idx_t * __restrict__ src1,
|
||||
block_type * __restrict__ dst,
|
||||
const int64_t ne_total,
|
||||
@@ -56,13 +56,23 @@ static __global__ void k_set_rows_quant(const float * __restrict__ src0,
|
||||
ggml_cuda_pdl_sync();
|
||||
const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12);
|
||||
|
||||
const float * src0_row = src0 + i01*s01 + i02*s02 + i03*s03;
|
||||
const src_t * src0_row = src0 + i01*s01 + i02*s02 + i03*s03;
|
||||
block_type * dst_row_ptr = dst + (dst_row*s1 + i02*s2 + i03*s3) / sizeof(block_type);
|
||||
|
||||
const float * src_block = src0_row + i00;
|
||||
const src_t * src_block = src0_row + i00;
|
||||
block_type * dst_block = dst_row_ptr + i00 / qk;
|
||||
|
||||
quantize_func(src_block, dst_block);
|
||||
if constexpr (std::is_same_v<src_t, float>) {
|
||||
quantize_func(src_block, dst_block);
|
||||
} else {
|
||||
// the quantize functions take F32 input: widen the block first
|
||||
float src_block_f32[qk];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < qk; ++j) {
|
||||
src_block_f32[j] = ggml_cuda_cast<float>(src_block[j]);
|
||||
}
|
||||
quantize_func(src_block_f32, dst_block);
|
||||
}
|
||||
|
||||
GGML_UNUSED(ne10);
|
||||
GGML_UNUSED(ne11);
|
||||
@@ -71,9 +81,9 @@ static __global__ void k_set_rows_quant(const float * __restrict__ src0,
|
||||
}
|
||||
|
||||
// Template dispatch function for quantized set_rows
|
||||
template<typename idx_t, typename block_type, int qk, void (*quantize_func)(const float*, block_type*)>
|
||||
template<typename src_t, typename idx_t, typename block_type, int qk, void (*quantize_func)(const float*, block_type*)>
|
||||
static void set_rows_cuda_quant(
|
||||
const float * src0_d, const idx_t * src1_d, block_type * dst_d,
|
||||
const src_t * src0_d, const idx_t * src1_d, block_type * dst_d,
|
||||
const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,
|
||||
const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,
|
||||
const size_t nb01, const size_t nb02, const size_t nb03,
|
||||
@@ -87,9 +97,9 @@ static void set_rows_cuda_quant(
|
||||
const dim3 block_size(CUDA_SET_ROWS_BLOCK_SIZE);
|
||||
const dim3 grid_size(num_blocks);
|
||||
|
||||
const int64_t s01 = nb01/sizeof(float);
|
||||
const int64_t s02 = nb02/sizeof(float);
|
||||
const int64_t s03 = nb03/sizeof(float);
|
||||
const int64_t s01 = nb01/sizeof(src_t);
|
||||
const int64_t s02 = nb02/sizeof(src_t);
|
||||
const int64_t s03 = nb03/sizeof(src_t);
|
||||
const int64_t s10 = nb10/sizeof(idx_t);
|
||||
const int64_t s11 = nb11/sizeof(idx_t);
|
||||
const int64_t s12 = nb12/sizeof(idx_t);
|
||||
@@ -104,7 +114,7 @@ static void set_rows_cuda_quant(
|
||||
const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11);
|
||||
const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12);
|
||||
|
||||
k_set_rows_quant<idx_t, block_type, qk, quantize_func><<<grid_size, block_size, 0, stream>>>(
|
||||
k_set_rows_quant<src_t, idx_t, block_type, qk, quantize_func><<<grid_size, block_size, 0, stream>>>(
|
||||
src0_d, src1_d, dst_d, ne_total, ne10, ne11, ne12, ne13, s01, s02, s03, s10, s11, s12, s1, s2, s3, ne00_fd,
|
||||
ne01_fd, ne02_fd, ne11_fd, ne12_fd);
|
||||
}
|
||||
@@ -268,7 +278,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
|
||||
stream
|
||||
);
|
||||
} else if (dst->type == GGML_TYPE_Q4_0) {
|
||||
set_rows_cuda_quant<idx_t, block_q4_0, QK4_0, quantize_f32_q4_0_block>(
|
||||
set_rows_cuda_quant<src_t, idx_t, block_q4_0, QK4_0, quantize_f32_q4_0_block>(
|
||||
src0_d, src1_d, (block_q4_0*)dst->data,
|
||||
ne00, ne01, ne02, ne03,
|
||||
ne10, ne11, ne12, ne13,
|
||||
@@ -278,7 +288,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
|
||||
stream
|
||||
);
|
||||
} else if (dst->type == GGML_TYPE_Q4_1) {
|
||||
set_rows_cuda_quant<idx_t, block_q4_1, QK4_1, quantize_f32_q4_1_block>(
|
||||
set_rows_cuda_quant<src_t, idx_t, block_q4_1, QK4_1, quantize_f32_q4_1_block>(
|
||||
src0_d, src1_d, (block_q4_1*)dst->data,
|
||||
ne00, ne01, ne02, ne03,
|
||||
ne10, ne11, ne12, ne13,
|
||||
@@ -288,7 +298,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
|
||||
stream
|
||||
);
|
||||
} else if (dst->type == GGML_TYPE_Q5_0) {
|
||||
set_rows_cuda_quant<idx_t, block_q5_0, QK5_0, quantize_f32_q5_0_block>(
|
||||
set_rows_cuda_quant<src_t, idx_t, block_q5_0, QK5_0, quantize_f32_q5_0_block>(
|
||||
src0_d, src1_d, (block_q5_0*)dst->data,
|
||||
ne00, ne01, ne02, ne03,
|
||||
ne10, ne11, ne12, ne13,
|
||||
@@ -298,7 +308,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
|
||||
stream
|
||||
);
|
||||
} else if (dst->type == GGML_TYPE_Q5_1) {
|
||||
set_rows_cuda_quant<idx_t, block_q5_1, QK5_1, quantize_f32_q5_1_block>(
|
||||
set_rows_cuda_quant<src_t, idx_t, block_q5_1, QK5_1, quantize_f32_q5_1_block>(
|
||||
src0_d, src1_d, (block_q5_1*)dst->data,
|
||||
ne00, ne01, ne02, ne03,
|
||||
ne10, ne11, ne12, ne13,
|
||||
@@ -308,7 +318,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
|
||||
stream
|
||||
);
|
||||
} else if (dst->type == GGML_TYPE_Q8_0) {
|
||||
set_rows_cuda_quant<idx_t, block_q8_0, QK8_0, quantize_f32_q8_0_block>(
|
||||
set_rows_cuda_quant<src_t, idx_t, block_q8_0, QK8_0, quantize_f32_q8_0_block>(
|
||||
src0_d, src1_d, (block_q8_0*)dst->data,
|
||||
ne00, ne01, ne02, ne03,
|
||||
ne10, ne11, ne12, ne13,
|
||||
@@ -318,7 +328,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
|
||||
stream
|
||||
);
|
||||
} else if (dst->type == GGML_TYPE_IQ4_NL) {
|
||||
set_rows_cuda_quant<idx_t, block_iq4_nl, QK4_NL, quantize_f32_iq4_nl_block>(
|
||||
set_rows_cuda_quant<src_t, idx_t, block_iq4_nl, QK4_NL, quantize_f32_iq4_nl_block>(
|
||||
src0_d, src1_d, (block_iq4_nl*)dst->data,
|
||||
ne00, ne01, ne02, ne03,
|
||||
ne10, ne11, ne12, ne13,
|
||||
@@ -382,12 +392,11 @@ void set_rows_cuda<half, int64_t>(ggml_backend_cuda_context & ctx, const ggml_te
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const ggml_tensor * src1 = dst->src[1];
|
||||
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16));
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16));
|
||||
GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32);
|
||||
|
||||
if (src0->type == GGML_TYPE_F32) {
|
||||
@@ -402,6 +411,12 @@ void ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
} else {
|
||||
set_rows_cuda<half, int32_t>(ctx, src0, src1, dst);
|
||||
}
|
||||
} else if (src0->type == GGML_TYPE_BF16) {
|
||||
if (src1->type == GGML_TYPE_I64) {
|
||||
set_rows_cuda<nv_bfloat16, int64_t>(ctx, src0, src1, dst);
|
||||
} else {
|
||||
set_rows_cuda<nv_bfloat16, int32_t>(ctx, src0, src1, dst);
|
||||
}
|
||||
} else {
|
||||
GGML_ABORT("unsupported type %s", ggml_type_name(src0->type));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#include "sleep.cuh"
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
// %globaltimer is a nanosecond wall clock, unlike clock64() it is unaffected by the SM clock and by frequency scaling
|
||||
static __device__ __forceinline__ uint64_t globaltimer_ns() {
|
||||
uint64_t t;
|
||||
asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
|
||||
return t;
|
||||
}
|
||||
|
||||
// a single thread is enough, the next kernel on the same stream cannot start before this one retires
|
||||
static __global__ void sleep_ns(const uint64_t ns) {
|
||||
const uint64_t t0 = globaltimer_ns();
|
||||
|
||||
while (globaltimer_ns() - t0 < ns) {}
|
||||
}
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
sleep_ns<<<1, 1, 0, ctx.stream()>>>(1000*(uint64_t) ggml_get_op_params_i32(dst, 0));
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
GGML_UNUSED(ctx);
|
||||
GGML_UNUSED(dst);
|
||||
GGML_ABORT("GGML_OP_SLEEP requires the %%globaltimer register, which is only available on CUDA");
|
||||
}
|
||||
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
@@ -0,0 +1,3 @@
|
||||
#include "common.cuh"
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
@@ -148,13 +148,14 @@ static void ssm_conv_f32_cuda(const float * src0, const float * src1, const floa
|
||||
};
|
||||
|
||||
switch (nc) {
|
||||
case 2: launch_kernel(std::integral_constant<int, 2 >{}); break;
|
||||
case 3: launch_kernel(std::integral_constant<int, 3 >{}); break;
|
||||
case 4: launch_kernel(std::integral_constant<int, 4 >{}); break;
|
||||
case 5: launch_kernel(std::integral_constant<int, 5 >{}); break;
|
||||
case 7: launch_kernel(std::integral_constant<int, 7 >{}); break;
|
||||
case 9: launch_kernel(std::integral_constant<int, 9 >{}); break;
|
||||
case 15: launch_kernel(std::integral_constant<int, 15>{}); break;
|
||||
default: GGML_ABORT("Only support kernel sizes 3, 4, 5, 7, 9, 15 right now.");
|
||||
default: GGML_ABORT("Only support kernel sizes 2, 3, 4, 5, 7, 9, 15 right now.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -184,13 +184,6 @@ static __global__ void top_k_radix_gather(
|
||||
constexpr int NBINS = 1 << RADIX_BITS;
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
constexpr int nwarps = BLOCK_SIZE / warp_size;
|
||||
const auto popc = [](const ggml_cuda_thread_mask_t mask) {
|
||||
if constexpr (warp_size == 64) {
|
||||
return __popcll(mask);
|
||||
} else {
|
||||
return __popc(mask);
|
||||
}
|
||||
};
|
||||
__shared__ int warp_ties[nwarps];
|
||||
|
||||
const int row = blockIdx.x / blocks_per_row;
|
||||
@@ -247,11 +240,11 @@ static __global__ void top_k_radix_gather(
|
||||
const bool is_tie = col < warp_end && top_k_float_to_ordered(row_src[col]) == threshold;
|
||||
|
||||
const ggml_cuda_thread_mask_t ties = ggml_cuda_ballot(is_tie);
|
||||
const int tie_pos = pos + popc(ties & (((ggml_cuda_thread_mask_t) 1 << lane) - 1));
|
||||
const int tie_pos = pos + ggml_cuda_popc(ties & (((ggml_cuda_thread_mask_t) 1 << lane) - 1));
|
||||
if (is_tie && tie_pos < pos_end) {
|
||||
row_dst[k - rank + tie_pos] = col;
|
||||
}
|
||||
pos += popc(ties);
|
||||
pos += ggml_cuda_popc(ties);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -580,13 +580,15 @@ static inline float ggml_f8_e4m3_to_fp32(uint8_t x) {
|
||||
return NAN;
|
||||
}
|
||||
|
||||
const int exp = (ax >> 3) & 0xF;
|
||||
const int man = ax & 0x7;
|
||||
const uint32_t exp = (ax >> 3) & 0xF;
|
||||
const uint32_t man = ax & 0x7;
|
||||
// OCP FP8 rev. 1.1, section 5.1: subnormal magnitude = M * 2^-9; normal magnitude = (1 + M / 8) * 2^(E - 7).
|
||||
// a normal value maps to fp32 exponent E + 120 and the top 3 mantissa bits
|
||||
const float value = exp == 0
|
||||
? ldexpf((float) man, -9)
|
||||
: ldexpf(1.0f + (float) man / 8.0f, exp - 7);
|
||||
return x & 0x80 ? -value : value;
|
||||
? (float) man * (1.0f / 512.0f)
|
||||
: fp32_from_bits(((exp + 120) << 23) | (man << 20));
|
||||
// set the sign bit without a branch, the sign of real data is random
|
||||
return fp32_from_bits(fp32_to_bits(value) | ((uint32_t) (x & 0x80) << 24));
|
||||
}
|
||||
|
||||
static inline int ggml_round_to_nearest_even(float x) {
|
||||
|
||||
@@ -2551,6 +2551,26 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_
|
||||
char base[256];
|
||||
char name[256];
|
||||
|
||||
const bool has_tensor = ggml_metal_device_get_props(ggml_metal_library_get_device(lib))->has_tensor;
|
||||
|
||||
if (has_tensor) {
|
||||
// nr0 output positions per threadgroup, nr1 output channels, same tiles as kernel_mul_mm
|
||||
snprintf(base, 256, "kernel_conv_2d_mm_tensor_%s_%s", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type));
|
||||
snprintf(name, 256, "%s", base);
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
|
||||
}
|
||||
|
||||
res.nr0 = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y;
|
||||
res.nr1 = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X;
|
||||
res.nsg = N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y;
|
||||
res.smem = res.nr0 * N_MM_NK_TOTAL * sizeof(ggml_fp16_t);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
const int nr0 = op->src[0]->ne[3] <= 32 ? 32 : 64;
|
||||
const int nr1 = 2048/nr0;
|
||||
|
||||
@@ -2564,13 +2584,27 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_
|
||||
|
||||
res.nr0 = nr0;
|
||||
res.nr1 = nr1;
|
||||
res.nsg = 4;
|
||||
res.nsg = N_SG_CONV_2D;
|
||||
|
||||
res.smem = nr0*nr1*sizeof(float);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm_reduce(ggml_metal_library_t lib, const ggml_tensor * op) {
|
||||
assert(op->op == GGML_OP_CONV_2D);
|
||||
GGML_UNUSED(op);
|
||||
|
||||
const char * name = "kernel_conv_2d_mm_reduce";
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw(ggml_metal_library_t lib, const ggml_tensor * op, bool tiled) {
|
||||
assert(op->op == GGML_OP_CONV_2D_DW);
|
||||
|
||||
|
||||
@@ -170,6 +170,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake (ggml_metal_library_t lib, enum ggml_type type);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm_reduce (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tiled);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
|
||||
@@ -1713,7 +1713,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
||||
return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32 && (ggml_is_contiguous_rows(op->src[0]));
|
||||
case GGML_OP_ROPE:
|
||||
case GGML_OP_ROPE_BACK:
|
||||
return ggml_rope_get_truncate(op);
|
||||
return ggml_rope_get_truncate(op) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
|
||||
case GGML_OP_IM2COL:
|
||||
return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32);
|
||||
case GGML_OP_CONV_2D:
|
||||
@@ -2019,7 +2019,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
||||
};
|
||||
}
|
||||
case GGML_OP_DIAG:
|
||||
return true;
|
||||
return op->src[0]->type == GGML_TYPE_F32;
|
||||
case GGML_OP_OPT_STEP_ADAMW:
|
||||
case GGML_OP_OPT_STEP_SGD:
|
||||
return has_simdgroup_reduction;
|
||||
@@ -2375,7 +2375,7 @@ void ggml_metal_buffer_memset_tensor(ggml_metal_buffer_t buf, struct ggml_tensor
|
||||
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
||||
|
||||
[encoder fillBuffer:bid_dst.metal
|
||||
range:NSMakeRange(bid_dst.offs, bid_dst.offs + size)
|
||||
range:NSMakeRange(bid_dst.offs, size)
|
||||
value:value];
|
||||
|
||||
[encoder endEncoding];
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
#define N_ARGMAX_NPART_MAX 256
|
||||
#define N_ARGMAX_CHUNK 4096
|
||||
|
||||
#define N_SG_CONV_2D 4
|
||||
|
||||
// kernel parameters for mat-vec threadgroups
|
||||
//
|
||||
// N_R0: number of src0 rows to process per simdgroup
|
||||
@@ -752,8 +754,14 @@ typedef struct {
|
||||
int32_t p1;
|
||||
int32_t d0;
|
||||
int32_t d1;
|
||||
int32_t nk; // reduction length per split
|
||||
} ggml_metal_kargs_conv_2d;
|
||||
|
||||
typedef struct {
|
||||
int32_t ne; // output elements
|
||||
int32_t nsplit;
|
||||
} ggml_metal_kargs_conv_2d_reduce;
|
||||
|
||||
typedef struct {
|
||||
uint64_t nb00; // kernel strides
|
||||
uint64_t nb01;
|
||||
|
||||
@@ -4959,6 +4959,33 @@ int ggml_metal_op_im2col(ggml_metal_op_t ctx, int idx) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// number of reduction splits of the conv_2d matmul kernels
|
||||
// A small output with a long reduction leaves the GPU idle, so the reduction is split until there are about 1M partial sums.
|
||||
static int ggml_metal_op_conv_2d_nsplit(const ggml_tensor * op, int32_t * nk) {
|
||||
const int64_t K = op->src[0]->ne[0]*op->src[0]->ne[1]*op->src[0]->ne[2];
|
||||
const int64_t nsteps = (K + 31)/32;
|
||||
const int64_t ne = ggml_nelements(op);
|
||||
|
||||
// at least 3 reduction steps of 32 per split, at most 32 splits
|
||||
const int64_t nsplit = std::max<int64_t>(1, std::min<int64_t>({ (1024*1024 + ne - 1)/ne, nsteps/3, 32 }));
|
||||
const int64_t steps = (nsteps + nsplit - 1)/nsplit;
|
||||
|
||||
if (nk) {
|
||||
*nk = (int32_t) (steps*32);
|
||||
}
|
||||
|
||||
return (int) ((nsteps + steps - 1)/steps);
|
||||
}
|
||||
|
||||
size_t ggml_metal_op_conv_2d_extra_tmp(const ggml_tensor * op) {
|
||||
assert(op->op == GGML_OP_CONV_2D);
|
||||
|
||||
const int nsplit = ggml_metal_op_conv_2d_nsplit(op, nullptr);
|
||||
|
||||
// partial sums of the reduction splits
|
||||
return nsplit > 1 ? nsplit*ggml_nelements(op)*sizeof(float) : 0;
|
||||
}
|
||||
|
||||
int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_tensor * op = ctx->node(idx);
|
||||
|
||||
@@ -5012,11 +5039,17 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
/*.p1 =*/ p1,
|
||||
/*.d0 =*/ d0,
|
||||
/*.d1 =*/ d1,
|
||||
/*.nk =*/ 0,
|
||||
};
|
||||
|
||||
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev);
|
||||
|
||||
if (props_dev->has_simdgroup_mm) {
|
||||
// the matmul kernels walk the patches with 32-bit byte offsets inside an image
|
||||
// larger images, possibly a strided view, take the generic kernel
|
||||
const uint64_t max_off = (ne12 - 1)*nb12 + (ne11 - 1)*nb11 + (ne10 - 1)*nb10 + sizeof(float);
|
||||
const bool fits_32 = max_off <= UINT32_MAX;
|
||||
|
||||
if (props_dev->has_simdgroup_mm && fits_32) {
|
||||
auto pipeline = ggml_metal_library_get_pipeline_conv_2d_mm(lib, op);
|
||||
|
||||
const int nr0 = pipeline.nr0;
|
||||
@@ -5026,15 +5059,48 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
const int64_t NP = ne3*ne1*ne0;
|
||||
const int64_t OC = ne03;
|
||||
|
||||
const int nsplit = ggml_metal_op_conv_2d_nsplit(op, &args.nk);
|
||||
|
||||
ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);
|
||||
ggml_metal_buffer_id bid_tmp = bid_dst;
|
||||
bid_tmp.offs += ggml_nbytes(op);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline);
|
||||
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
|
||||
ggml_metal_encoder_set_buffer (enc, nsplit > 1 ? bid_tmp : bid_dst, 3);
|
||||
|
||||
ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, 1, 32, nsg, 1);
|
||||
if (props_dev->has_tensor) {
|
||||
// per image: nr0 positions times nr1 channels per threadgroup
|
||||
const int64_t OHW = ne1*ne0;
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (OC + nr1 - 1)/nr1, (OHW + nr0 - 1)/nr0, ne3*nsplit, 32, nsg, 1);
|
||||
} else {
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, nsplit, 32, nsg, 1);
|
||||
}
|
||||
|
||||
if (nsplit > 1) {
|
||||
GGML_ASSERT(ggml_is_contiguous(op));
|
||||
|
||||
// sync the 2 kernels
|
||||
ggml_metal_op_concurrency_reset(ctx);
|
||||
|
||||
ggml_metal_kargs_conv_2d_reduce args0 = {
|
||||
/*.ne =*/ (int32_t) ggml_nelements(op),
|
||||
/*.nsplit =*/ nsplit,
|
||||
};
|
||||
|
||||
auto pipeline0 = ggml_metal_library_get_pipeline_conv_2d_mm_reduce(lib, op);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline0);
|
||||
ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_tmp, 1);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_dst, 2);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (args0.ne + 255)/256, 1, 1, 256, 1, 1);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -49,6 +49,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_rows(const struct ggml_tensor * op);
|
||||
|
||||
size_t ggml_metal_op_argmax_extra(const struct ggml_tensor * op);
|
||||
|
||||
size_t ggml_metal_op_conv_2d_extra_tmp(const struct ggml_tensor * op);
|
||||
|
||||
int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_acc (ggml_metal_op_t ctx, int idx);
|
||||
|
||||
@@ -311,6 +311,10 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
|
||||
{
|
||||
res = 2*sizeof(int32_t)*ggml_nelements(tensor->src[0]);
|
||||
} break;
|
||||
case GGML_OP_CONV_2D:
|
||||
{
|
||||
res += ggml_metal_op_conv_2d_extra_tmp(tensor);
|
||||
} break;
|
||||
case GGML_OP_ARGMAX:
|
||||
{
|
||||
res += ggml_metal_op_argmax_extra(tensor);
|
||||
|
||||
@@ -285,7 +285,7 @@ kernel void kernel_conv_2d_mm(
|
||||
ushort tiitg[[thread_index_in_threadgroup]],
|
||||
ushort tiisg[[thread_index_in_simdgroup]],
|
||||
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
|
||||
constexpr short NSG = 4;
|
||||
constexpr short NSG = N_SG_CONV_2D;
|
||||
constexpr short NTH = 32*NSG;
|
||||
constexpr short NK = 32;
|
||||
constexpr short NKB = NK/8;
|
||||
@@ -300,6 +300,10 @@ kernel void kernel_conv_2d_mm(
|
||||
const int K = args.IC*args.KH*args.KW;
|
||||
const int NP = args.N*args.OH*args.OW;
|
||||
|
||||
// this threadgroup reduces [k_begin, k_end), the splits are summed by kernel_conv_2d_mm_reduce
|
||||
const int k_begin = tgpig.z*args.nk;
|
||||
const int k_end = min(K, k_begin + args.nk);
|
||||
|
||||
const int r0 = tgpig.y*NR0;
|
||||
const int r1 = tgpig.x*NR1;
|
||||
|
||||
@@ -308,8 +312,14 @@ kernel void kernel_conv_2d_mm(
|
||||
const short lb_n = 8*sgitg + tiisg%8;
|
||||
const short lb_k = 8*(tiisg/8);
|
||||
|
||||
// the patches are walked with 32-bit byte offsets inside the image, the host checks that an image fits
|
||||
const uint nb10 = args.nb10;
|
||||
const uint nb11 = args.nb11;
|
||||
const uint nb12 = args.nb12;
|
||||
|
||||
int ix0[NPL];
|
||||
int iy0[NPL];
|
||||
uint off0[NPL];
|
||||
bool ip_ok[NPL];
|
||||
|
||||
device const char * srcb[NPL];
|
||||
@@ -324,6 +334,8 @@ kernel void kernel_conv_2d_mm(
|
||||
ix0[t] = iow*args.s0 - args.p0;
|
||||
iy0[t] = ioh*args.s1 - args.p1;
|
||||
|
||||
off0[t] = iy0[t]*nb11 + ix0[t]*nb10;
|
||||
|
||||
ip_ok[t] = ip < NP;
|
||||
|
||||
srcb[t] = src + in*args.nb13;
|
||||
@@ -337,7 +349,7 @@ kernel void kernel_conv_2d_mm(
|
||||
mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
|
||||
}
|
||||
|
||||
for (int k0 = 0; k0 < K; k0 += NK) {
|
||||
for (int k0 = k_begin; k0 < k_end; k0 += NK) {
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
{
|
||||
@@ -349,7 +361,7 @@ kernel void kernel_conv_2d_mm(
|
||||
const int oc = r0 + row;
|
||||
|
||||
half v = 0;
|
||||
if (kk < K && oc < args.OC) {
|
||||
if (kk < k_end && oc < args.OC) {
|
||||
v = *((device const TK *)(weights + oc*args.nb03) + kk);
|
||||
}
|
||||
|
||||
@@ -364,7 +376,11 @@ kernel void kernel_conv_2d_mm(
|
||||
|
||||
int kw = kk % args.KW;
|
||||
int kh = (kk / args.KW) % args.KH;
|
||||
int ic = kk / (args.KW*args.KH);
|
||||
int dx = kw*args.d0;
|
||||
int dy = kh*args.d1;
|
||||
|
||||
// byte offset of the reduction index kk, relative to the patch origin
|
||||
uint offk = (kk / (args.KW*args.KH))*nb12 + dy*nb11 + dx*nb10;
|
||||
|
||||
FOR_UNROLL (short i = 0; i < 8; i++) {
|
||||
const short k = lb_k + i;
|
||||
@@ -372,12 +388,12 @@ kernel void kernel_conv_2d_mm(
|
||||
FOR_UNROLL (short t = 0; t < NPL; t++) {
|
||||
const short n = 32*t + lb_n;
|
||||
|
||||
const int ix = ix0[t] + kw*args.d0;
|
||||
const int iy = iy0[t] + kh*args.d1;
|
||||
const int ix = ix0[t] + dx;
|
||||
const int iy = iy0[t] + dy;
|
||||
|
||||
half v = 0;
|
||||
if (ip_ok[t] && kk < K && ix >= 0 && ix < args.IW && iy >= 0 && iy < args.IH) {
|
||||
v = *(device const float *)(srcb[t] + ic*args.nb12 + iy*args.nb11 + ix*args.nb10);
|
||||
if (ip_ok[t] && kk < k_end && (uint) ix < (uint) args.IW && (uint) iy < (uint) args.IH) {
|
||||
v = *(device const float *)(srcb[t] + (off0[t] + offk));
|
||||
}
|
||||
|
||||
const short ib = NKB*(n/8) + k/8;
|
||||
@@ -386,11 +402,17 @@ kernel void kernel_conv_2d_mm(
|
||||
}
|
||||
|
||||
kk++;
|
||||
dx += args.d0;
|
||||
offk += args.d0*nb10;
|
||||
if (++kw == args.KW) {
|
||||
kw = 0;
|
||||
kw = 0;
|
||||
dx = 0;
|
||||
dy += args.d1;
|
||||
offk += args.d1*nb11 - args.KW*args.d0*nb10;
|
||||
if (++kh == args.KH) {
|
||||
kh = 0;
|
||||
ic++;
|
||||
kh = 0;
|
||||
dy = 0;
|
||||
offk += nb12 - args.KH*args.d1*nb11;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -398,7 +420,7 @@ kernel void kernel_conv_2d_mm(
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
const short nkb = min((K - k0 + 7)/8, (int) NKB);
|
||||
const short nkb = min((k_end - k0 + 7)/8, (int) NKB);
|
||||
|
||||
for (short ik = 0; ik < nkb; ik++) {
|
||||
FOR_UNROLL (short i = 0; i < 4; i++) {
|
||||
@@ -436,7 +458,8 @@ kernel void kernel_conv_2d_mm(
|
||||
const int oh = (p / args.OW) % args.OH;
|
||||
const int n = p / (args.OW*args.OH);
|
||||
|
||||
device char * dstp = dst + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
|
||||
// with several splits, dst is the partial sum buffer, one contiguous output per split
|
||||
device char * dstp = dst + (uint64_t) tgpig.z*args.N*args.nb3 + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
|
||||
|
||||
for (short j = 0; j < NR0*NR1/NTH; j++) {
|
||||
const short row = tiitg/NR1 + (NTH/NR1)*j;
|
||||
@@ -448,6 +471,158 @@ kernel void kernel_conv_2d_mm(
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef GGML_METAL_HAS_TENSOR
|
||||
// conv_2d as a matmul on the tensor cores, like kernel_mul_mm
|
||||
// The patches of NRA output positions are gathered into threadgroup memory, the kernel rows are read from device memory.
|
||||
// One threadgroup computes NRA positions of one image times NRB output channels, over the reduction range of its split.
|
||||
template <typename TK>
|
||||
kernel void kernel_conv_2d_mm_tensor(
|
||||
constant ggml_metal_kargs_conv_2d & args,
|
||||
device const char * weights,
|
||||
device const char * src,
|
||||
device char * dst,
|
||||
threadgroup char * shmem [[threadgroup(0)]],
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
uint3 tgpg[[threadgroups_per_grid]],
|
||||
ushort tiitg[[thread_index_in_threadgroup]]) {
|
||||
constexpr int NRB = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X; // output channels
|
||||
constexpr int NRA = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y; // output positions
|
||||
constexpr int NK = N_MM_NK_TOTAL;
|
||||
constexpr int NTH = N_SIMDWIDTH * N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y;
|
||||
|
||||
// each thread gathers NK/(NTH/NRA) consecutive reduction indices of one position
|
||||
constexpr int KPT = NK/(NTH/NRA);
|
||||
|
||||
const int K = args.IC*args.KH*args.KW;
|
||||
const int OHW = args.OH*args.OW;
|
||||
|
||||
const int nsplit = tgpg.z/args.N;
|
||||
const int img = tgpig.z/nsplit;
|
||||
const int split = tgpig.z%nsplit;
|
||||
|
||||
const int k_begin = split*args.nk;
|
||||
const int k_end = min(K, k_begin + args.nk);
|
||||
|
||||
const int ra = tgpig.y*NRA;
|
||||
const int rb = tgpig.x*NRB;
|
||||
|
||||
threadgroup half * sa = (threadgroup half *) shmem;
|
||||
|
||||
// the position this thread gathers
|
||||
const int row = tiitg%NRA;
|
||||
const int kc = (tiitg/NRA)*KPT;
|
||||
const int p = ra + row;
|
||||
const bool pok = p < OHW;
|
||||
const int ix0 = (p%args.OW)*args.s0 - args.p0;
|
||||
const int iy0 = (p/args.OW)*args.s1 - args.p1;
|
||||
|
||||
device const char * srcb = src + img*args.nb13;
|
||||
|
||||
// the patch is walked with 32-bit byte offsets inside the image, the host checks that an image fits
|
||||
const uint nb10 = args.nb10;
|
||||
const uint nb11 = args.nb11;
|
||||
const uint nb12 = args.nb12;
|
||||
|
||||
const uint off0 = iy0*nb11 + ix0*nb10;
|
||||
|
||||
device TK * ptrB = (device TK *) weights;
|
||||
const int strideB = args.nb03/sizeof(TK);
|
||||
|
||||
mpp::tensor_ops::matmul2d<
|
||||
mpp::tensor_ops::matmul2d_descriptor(
|
||||
NRB, NRA, static_cast<int>(dynamic_extent), false, true, true,
|
||||
mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
|
||||
execution_simdgroups<N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y>> mm;
|
||||
|
||||
auto tB0 = tensor(ptrB, dextents<int32_t, 2>(K, args.OC), array<int, 2>({1, strideB}));
|
||||
auto tA0 = tensor(sa, dextents<int32_t, 2>(NK, NRA));
|
||||
auto cT = mm.get_destination_cooperative_tensor<decltype(tB0), decltype(tA0), float>();
|
||||
|
||||
for (int k0 = k_begin; k0 < k_end; k0 += NK) {
|
||||
// the reduction indices past k_end are beyond the extent given to the matmul, they are not gathered
|
||||
if (k0 + kc < k_end) {
|
||||
// gather KPT patch values of this position, walking the reduction index incrementally
|
||||
int kk = k0 + kc;
|
||||
int kw = kk % args.KW;
|
||||
int kh = (kk / args.KW) % args.KH;
|
||||
int dx = kw*args.d0;
|
||||
int dy = kh*args.d1;
|
||||
|
||||
// byte offset of the reduction index kk, relative to the patch origin
|
||||
uint offk = (kk / (args.KW*args.KH))*nb12 + dy*nb11 + dx*nb10;
|
||||
|
||||
FOR_UNROLL (short i = 0; i < KPT; i++) {
|
||||
const int ix = ix0 + dx;
|
||||
const int iy = iy0 + dy;
|
||||
|
||||
half v = 0;
|
||||
if (pok && kk < k_end && (uint) ix < (uint) args.IW && (uint) iy < (uint) args.IH) {
|
||||
v = *(device const float *)(srcb + (off0 + offk));
|
||||
}
|
||||
sa[row*NK + kc + i] = v;
|
||||
|
||||
kk++;
|
||||
dx += args.d0;
|
||||
offk += args.d0*nb10;
|
||||
if (++kw == args.KW) {
|
||||
kw = 0;
|
||||
dx = 0;
|
||||
dy += args.d1;
|
||||
offk += args.d1*nb11 - args.KW*args.d0*nb10;
|
||||
if (++kh == args.KH) {
|
||||
kh = 0;
|
||||
dy = 0;
|
||||
offk += nb12 - args.KH*args.d1*nb11;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
// clamp the reduction extent so neither operand is read past the end of the split
|
||||
const int kExt = min(NK, k_end - k0);
|
||||
|
||||
auto tAv = tensor(sa, dextents<int32_t, 2>(kExt, NRA), array<int, 2>({1, NK}));
|
||||
auto tBv = tensor(ptrB + k0 + rb*strideB, dextents<int32_t, 2>(kExt, args.OC - rb), array<int, 2>({1, strideB}));
|
||||
|
||||
mm.run(tBv, tAv, cT);
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
}
|
||||
|
||||
// dst holds [OHW, OC] per image, with several splits it is the partial sum buffer, one contiguous output per split
|
||||
const int tileM = min(NRA, OHW - ra);
|
||||
const int tileN = min(NRB, args.OC - rb);
|
||||
|
||||
device float * dstTile = (device float *) (dst + (uint64_t) split*args.N*args.nb3 + img*args.nb3) + (uint64_t) rb*OHW + ra;
|
||||
|
||||
auto tD = tensor(dstTile, dextents<int32_t, 2>(tileM, tileN), array<int, 2>({1, OHW}));
|
||||
cT.store(tD);
|
||||
}
|
||||
|
||||
typedef decltype(kernel_conv_2d_mm_tensor<float>) kernel_conv_2d_mm_tensor_t;
|
||||
|
||||
template [[host_name("kernel_conv_2d_mm_tensor_f32_f32")]] kernel kernel_conv_2d_mm_tensor_t kernel_conv_2d_mm_tensor<float>;
|
||||
template [[host_name("kernel_conv_2d_mm_tensor_f16_f32")]] kernel kernel_conv_2d_mm_tensor_t kernel_conv_2d_mm_tensor<half>;
|
||||
#endif
|
||||
|
||||
kernel void kernel_conv_2d_mm_reduce(
|
||||
constant ggml_metal_kargs_conv_2d_reduce & args,
|
||||
device const float * tmp,
|
||||
device float * dst,
|
||||
uint tpig[[thread_position_in_grid]]) {
|
||||
if ((int) tpig >= args.ne) {
|
||||
return;
|
||||
}
|
||||
|
||||
float sum = 0.0f;
|
||||
for (int s = 0; s < args.nsplit; ++s) {
|
||||
sum += tmp[(uint64_t) s*args.ne + tpig];
|
||||
}
|
||||
dst[tpig] = sum;
|
||||
}
|
||||
|
||||
typedef decltype(kernel_conv_2d_mm<float, 64, 32>) kernel_conv_2d_mm_t;
|
||||
|
||||
template [[host_name("kernel_conv_2d_mm_f32_f32_64x32")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<float, 64, 32>;
|
||||
|
||||
@@ -323,6 +323,9 @@ set(GGML_OPENCL_KERNELS
|
||||
mul_mm_f16_f32_kq_kqv
|
||||
mul_mm_f16_f32_kq_p8
|
||||
mul_mm_q8_f32_kqv
|
||||
mul_mm_q8_kq_p8
|
||||
mul_mm_q8_kq_p8_dk512
|
||||
flash_attn_q8_rows
|
||||
conv2d
|
||||
conv2d_f16_f32
|
||||
conv2_dw
|
||||
|
||||
@@ -781,6 +781,10 @@ struct ggml_backend_opencl_context {
|
||||
ggml_cl_buffer prealloc_fa_pd;
|
||||
ggml_cl_buffer prealloc_fa_bmax;
|
||||
ggml_cl_buffer prealloc_fa_bsum;
|
||||
ggml_cl_buffer prealloc_fa_k8;
|
||||
ggml_cl_buffer prealloc_fa_k8_scale;
|
||||
ggml_cl_buffer prealloc_fa_q8;
|
||||
ggml_cl_buffer prealloc_fa_q8_scale;
|
||||
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
ggml_cl_buffer prealloc_adreno_xmem_const;
|
||||
@@ -899,6 +903,10 @@ struct ggml_backend_opencl_context {
|
||||
cl_kernel kernel_fa_kqv_direct_f16 = nullptr;
|
||||
cl_kernel kernel_fa_kqv_direct_reduce = nullptr;
|
||||
cl_kernel kernel_fa_v_transpose_q8 = nullptr;
|
||||
cl_kernel kernel_fa_q8_rows_f16 = nullptr;
|
||||
cl_kernel kernel_fa_q8_rows_f32 = nullptr;
|
||||
cl_kernel kernel_mul_mm_q8_kq_p8 = nullptr;
|
||||
cl_kernel kernel_mul_mm_q8_kq_p8_dk512 = nullptr;
|
||||
ggml_opencl_fa_kernels fa;
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
ggml_cl_adreno_xmem_attn_state adreno_xmem_attn;
|
||||
@@ -1473,6 +1481,7 @@ struct ggml_backend_opencl_context {
|
||||
cl_kernel kernel_gemv_noshuffle_q5_k_f32_mc3; // multi-column (N=3) verify GEMV (spec/MTP)
|
||||
cl_kernel kernel_gemm_noshuffle_q5_k_f32;
|
||||
cl_kernel kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin;
|
||||
cl_kernel kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin;
|
||||
cl_kernel kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin;
|
||||
cl_kernel kernel_gemv_noshuffle_q5_k_f32_32b_trans;
|
||||
cl_kernel kernel_gemv_noshuffle_q5_0_f32;
|
||||
@@ -1505,7 +1514,18 @@ struct ggml_backend_opencl_context {
|
||||
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
|
||||
~ggml_backend_opencl_context() {
|
||||
for (cl_kernel kernel : {kernel_mul_mm_f16_f32_kq_p8, kernel_mul_mm_q8_f32_kqv, kernel_fa_p8_fixup, kernel_fa_v_transpose_q8, kernel_fa_kqv_direct_f16, kernel_fa_kqv_direct_reduce}) {
|
||||
for (cl_kernel kernel : {
|
||||
kernel_mul_mm_f16_f32_kq_p8,
|
||||
kernel_mul_mm_q8_f32_kqv,
|
||||
kernel_fa_p8_fixup,
|
||||
kernel_fa_v_transpose_q8,
|
||||
kernel_fa_kqv_direct_f16,
|
||||
kernel_fa_kqv_direct_reduce,
|
||||
kernel_fa_q8_rows_f16,
|
||||
kernel_fa_q8_rows_f32,
|
||||
kernel_mul_mm_q8_kq_p8,
|
||||
kernel_mul_mm_q8_kq_p8_dk512
|
||||
}) {
|
||||
if (kernel != nullptr) {
|
||||
CL_CHECK(clReleaseKernel(kernel));
|
||||
}
|
||||
@@ -1647,7 +1667,7 @@ static cl_program build_program_from_source_ex(cl_context ctx, cl_device_id dev,
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts) {
|
||||
static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts, bool fatal = true) {
|
||||
cl_context ctx = backend_ctx->context;
|
||||
cl_device_id dev = backend_ctx->device;
|
||||
|
||||
@@ -1659,7 +1679,7 @@ static cl_program build_program_from_source(ggml_backend_opencl_context * backen
|
||||
return p_cached;
|
||||
}
|
||||
|
||||
cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, /*fatal=*/true);
|
||||
cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, fatal);
|
||||
|
||||
// Best-effort save of the freshly-built binary (no-op if cache disabled).
|
||||
if (p != nullptr) {
|
||||
@@ -3994,6 +4014,52 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
}
|
||||
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
|
||||
#ifdef GGML_OPENCL_EMBED_KERNELS
|
||||
const std::string rows_src {
|
||||
#include "flash_attn_q8_rows.cl.h"
|
||||
};
|
||||
const std::string kq_src {
|
||||
#include "mul_mm_q8_kq_p8.cl.h"
|
||||
};
|
||||
#else
|
||||
const std::string rows_src = read_file("flash_attn_q8_rows.cl");
|
||||
const std::string kq_src = read_file("mul_mm_q8_kq_p8.cl");
|
||||
#endif
|
||||
cl_program prog = build_program_from_source(backend_ctx, rows_src.c_str(), compile_opts, /*fatal=*/false);
|
||||
if (prog != nullptr) {
|
||||
CL_CHECK((backend_ctx->kernel_fa_q8_rows_f16 = clCreateKernel(prog, "kernel_fa_q8_rows_f16", &err), err));
|
||||
CL_CHECK((backend_ctx->kernel_fa_q8_rows_f32 = clCreateKernel(prog, "kernel_fa_q8_rows_f32", &err), err));
|
||||
CL_CHECK(clReleaseProgram(prog));
|
||||
}
|
||||
prog = build_program_from_source(backend_ctx, kq_src.c_str(), compile_opts, /*fatal=*/false);
|
||||
if (prog != nullptr) {
|
||||
CL_CHECK((backend_ctx->kernel_mul_mm_q8_kq_p8 = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8", &err), err));
|
||||
CL_CHECK(clReleaseProgram(prog));
|
||||
}
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
// mul_mm_q8_kq_p8_dk512
|
||||
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
|
||||
#ifdef GGML_OPENCL_EMBED_KERNELS
|
||||
const std::string kernel_src {
|
||||
#include "mul_mm_q8_kq_p8_dk512.cl.h"
|
||||
};
|
||||
#else
|
||||
const std::string kernel_src = read_file("mul_mm_q8_kq_p8_dk512.cl");
|
||||
#endif
|
||||
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts, /*fatal=*/false);
|
||||
if (prog != nullptr) {
|
||||
cl_kernel kernel = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8_dk512", &err);
|
||||
CL_CHECK(clReleaseProgram(prog));
|
||||
if (err == CL_SUCCESS) {
|
||||
backend_ctx->kernel_mul_mm_q8_kq_p8_dk512 = kernel;
|
||||
}
|
||||
}
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
// mul_mm_f16_f32_kq_p8
|
||||
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
|
||||
#ifdef GGML_OPENCL_EMBED_KERNELS
|
||||
@@ -6845,6 +6911,7 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
|
||||
backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans = nullptr;
|
||||
backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin = nullptr;
|
||||
backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin = nullptr;
|
||||
backend_ctx->kernel_gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8_bin = nullptr;
|
||||
{
|
||||
if (backend_ctx->has_vector_subgroup_broadcast) {
|
||||
@@ -6879,6 +6946,17 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q5_k_f32_32b_trans_ila_a7", &bin_size);
|
||||
if (kernel_bin && bin_size > 0) {
|
||||
cl_program bin_prog =
|
||||
build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, "", bin_size);
|
||||
|
||||
CL_CHECK((backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin =
|
||||
clCreateKernel(bin_prog, "kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7", &err), err));
|
||||
CL_CHECK(clReleaseProgram(bin_prog));
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_noshuffle_q5_k_q8_1_dp4a_ila_a8", &bin_size);
|
||||
if (kernel_bin && bin_size > 0) {
|
||||
cl_program bin_prog =
|
||||
@@ -11850,7 +11928,8 @@ inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
|
||||
inline bool use_q5_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
if (!backend_ctx->kernel_gemv_noshuffle_q5_k_f32_32b_trans ||
|
||||
!backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin) {
|
||||
!(backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin ||
|
||||
backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin)) {
|
||||
return false;
|
||||
}
|
||||
return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0);
|
||||
@@ -12269,7 +12348,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
|
||||
case GGML_OP_TRANSPOSE:
|
||||
return true;
|
||||
case GGML_OP_DIAG:
|
||||
return true;
|
||||
return op->src[0]->type == GGML_TYPE_F32;
|
||||
case GGML_OP_ROPE: {
|
||||
const int mode = ((const int32_t *) op->op_params)[2];
|
||||
if (!ggml_rope_get_truncate(op)) {
|
||||
@@ -12291,7 +12370,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
|
||||
}
|
||||
case GGML_OP_SOLVE_TRI:
|
||||
return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]);
|
||||
@@ -22908,12 +22987,58 @@ static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_te
|
||||
}
|
||||
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
|
||||
static void ggml_cl_fa_q8_rows(
|
||||
ggml_backend_t backend, const ggml_tensor * src, const ggml_tensor * dst,
|
||||
cl_mem values, cl_mem scales, int64_t row0, int nrow
|
||||
) {
|
||||
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
|
||||
auto * extra = (ggml_tensor_extra_cl *) (src->view_src ? src->view_src->extra : src->extra);
|
||||
const cl_ulong offset = extra->offset + src->view_offs + row0 * src->nb[1];
|
||||
const cl_ulong nb1 = src->nb[1];
|
||||
const cl_ulong nb2 = src->nb[2];
|
||||
const int dk = src->ne[0];
|
||||
const int nhead = src->ne[2];
|
||||
cl_kernel kernel;
|
||||
switch (src->type) {
|
||||
case GGML_TYPE_F16: kernel = backend_ctx->kernel_fa_q8_rows_f16; break;
|
||||
case GGML_TYPE_F32: kernel = backend_ctx->kernel_fa_q8_rows_f32; break;
|
||||
default: GGML_ABORT("unsupported int8 KQ input type");
|
||||
}
|
||||
cl_uint a = 0;
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &offset));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb2));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &values));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &scales));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dk));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nrow));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nhead));
|
||||
const size_t blocks = (size_t) (dk / 32) * nrow * nhead;
|
||||
size_t gws[3] = {(blocks + 63) / 64 * 64, 1, 1};
|
||||
size_t lws[3] = {64, 1, 1};
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
}
|
||||
|
||||
static void ggml_cl_fa_kq_int8_prepare(ggml_backend_t backend, const ggml_tensor * dst, int64_t n_q_chunk) {
|
||||
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
|
||||
const ggml_tensor * q = dst->src[0];
|
||||
const ggml_tensor * k = dst->src[1];
|
||||
const size_t k_bytes = (size_t) k->ne[0] * k->ne[1] * k->ne[2];
|
||||
const size_t q_bytes = (size_t) q->ne[0] * n_q_chunk * q->ne[2];
|
||||
backend_ctx->prealloc_fa_k8.allocate(backend_ctx->context, k_bytes);
|
||||
backend_ctx->prealloc_fa_k8_scale.allocate(backend_ctx->context, k_bytes / 32 * sizeof(cl_half));
|
||||
backend_ctx->prealloc_fa_q8.allocate(backend_ctx->context, q_bytes);
|
||||
backend_ctx->prealloc_fa_q8_scale.allocate(backend_ctx->context, q_bytes / 32 * sizeof(cl_half));
|
||||
ggml_cl_fa_q8_rows(backend, k, dst, backend_ctx->prealloc_fa_k8.buffer, backend_ctx->prealloc_fa_k8_scale.buffer, 0, k->ne[1]);
|
||||
}
|
||||
|
||||
static void ggml_cl_fa_kq_p8_dispatch(
|
||||
ggml_backend_t backend,
|
||||
const ggml_tensor * q, const ggml_tensor * k,
|
||||
const ggml_tensor * mask, const ggml_tensor * sinks,
|
||||
const ggml_tensor * dst,
|
||||
int64_t q0, int nqc, float scale, float max_bias, int p_pitch
|
||||
int64_t q0, int nqc, float scale, float max_bias, int p_pitch, bool kq_int8
|
||||
) {
|
||||
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
|
||||
auto * extra_k = (ggml_tensor_extra_cl *) (k->view_src ? k->view_src->extra : k->extra);
|
||||
@@ -22926,64 +23051,94 @@ static void ggml_cl_fa_kq_p8_dispatch(
|
||||
const int D_A = k->ne[2];
|
||||
const int D_B = q->ne[2];
|
||||
|
||||
cl_int err;
|
||||
|
||||
cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
|
||||
cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
cl_image_format fmt = {CL_RGBA, CL_FLOAT};
|
||||
cl_image_desc desc = {};
|
||||
desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
|
||||
desc.image_width = region.size / 16;
|
||||
desc.buffer = k_sub;
|
||||
cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
|
||||
region.size = (size_t) K * N * D_B * sizeof(float);
|
||||
cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
const cl_ulong off_mask = extra_m->offset + mask->view_offs + q0 * mask->nb[1];
|
||||
const cl_ulong mask_nb1 = mask->nb[1];
|
||||
const int n_head_log2 = 1u << (unsigned) floorf(log2f((float) D_B));
|
||||
const float m0 = powf(2.0f, -max_bias / n_head_log2);
|
||||
const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
|
||||
|
||||
cl_kernel kernel;
|
||||
cl_uint a = 0;
|
||||
|
||||
cl_kernel kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
|
||||
if (kq_int8) {
|
||||
ggml_cl_fa_q8_rows(backend, q, dst, backend_ctx->prealloc_fa_q8.buffer, backend_ctx->prealloc_fa_q8_scale.buffer, q0, nqc);
|
||||
kernel = K <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8_scale.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8_scale.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
|
||||
size_t gws[3] = {64 * (((size_t) N + 31) / 32) * (D_B / D_A), ((size_t) M + 511) / 512, (size_t) D_A};
|
||||
size_t lws[3] = {64, 1, 1};
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
} else {
|
||||
cl_int err;
|
||||
|
||||
const size_t n_tiles = (N + 31) / 32;
|
||||
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
|
||||
cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
|
||||
cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
|
||||
size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
|
||||
cl_image_format fmt = {CL_RGBA, CL_FLOAT};
|
||||
cl_image_desc desc = {};
|
||||
desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
|
||||
desc.image_width = region.size / 16;
|
||||
desc.buffer = k_sub;
|
||||
cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
CL_CHECK(clReleaseMemObject(k_img));
|
||||
CL_CHECK(clReleaseMemObject(k_sub));
|
||||
CL_CHECK(clReleaseMemObject(q_sub));
|
||||
region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
|
||||
region.size = (size_t) K * N * D_B * sizeof(float);
|
||||
cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
|
||||
|
||||
const size_t n_tiles = (N + 31) / 32;
|
||||
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
|
||||
|
||||
const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
|
||||
size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
|
||||
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
CL_CHECK(clReleaseMemObject(k_img));
|
||||
CL_CHECK(clReleaseMemObject(k_sub));
|
||||
CL_CHECK(clReleaseMemObject(q_sub));
|
||||
}
|
||||
|
||||
cl_mem sinks_buf = extra_q->data_device;
|
||||
cl_ulong sinks_off = 0;
|
||||
@@ -23247,7 +23402,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
|
||||
ggml_backend_t backend,
|
||||
const ggml_tensor * q, const ggml_tensor * k,
|
||||
ggml_tensor * dst,
|
||||
int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct
|
||||
int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct, bool kq_int8
|
||||
) {
|
||||
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
|
||||
const ggml_tensor * v = dst->src[2];
|
||||
@@ -23263,8 +23418,31 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
|
||||
const int64_t kqv_m = pad_up(dv, 64);
|
||||
const int64_t tile_n = pad_up(n_q_chunk, 32);
|
||||
|
||||
// Pad the int8 row pitch by one cache line, as in the P8 KQV kernel.
|
||||
const int kv_pitch = (int) n_kv + 64;
|
||||
static const int64_t kv_pitch_round = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND");
|
||||
const int64_t v = (e && e[0]) ? strtoll(e, nullptr, 10) : 32768;
|
||||
return v > 0 && v <= INT_MAX && (v & (v - 1)) == 0 ? v : (int64_t) 0;
|
||||
}();
|
||||
static const int64_t kv_pitch_round_min = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND_MIN");
|
||||
return (e && e[0]) ? strtoll(e, nullptr, 10) : (int64_t) 16384;
|
||||
}();
|
||||
static const int64_t kv_pitch_round_max_dv = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND_MAX_DV");
|
||||
return (e && e[0]) ? strtoll(e, nullptr, 10) : (int64_t) 256;
|
||||
}();
|
||||
|
||||
// Round long P and V^T rows to reduce cache conflicts, then add one cache line.
|
||||
int64_t kv_pitch_value = n_kv + 64;
|
||||
if (kv_pitch_round > 0 && n_kv > kv_pitch_round_min && dv <= kv_pitch_round_max_dv) {
|
||||
const int64_t rounded_pitch = pad_up(n_kv, kv_pitch_round) + 64;
|
||||
if (rounded_pitch <= INT_MAX &&
|
||||
(size_t) rounded_pitch <= SIZE_MAX / n_q_chunk / n_head &&
|
||||
(size_t) rounded_pitch <= SIZE_MAX / dv / n_head_kv) {
|
||||
kv_pitch_value = rounded_pitch;
|
||||
}
|
||||
}
|
||||
const int kv_pitch = (int) kv_pitch_value;
|
||||
const size_t nblk = (size_t) kv_pitch / 32;
|
||||
|
||||
backend_ctx->prealloc_fa_pq.allocate(backend_ctx->context, (size_t) kv_pitch * n_q_chunk * n_head);
|
||||
@@ -23308,13 +23486,17 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
}
|
||||
|
||||
if (kq_int8) {
|
||||
ggml_cl_fa_kq_int8_prepare(backend, dst, n_q_chunk);
|
||||
}
|
||||
|
||||
ggml_tensor_extra_cl extra_kqv;
|
||||
ggml_tensor kqv;
|
||||
|
||||
for (int64_t q0 = 0; q0 < n_q; q0 += n_q_chunk) {
|
||||
const int64_t nqc = MIN(n_q_chunk, n_q - q0);
|
||||
|
||||
ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch);
|
||||
ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch, kq_int8);
|
||||
|
||||
ggml_cl_fa_decompose_scratch_tensor(kqv, extra_kqv, backend_ctx->prealloc_fa_kqv.buffer, GGML_TYPE_F32, dv, nqc, n_head);
|
||||
|
||||
@@ -23371,6 +23553,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
|
||||
// head size. GGML_OPENCL_FA_DECOMPOSE_MB sets the KQ scratch budget (default 256).
|
||||
// GGML_OPENCL_FA_KQ_P8=0 disables U8 probabilities and int8 KQV.
|
||||
// GGML_OPENCL_FA_KQV_DIRECT=0 disables direct PV for small query batches.
|
||||
// GGML_OPENCL_FA_KQ_INT8=0 keeps F16 KQ; =1 enables int8 KQ for supported shapes.
|
||||
static bool ggml_cl_flash_attn_decompose_dispatch(
|
||||
ggml_backend_t backend,
|
||||
const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst
|
||||
@@ -23500,12 +23683,46 @@ static bool ggml_cl_flash_attn_decompose_dispatch(
|
||||
return false;
|
||||
}
|
||||
|
||||
static const int kq_int8_env = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KQ_INT8");
|
||||
return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
|
||||
}();
|
||||
static const int dk512_env = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KQ_DK512_F16");
|
||||
return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
|
||||
}();
|
||||
const bool dk512 = dk > 256 && dk <= 512 && n_q >= 32 &&
|
||||
(dk512_env == 1 || (dk512_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E));
|
||||
const bool kq_int8_possible = kq_p8 && (dk <= 256 || dk512) && dk % 32 == 0 &&
|
||||
backend_ctx->kernel_fa_q8_rows_f16 != nullptr && backend_ctx->kernel_fa_q8_rows_f32 != nullptr &&
|
||||
n_head <= INT_MAX && n_head_kv <= INT_MAX &&
|
||||
n_q <= INT_MAX / n_head / (dk / 32) && n_kv <= INT_MAX / n_head_kv / (dk / 32) &&
|
||||
(dst->src[4] == nullptr || dk > 64) &&
|
||||
(kq_int8_env == 1 || (kq_int8_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && n_kv >= 2048)) &&
|
||||
backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f16) >= 64 &&
|
||||
backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f32) >= 64;
|
||||
cl_kernel kq_int8_kernel = nullptr;
|
||||
if (kq_int8_possible) {
|
||||
kq_int8_kernel = dk <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
|
||||
}
|
||||
const bool kq_int8 = kq_int8_kernel != nullptr && backend_ctx->get_kernel_workgroup_size(kq_int8_kernel) >= 64;
|
||||
|
||||
// Fewer query tiles per KV block reduce cache pressure in the int8 KQ kernel.
|
||||
static const int64_t kq_int8_max_chunk = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KQ_INT8_MAX_CHUNK");
|
||||
const int v = (e && e[0]) ? atoi(e) : 96;
|
||||
return (int64_t) (v >= 32 ? (v / 32) * 32 : 96);
|
||||
}();
|
||||
if (kq_int8) {
|
||||
n_q_chunk = MIN(n_q_chunk, kq_int8_max_chunk);
|
||||
}
|
||||
|
||||
// even out the chunks, then align to the 32-wide n-tile
|
||||
const int64_t n_chunks = (n_q + n_q_chunk - 1) / n_q_chunk;
|
||||
n_q_chunk = MIN(n_q_chunk, ((n_q + n_chunks - 1) / n_chunks + 31) & ~(int64_t) 31);
|
||||
|
||||
if (kq_p8) {
|
||||
ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct);
|
||||
ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct, kq_int8);
|
||||
} else {
|
||||
ggml_cl_flash_attn_decompose(backend, q, k, dst, n_q_chunk, scale, max_bias);
|
||||
}
|
||||
@@ -29557,6 +29774,77 @@ static void ggml_cl_mul_mat_q5_K_f32_adreno_ila(ggml_backend_t backend, const gg
|
||||
const int gemm_tile_n = 64;
|
||||
int N_pad = CEIL_DIV(N, gemm_tile_n) * gemm_tile_n;
|
||||
|
||||
const bool have_a8 = backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a8_bin != nullptr;
|
||||
|
||||
if (!have_a8) {
|
||||
cl_mem b_src_sub = nullptr;
|
||||
cl_mem b_pad_sub = nullptr;
|
||||
cl_mem b_buf = nullptr;
|
||||
region.origin = offset1;
|
||||
region.size = (size_t)K * N * sizeof(float);
|
||||
CL_CHECK((b_src_sub = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
|
||||
if (N_pad == N) {
|
||||
b_buf = b_src_sub;
|
||||
} else {
|
||||
size_t padded_bytes = (size_t)K * N_pad * sizeof(float);
|
||||
backend_ctx->prealloc_act_trans.allocate(context, padded_bytes);
|
||||
region.origin = 0;
|
||||
region.size = padded_bytes;
|
||||
CL_CHECK((b_pad_sub = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err));
|
||||
|
||||
const float b_pad_zero = 0.0f;
|
||||
CL_CHECK(clEnqueueFillBuffer(backend_ctx->queue, b_pad_sub, &b_pad_zero, sizeof(float),
|
||||
0, padded_bytes, 0, NULL, NULL));
|
||||
CL_CHECK(clEnqueueCopyBuffer(backend_ctx->queue, b_src_sub, b_pad_sub, 0, 0, (size_t)K * N * sizeof(float), 0, NULL, NULL));
|
||||
b_buf = b_pad_sub;
|
||||
}
|
||||
|
||||
img_fmt = { CL_R, CL_FLOAT };
|
||||
memset(&img_desc, 0, sizeof(img_desc));
|
||||
img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
|
||||
img_desc.image_width = (size_t)K * N_pad;
|
||||
img_desc.buffer = b_buf;
|
||||
cl_mem b_img_a7;
|
||||
CL_CHECK((b_img_a7 = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, NULL, &err), err));
|
||||
|
||||
const int c_offset_for_kernel = (int)offsetd;
|
||||
|
||||
int line_stride_B = K * (int)sizeof(float);
|
||||
int line_stride_C = M * (int)sizeof(float);
|
||||
|
||||
kernel = backend_ctx->kernel_gemm_noshuffle_q5_k_f32_32b_trans_ila_a7_bin;
|
||||
cl_uint k_arg = 0;
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->q_img));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->qh));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->d));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->dm));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extra0_q5_k->s));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &b_img_a7));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &offset1));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_mem), &extrad->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &c_offset_for_kernel));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne00));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(cl_uint), &ne01));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &line_stride_C));
|
||||
CL_CHECK(clSetKernelArg(kernel, k_arg++, sizeof(int), &N));
|
||||
|
||||
size_t local_work_size[3] = { 64, 2, 4 };
|
||||
size_t m_tiles = (size_t)CEIL_DIV(M, 64);
|
||||
size_t global_y = ((m_tiles + 1) / 2) * 2;
|
||||
size_t global_work_size[3] = { 64, global_y, (size_t)CEIL_DIV(N_pad, 32) };
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst);
|
||||
|
||||
CL_CHECK(clReleaseMemObject(b_img_a7));
|
||||
if (b_src_sub) {
|
||||
CL_CHECK(clReleaseMemObject(b_src_sub));
|
||||
}
|
||||
if (b_pad_sub) {
|
||||
CL_CHECK(clReleaseMemObject(b_pad_sub));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
cl_mem b_sub_buf = nullptr;
|
||||
cl_mem b_padded = nullptr;
|
||||
cl_mem b_buf = nullptr;
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
|
||||
// Store 32-element blocks as separate int8 values and half scales.
|
||||
|
||||
kernel void kernel_fa_q8_rows_f16(
|
||||
global const char * src, ulong off_src, ulong nb1, ulong nb2,
|
||||
global char * dq, global half * dd,
|
||||
int ne0, int nrow, int nhead
|
||||
) {
|
||||
const int nblk = ne0 / 32;
|
||||
const int gid = get_global_id(0);
|
||||
if (gid >= nblk*nrow*nhead) {
|
||||
return;
|
||||
}
|
||||
const int b = gid % nblk;
|
||||
const int r = (gid / nblk) % nrow;
|
||||
const int h = gid / (nblk*nrow);
|
||||
|
||||
global const half * p = (global const half *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
|
||||
float v[32];
|
||||
float amax = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
const float4 x = convert_float4(vload4(i, p));
|
||||
v[i*4+0] = x.s0;
|
||||
v[i*4+1] = x.s1;
|
||||
v[i*4+2] = x.s2;
|
||||
v[i*4+3] = x.s3;
|
||||
amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
|
||||
}
|
||||
const float d = amax / 127.0f;
|
||||
const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
|
||||
const size_t row = (size_t)h*nrow + r;
|
||||
dd[row*nblk + b] = (half)d;
|
||||
global char * q = dq + row*ne0 + b*32;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_fa_q8_rows_f32(
|
||||
global const char * src, ulong off_src, ulong nb1, ulong nb2,
|
||||
global char * dq, global half * dd,
|
||||
int ne0, int nrow, int nhead
|
||||
) {
|
||||
const int nblk = ne0 / 32;
|
||||
const int gid = get_global_id(0);
|
||||
if (gid >= nblk*nrow*nhead) {
|
||||
return;
|
||||
}
|
||||
const int b = gid % nblk;
|
||||
const int r = (gid / nblk) % nrow;
|
||||
const int h = gid / (nblk*nrow);
|
||||
|
||||
global const float * p = (global const float *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
|
||||
float v[32];
|
||||
float amax = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
const float4 x = vload4(i, p);
|
||||
v[i*4+0] = x.s0;
|
||||
v[i*4+1] = x.s1;
|
||||
v[i*4+2] = x.s2;
|
||||
v[i*4+3] = x.s3;
|
||||
amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
|
||||
}
|
||||
const float d = amax / 127.0f;
|
||||
const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
|
||||
const size_t row = (size_t)h*nrow + r;
|
||||
dd[row*nblk + b] = (half)d;
|
||||
global char * q = dq + row*ne0 + b*32;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
|
||||
#ifdef cl_khr_integer_dot_product
|
||||
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
|
||||
#endif
|
||||
|
||||
#define KQ_TN 32
|
||||
#define KQ_MB 8
|
||||
#define KQ_TM 64
|
||||
#define KQ_WG 64
|
||||
#define KQ_DK_MAX 256
|
||||
|
||||
__attribute__((qcom_wave_pair_mode(1)))
|
||||
kernel void kernel_mul_mm_q8_kq_p8(
|
||||
global const uint * kq, // K int8, [head_kv][n_kv][dk] as packed uints
|
||||
global const half * kd, // K scales, [head_kv][n_kv][dk/32]
|
||||
global const uint * qq, // Q int8, [head][n_q][dk] as packed uints
|
||||
global const half * qd, // Q scales, [head][n_q][dk/32]
|
||||
global uchar * qp, // u8 P, [head][n_q][p_pitch]
|
||||
global float * bmax, // per 32-row block max, [head][n_q][p_pitch/32]
|
||||
global float * bsum, // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
|
||||
global const char * mask, // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
|
||||
ulong offset_mask,
|
||||
ulong mask_nb1,
|
||||
float scale,
|
||||
float max_bias,
|
||||
float m0,
|
||||
float m1,
|
||||
int n_head_log2,
|
||||
int dk,
|
||||
int n_kv,
|
||||
int n_q,
|
||||
int n_head,
|
||||
int n_head_kv,
|
||||
int p_pitch // bytes per P row, >= n_kv, multiple of 32
|
||||
) {
|
||||
const int lid = get_local_id(0);
|
||||
const int gsz = n_head / n_head_kv;
|
||||
const int g0 = get_group_id(0);
|
||||
const int head_kv = get_group_id(2);
|
||||
const int head = head_kv*gsz + (g0 % gsz);
|
||||
const int qn0 = (g0 / gsz) * KQ_TN;
|
||||
const int kv0 = get_group_id(1)*KQ_TM*KQ_MB + lid;
|
||||
|
||||
const int nblk = dk / 32;
|
||||
const int nu = dk / 4;
|
||||
|
||||
__local uint sh_q [KQ_TN][KQ_DK_MAX/4];
|
||||
__local half sh_qd[KQ_TN][KQ_DK_MAX/32];
|
||||
__local float p8_lds[1024];
|
||||
|
||||
const size_t qbase = (size_t)head*n_q;
|
||||
for (int i = lid; i < KQ_TN*(KQ_DK_MAX/16); i += KQ_WG) {
|
||||
const int t = i / (KQ_DK_MAX/16);
|
||||
const int u = (i % (KQ_DK_MAX/16)) * 4;
|
||||
const int qi = qn0 + t;
|
||||
uint4 v = (uint4)(0u);
|
||||
if (qi < n_q && u < nu) {
|
||||
v = vload4(0, &qq[(qbase + qi)*nu + u]);
|
||||
}
|
||||
vstore4(v, 0, &sh_q[t][u]);
|
||||
}
|
||||
for (int i = lid; i < KQ_TN*(KQ_DK_MAX/32); i += KQ_WG) {
|
||||
const int t = i / (KQ_DK_MAX/32);
|
||||
const int b = i % (KQ_DK_MAX/32);
|
||||
const int qi = qn0 + t;
|
||||
sh_qd[t][b] = (qi < n_q && b < nblk) ? qd[(qbase + qi)*nblk + b] : (half)0.0f;
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
|
||||
float slope = 1.0f;
|
||||
if (max_bias > 0.0f) {
|
||||
const float base = head < n_head_log2 ? m0 : m1;
|
||||
const int ex = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
|
||||
slope = pow(base, ex);
|
||||
}
|
||||
global const half * mrow = (global const half *)(mask + offset_mask);
|
||||
const uint mstride = (uint)(mask_nb1 / 2);
|
||||
const int pblk = p_pitch / 32;
|
||||
|
||||
for (int m = 0; m < KQ_MB; ++m) {
|
||||
const int kv = kv0 + m*KQ_TM;
|
||||
if (kv - lid >= n_kv) {
|
||||
break; // uniform across the workgroup
|
||||
}
|
||||
|
||||
float acc[KQ_TN];
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
acc[t] = 0.0f;
|
||||
}
|
||||
|
||||
const int kl = min(kv, n_kv - 1);
|
||||
const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
|
||||
const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
|
||||
|
||||
for (int b = 0; b < nblk; ++b) {
|
||||
const uint4 w0 = vload4(0, &kq[kbase + b*8]);
|
||||
const uint4 w1 = vload4(0, &kq[kbase + b*8 + 4]);
|
||||
const float dks = (float)kd[kdbas + b];
|
||||
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
const uint4 a0 = vload4(0, &sh_q[t][b*8]);
|
||||
const uint4 a1 = vload4(0, &sh_q[t][b*8 + 4]);
|
||||
int raw = 0;
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
|
||||
acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
|
||||
}
|
||||
}
|
||||
|
||||
// Each lane owns one complete 32-row softmax block after the transpose.
|
||||
const int kvblk0 = kv - lid;
|
||||
#pragma unroll
|
||||
for (int half_ = 0; half_ < 2; ++half_) {
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 16; ++j) {
|
||||
p8_lds[j*64 + lid] = acc[half_*16 + j];
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
if (lid < 32) {
|
||||
const int j = lid >> 1;
|
||||
const int b = lid & 1;
|
||||
const int n = qn0 + half_*16 + j;
|
||||
if (n < n_q) {
|
||||
const int kvb = kvblk0 + b*32;
|
||||
const __local float * src = p8_lds + j*64 + b*32;
|
||||
float16 s0 = vload16(0, src);
|
||||
float16 s1 = vload16(1, src);
|
||||
const global half * mp = mrow + (size_t)n*mstride + kvb;
|
||||
const float16 m0v = convert_float16(vload16(0, mp));
|
||||
const float16 m1v = convert_float16(vload16(1, mp));
|
||||
s0 = s0*scale + slope*m0v;
|
||||
s1 = s1*scale + slope*m1v;
|
||||
const float16 mx16 = fmax(s0, s1);
|
||||
const float8 mx8 = fmax(mx16.lo, mx16.hi);
|
||||
const float4 mx4 = fmax(mx8.lo, mx8.hi);
|
||||
const float2 mx2 = fmax(mx4.lo, mx4.hi);
|
||||
const float amax = fmax(mx2.x, mx2.y);
|
||||
const size_t prow = qbase + n;
|
||||
const int blk = kvb >> 5;
|
||||
float sum = 0.0f;
|
||||
uchar16 q0 = (uchar16)(0);
|
||||
uchar16 q1 = (uchar16)(0);
|
||||
// finite sentinel: the program builds with -cl-finite-math-only
|
||||
if (amax > -1.0e30f) {
|
||||
const float16 e0 = exp(s0 - amax);
|
||||
const float16 e1 = exp(s1 - amax);
|
||||
const float16 a16 = e0 + e1;
|
||||
const float8 a8 = a16.lo + a16.hi;
|
||||
const float4 a4 = a8.lo + a8.hi;
|
||||
const float2 a2 = a4.lo + a4.hi;
|
||||
sum = a2.x + a2.y;
|
||||
q0 = convert_uchar16_sat_rte(e0*255.0f);
|
||||
q1 = convert_uchar16_sat_rte(e1*255.0f);
|
||||
}
|
||||
global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
|
||||
vstore16(q0, 0, qdst);
|
||||
vstore16(q1, 1, qdst);
|
||||
bmax[prow*pblk + blk] = amax;
|
||||
bsum[prow*pblk + blk] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
|
||||
#ifdef cl_khr_integer_dot_product
|
||||
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
|
||||
#endif
|
||||
|
||||
#define KQ_TN 32
|
||||
#define KQ_MB 8
|
||||
#define KQ_TM 64
|
||||
#define KQ_WG 64
|
||||
#define KQ_DKC 256
|
||||
|
||||
__attribute__((qcom_wave_pair_mode(1)))
|
||||
kernel void kernel_mul_mm_q8_kq_p8_dk512(
|
||||
global const uint * kq, // K int8, [head_kv][n_kv][dk] as packed uints
|
||||
global const half * kd, // K scales, [head_kv][n_kv][dk/32]
|
||||
global const uint * qq, // Q int8, [head][n_q][dk] as packed uints
|
||||
global const half * qd, // Q scales, [head][n_q][dk/32]
|
||||
global uchar * qp, // u8 P, [head][n_q][p_pitch]
|
||||
global float * bmax, // per 32-row block max, [head][n_q][p_pitch/32]
|
||||
global float * bsum, // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
|
||||
global const char * mask, // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
|
||||
ulong offset_mask,
|
||||
ulong mask_nb1,
|
||||
float scale,
|
||||
float max_bias,
|
||||
float m0,
|
||||
float m1,
|
||||
int n_head_log2,
|
||||
int dk,
|
||||
int n_kv,
|
||||
int n_q,
|
||||
int n_head,
|
||||
int n_head_kv,
|
||||
int p_pitch // bytes per P row, >= n_kv, multiple of 32
|
||||
) {
|
||||
const int lid = get_local_id(0);
|
||||
const int gsz = n_head / n_head_kv;
|
||||
const int g0 = get_group_id(0);
|
||||
const int head_kv = get_group_id(2);
|
||||
const int head = head_kv*gsz + (g0 % gsz);
|
||||
const int qn0 = (g0 / gsz) * KQ_TN;
|
||||
const int kv0 = get_group_id(1)*KQ_TM*KQ_MB + lid;
|
||||
|
||||
const int nblk = dk / 32;
|
||||
const int nu = dk / 4;
|
||||
|
||||
// Reuse the 256-wide Q slice for the F32 softmax scores after the contraction.
|
||||
__local uint sh_q [KQ_TN*KQ_DKC/4];
|
||||
__local half sh_qd[KQ_TN][KQ_DKC/32];
|
||||
__local float * p8_lds = (__local float *)sh_q;
|
||||
|
||||
const size_t qbase = (size_t)head*n_q;
|
||||
|
||||
float slope = 1.0f;
|
||||
if (max_bias > 0.0f) {
|
||||
const float base = head < n_head_log2 ? m0 : m1;
|
||||
const int ex = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
|
||||
slope = pow(base, ex);
|
||||
}
|
||||
global const half * mrow = (global const half *)(mask + offset_mask);
|
||||
const uint mstride = (uint)(mask_nb1 / 2);
|
||||
const int pblk = p_pitch / 32;
|
||||
|
||||
for (int m = 0; m < KQ_MB; ++m) {
|
||||
const int kv = kv0 + m*KQ_TM;
|
||||
if (kv - lid >= n_kv) {
|
||||
break; // uniform across the workgroup
|
||||
}
|
||||
|
||||
float acc[KQ_TN];
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
acc[t] = 0.0f;
|
||||
}
|
||||
|
||||
const int kl = min(kv, n_kv - 1);
|
||||
const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
|
||||
const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
|
||||
|
||||
for (int c0 = 0; c0 < nblk; c0 += KQ_DKC/32) {
|
||||
const int cb = min(KQ_DKC/32, nblk - c0);
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
for (int i = lid; i < KQ_TN*(KQ_DKC/16); i += KQ_WG) {
|
||||
const int t = i / (KQ_DKC/16);
|
||||
const int u = (i % (KQ_DKC/16)) * 4;
|
||||
const int qi = qn0 + t;
|
||||
uint4 v = (uint4)(0u);
|
||||
if (qi < n_q && u < cb*8) {
|
||||
v = vload4(0, &qq[(qbase + qi)*nu + c0*8 + u]);
|
||||
}
|
||||
vstore4(v, 0, &sh_q[t*(KQ_DKC/4) + u]);
|
||||
}
|
||||
for (int i = lid; i < KQ_TN*(KQ_DKC/32); i += KQ_WG) {
|
||||
const int t = i / (KQ_DKC/32);
|
||||
const int b = i % (KQ_DKC/32);
|
||||
const int qi = qn0 + t;
|
||||
sh_qd[t][b] = (qi < n_q && b < cb) ? qd[(qbase + qi)*nblk + c0 + b] : (half)0.0f;
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
|
||||
for (int b = 0; b < cb; ++b) {
|
||||
const uint4 w0 = vload4(0, &kq[kbase + (c0 + b)*8]);
|
||||
const uint4 w1 = vload4(0, &kq[kbase + (c0 + b)*8 + 4]);
|
||||
const float dks = (float)kd[kdbas + c0 + b];
|
||||
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
const uint4 a0 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8]);
|
||||
const uint4 a1 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8 + 4]);
|
||||
int raw = 0;
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
|
||||
acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Finish reading the Q slice before writing scores to the same local memory.
|
||||
const int kvblk0 = kv - lid;
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
p8_lds[t*64 + lid] = acc[t];
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
{
|
||||
const int j = lid >> 1;
|
||||
const int b = lid & 1;
|
||||
const int n = qn0 + j;
|
||||
if (n < n_q) {
|
||||
const int kvb = kvblk0 + b*32;
|
||||
__local float * src = p8_lds + j*64 + b*32;
|
||||
const global half * mp = mrow + (size_t)n*mstride + kvb;
|
||||
float4 s = vload4(0, src)*scale + slope*convert_float4(vload4(0, mp));
|
||||
vstore4(s, 0, src);
|
||||
float amax = fmax(fmax(s.x, s.y), fmax(s.z, s.w));
|
||||
for (int c = 1; c < 8; ++c) {
|
||||
s = vload4(c, src)*scale + slope*convert_float4(vload4(c, mp));
|
||||
vstore4(s, c, src);
|
||||
amax = fmax(amax, fmax(fmax(s.x, s.y), fmax(s.z, s.w)));
|
||||
}
|
||||
const size_t prow = qbase + n;
|
||||
const int blk = kvb >> 5;
|
||||
global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
|
||||
float sum = 0.0f;
|
||||
// finite sentinel: the program builds with -cl-finite-math-only
|
||||
if (amax > -1.0e30f) {
|
||||
for (int c = 0; c < 8; ++c) {
|
||||
const float4 e = exp(vload4(c, src) - amax);
|
||||
sum += (e.x + e.y) + (e.z + e.w);
|
||||
vstore4(convert_uchar4_sat_rte(e*255.0f), c, qdst);
|
||||
}
|
||||
} else {
|
||||
for (int c = 0; c < 8; ++c) {
|
||||
vstore4((uchar4)(0), c, qdst);
|
||||
}
|
||||
}
|
||||
bmax[prow*pblk + blk] = amax;
|
||||
bsum[prow*pblk + blk] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -31,3 +31,20 @@ if (GGML_OPENVINO)
|
||||
message(FATAL_ERROR "OpenVINO: OpenVINO toolkit supports x86-64 and arm64 but not ${CMAKE_SYSTEM_PROCESSOR}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# Released runtimes retain the ordinary cache translators.
|
||||
include(CheckCXXSourceCompiles)
|
||||
unset(GGML_OPENVINO_HAS_CACHE_UPDATE CACHE)
|
||||
set(CMAKE_REQUIRED_LIBRARIES openvino::runtime)
|
||||
set(CMAKE_REQUIRED_INCLUDES "${CMAKE_CURRENT_SOURCE_DIR}/openvino/op")
|
||||
check_cxx_source_compiles("#include <recurrent_cache_update.hpp>
|
||||
int main() {
|
||||
auto conv = std::make_shared<ov::op::internal::RecurrentConvCacheUpdate>(ov::OutputVector{}, 3);
|
||||
auto gdn = std::make_shared<ov::op::internal::RecurrentGatedDeltaNetUpdate>(ov::OutputVector{});
|
||||
}"
|
||||
GGML_OPENVINO_HAS_CACHE_UPDATE)
|
||||
unset(CMAKE_REQUIRED_LIBRARIES)
|
||||
unset(CMAKE_REQUIRED_INCLUDES)
|
||||
if(GGML_OPENVINO_HAS_CACHE_UPDATE)
|
||||
target_compile_definitions(ggml-openvino PRIVATE GGML_OPENVINO_HAS_CACHE_UPDATE=1)
|
||||
endif()
|
||||
|
||||
@@ -1,3 +1,38 @@
|
||||
# Qwen MTP
|
||||
|
||||
The default Qwen3.5/Qwen3.6 GPU path is dynamic stateless execution with in-place fused recurrent ops and stable MTP sampling. Build against the matching OpenVINO runtime/GPU plugin with independent `RecurrentConvCacheUpdate` and `RecurrentGatedDeltaNetUpdate` ops. The native Update ops return computation results and update writable cache inputs in place, including rollback snapshots where required. No recurrent/sampling opt-in flag is needed on supported graphs. See the [development documentation index](dev_doc/README.md) for the implementation and the MTP sampling graph.
|
||||
|
||||
Use `--spec-type draft-mtp --spec-draft-n-max 2` with a GGUF that contains MTP weights. Parallel execution uses `-np` and `--kv-unified`. CPU snapshot execution, NPU execution, and forced static MTP are not supported.
|
||||
|
||||
Stateless recurrent execution uses the same reset, reorder, and partial cache-writeback paths at every slot capacity, including one slot without MTP.
|
||||
|
||||
The matched stateless GPU path uses `RecurrentConvCacheUpdate` and
|
||||
`RecurrentGatedDeltaNetUpdate`. Each cache is a writable Parameter backed by one
|
||||
stable caller-owned USM allocation. The ops return only convolution or attention;
|
||||
there are no recurrent cache Results. Required source rows are staged before
|
||||
writes, preserving resets, cyclic reorders and rollback. Cache readers must be
|
||||
exclusive, including Reshape aliases.
|
||||
`GGML_OPENVINO_DISABLE_FUSED_RECURRENCE=1` retains the ordinary cache decomposition.
|
||||
Unmatched snapshot GDN uses ordinary GDN token prefixes with functional writeback.
|
||||
Build both repos against the matching single-output op spec. Translation revision
|
||||
15 requires regenerating compiled-model blobs once without cache-only mode.
|
||||
|
||||
Cache-only MTP serving still requires the separate OpenVINO dynamic output-binding fix (PR #39354). Removing recurrent cache Results does not eliminate that independent trigger in imported MTP graphs.
|
||||
|
||||
Set `GGML_OPENVINO_ENABLE_KV_4D=1` to enable the upstream stateless in-place
|
||||
attention KV update pattern where eligible. It binds full-capacity cache tensors
|
||||
and lets the GPU fusion expose the live prefix to attention. The current upstream
|
||||
eligibility check supports one physical stream (including unified KV); explicit
|
||||
split multi-slot KV retains the ordinary full-cache path. This option does not
|
||||
change recurrent cache handling. Unified KV is recommended for multi-slot GPU
|
||||
execution because it enables this in-place attention update.
|
||||
|
||||
The recommended Qwen3.6-35B-A3B Q4_K_M setup uses GPU with `GGML_OPENVINO_REQUANT_KQUANT=q4_asym64_all` and `GGML_OPENVINO_NATIVE_SOFTPLUS=1`. Leave `GGML_OPENVINO_KV_SLICE` at its default (`1`). See [attention KV layout](dev_doc/OPENVINO-RECURRENT-CACHE.md#attention-kv-layout) for eligibility.
|
||||
|
||||
`q4_asym64_all` converts Q4_K/Q5_K/Q6_K/Q8_0/Q5_1 weights, including dense Q8_0 projections, to asymmetric INT4 with group size 64. `token_embd.weight` and `output.weight` retain symmetric channel-wise INT8. Q8_0/Q5_1 rows whose width is not divisible by 64 also use channel-wise INT8. This follows the primary/backup precision scheme of NNCF `INT4_ASYM`, `ratio=1.0`, `group_size=64`, `backup_mode=INT8_SYM` for these supported source types; it requantizes GGUF values rather than the original floating-point model and leaves floating-point weights unchanged. Earlier versions kept dense Q8_0 at INT8/group-32. Repopulate compiled-model caches without cache-only mode after this change; the cache fingerprint invalidates the old precision policy.
|
||||
|
||||
See the [implementation guide](dev_doc/OPENVINO-RECURRENT-CACHE.md) for buffer ownership and rollback.
|
||||
|
||||
# Compiled model cache
|
||||
|
||||
`GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` exports compiled CPU/GPU graphs with their weights. It bypasses the plugin-level `GGML_OPENVINO_CACHE_DIR` and uses `OPTIMIZE_SPEED`, so weightless caching is disabled.
|
||||
@@ -7,7 +42,6 @@ One directory can hold blobs for different models and compilation settings. Run
|
||||
```sh
|
||||
GGML_OPENVINO_DEVICE=GPU \
|
||||
GGML_OPENVINO_NATIVE_SOFTPLUS=1 \
|
||||
GGML_OPENVINO_DISABLE_KV_SLICE=1 \
|
||||
GGML_OPENVINO_REQUANT_KQUANT=q4_asym64_all \
|
||||
GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/path/to/qwen-cache \
|
||||
./build/ReleaseOV/bin/llama-bench -m /path/to/model.gguf -r 1
|
||||
@@ -20,7 +54,6 @@ On later runs, supply the same compilation settings and enable `GGML_OPENVINO_CO
|
||||
```sh
|
||||
GGML_OPENVINO_DEVICE=GPU \
|
||||
GGML_OPENVINO_NATIVE_SOFTPLUS=1 \
|
||||
GGML_OPENVINO_DISABLE_KV_SLICE=1 \
|
||||
GGML_OPENVINO_REQUANT_KQUANT=q4_asym64_all \
|
||||
GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/path/to/qwen-cache \
|
||||
GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1 \
|
||||
@@ -29,10 +62,22 @@ GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1 \
|
||||
|
||||
Cache-only mode allocates backend address space without filling weight pages. On Windows, this also uses system commit capacity. The model-buffer size in the loader log is this virtual size. Weight uploads only record source identity; they do not read or requantize the weights. Graph conversion and compilation are skipped. Runtime buffers are still allocated and populated normally.
|
||||
|
||||
Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode.
|
||||
Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_EXACT_ZERO_POINT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK`, and `GGML_OPENVINO_DEBUG_NODE`, `GGML_OPENVINO_ENABLE_KV_4D`, and `GGML_OPENVINO_DISABLE_FUSED_RECURRENCE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode.
|
||||
|
||||
`GGML_OPENVINO_KV_SLICE` is a runtime binding option for dynamic graphs and can differ between cache population and cache-only runs. The same dynamic compiled graph accepts full-capacity and live-prefix attention K/V tensors. Slicing is enabled by default for contiguous ordinary K/V roots with one allocated stream, one active sequence at offset zero, and no sliding-window attention. Recurrent convolution/GDN caches and their rollback snapshots always retain their full stored row bank. Other layouts use full bindings. Repopulate caches made before this option was removed from the fingerprint once without cache-only mode.
|
||||
|
||||
A missing or incompatible graph fails with an error instead of compiling with absent weights. The fingerprint uses the dynamic graph's topology, ports, model parameters, weights, settings, and OpenVINO version; changing only dynamic token or KV sizes does not require a new entry. A different workload can still require another graph; populate it first without cache-only mode.
|
||||
|
||||
## MTP sampling and active slots
|
||||
|
||||
Qwen3.5/Qwen3.6 MTP uses a pure top-k backend sampler. Dynamic stateless execution retains its sampling branches for every configured slot and output row, including inactive branches. One packed `sampling_rows` input selects logits rows at runtime. Graph node names and candidate-output ports stay fixed, so changing the active subset of four slots reuses the same compiled draft model instead of creating up to 15 variants.
|
||||
|
||||
Active candidate IDs and scores are written to the normal GGML outputs. Inactive branches read row 0 (the padding row when there are no logits) and write to private scratch tensors (device USM on GPU), without changing sampler state or touching inactive GGML buffers. The top-k computations remain separate per branch; this trades some extra sampling work for a stable graph interface. Other sampler chains retain their existing branch-selection behavior.
|
||||
|
||||
This is one draft model per compatible graph configuration, not one compiled blob for the whole server. The trunk and other graph stages still have their own entries. Different sampler configurations, recurrent capacities, rollback modes or incompatible KV layouts can require additional models. Cache-only mode still requires initial preparation; changing just the active subset no longer requires exhaustive subset priming.
|
||||
|
||||
The generic recurrent-cache translation invalidates blobs made with the former single-slot shortcuts. Run once without `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY` to populate compatible entries before enabling cache-only mode again.
|
||||
|
||||
## Restrictions
|
||||
|
||||
- Cache-only mode requires Linux or Windows and mmap loading (`--load-mode mmap`, or the default when all selected devices support mmap). Do not use tensor validation or mlock when trying to avoid weight reads.
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# Stable multislot MTP graph
|
||||
|
||||
The captured UD draft graph contains 107 nodes regardless of which of four slots are active. It is the single MTP attention/MoE layer, not the recurrent trunk. Two speculative tokens require two consecutive draft calls, batched over active sequences.
|
||||
|
||||
```text
|
||||
MTP token IDs + input hidden state
|
||||
|
|
||||
0-84: MTP layer and normalization, N active sequences
|
||||
|
|
||||
85: logits [248320, N]
|
||||
86: PAD adds one safe row
|
||||
|
|
||||
+-- 87-91: slot 0 sampling
|
||||
+-- 92-96: slot 1 sampling
|
||||
+-- 97-101: slot 2 sampling
|
||||
+-- 102-106: slot 3 sampling
|
||||
```
|
||||
|
||||
Each branch contains VIEW, two Reshapes, TopK (ten candidate IDs), and GET_ROWS (their scores). Ten is the candidate count, not the speculative-token count.
|
||||
|
||||
| Active slots | Sampling COMPUTE flags, slots 0-3 | Raw/current nodes | Old filtered nodes | Packed sampling_rows |
|
||||
|---|---|---:|---:|---|
|
||||
| 3 | `[0,0,0,1]` | 107 | 92 | `[0,0,0,0]` |
|
||||
| 0,2 | `[1,0,1,0]` | 107 | 97 | `[0,0,1,0]` |
|
||||
| All four | `[1,1,1,1]` | 107 | 107 | `[0,1,2,3]` |
|
||||
|
||||
Old counts apply the old COMPUTE filter to the captures. Unselected nodes remain in the raw cgraph. Previously, removing them changed translated topology, slot-specific row Parameter names and output ports; even equal-sized subsets such as `{0,1}` and `{0,2}` differed. This could trigger up to 15 draft variants, with compilation/import on first use or a missing-cache error in cache-only mode.
|
||||
|
||||
Now all pure top-k branches remain. Their names include stable graph ordinals even when inactive. One int64 `sampling_rows` Parameter supplies row indices at runtime. For `{0,2}`, the two logits rows belong to slots 0 and 2:
|
||||
|
||||
```text
|
||||
sampling_rows = [0, 0, 1, 0]
|
||||
slot 0 -> row 0 -> normal candidate buffers
|
||||
slot 1 -> row 0 -> private scratch, ignored
|
||||
slot 2 -> row 1 -> normal candidate buffers
|
||||
slot 3 -> row 0 -> private scratch, ignored
|
||||
```
|
||||
|
||||
The vector is not an active mask. Host-side COMPUTE flags select normal versus scratch output destinations. PAD provides a valid row when there are no logits. Inactive outputs use device USM on GPU and do not affect sampler state or acceptance. Other sampler chains retain their selected-branch semantics.
|
||||
|
||||
Only small sampling work executes for inactive slots. The MTP layer and output projection still process N active rows. Top-k is separate per branch, not yet one batched TopK. The observed draft interface has 10 inputs and 12 outputs: eight candidate ID/score outputs, hidden state, full logits and two KV Results.
|
||||
|
||||
Dump shapes use GGML order `[width, rows, dim2, dim3]`; OV reverses it. `view_offs` is in bytes: one logits row occupies `248320 * 4 = 993280` bytes. Buffer labels describe GGML storage, not the device executing imported GPU operations.
|
||||
|
||||
The captures used cache-only mode, four slots, context 8192 and the default two-token MTP settings.
|
||||
@@ -0,0 +1,62 @@
|
||||
# Current Qwen GPU implementation
|
||||
|
||||
The default Qwen3.5/Qwen3.6 path is dynamic stateless execution with `RecurrentConvCacheUpdate`, `RecurrentGatedDeltaNetUpdate`, and stable pure-top-k MTP translation. It supports one or multiple slots and rollback. The implementation applies to `qwen35` and `qwen35moe`; the reference model is the UD Qwen3.6-35B-A3B MTP GGUF.
|
||||
|
||||
Build against the matching OpenVINO runtime/GPU plugin with single-output Update ops. Configure `GGML_OPENVINO_DEVICE=GPU`, native softplus, and the documented requantization policy. The cache translation revision is 15. Historical measurements retain their original source revisions.
|
||||
|
||||
The separate dynamic output-binding fix in OpenVINO PR #39354 remains required for cache-only MTP serving. Native cache-op export/import tests pass without it, but Qwen3.6 imported MTP graphs still trigger the missing-input-memory error without the fix.
|
||||
|
||||
## Buffer ownership and operation boundaries
|
||||
|
||||
| State | Storage and lifetime | Update path |
|
||||
|---|---|---|
|
||||
| Attention K/V | Full caller-owned USM allocation, sized by context and KV layout | Native stateless KV fusion when eligible; SDPA consumes the live prefix |
|
||||
| Convolution `cache_r` | Full caller-owned USM row bank, sized by slot/snapshot capacity | Fused reset, original-row gather/reorder, convolution and ordered history/snapshot writes |
|
||||
| GDN `cache_s` | Full caller-owned USM row bank, sized by slot/snapshot capacity | Fused reset, original-row gather/reorder, GDN and ordered final/snapshot writes |
|
||||
| Runtime controls | Reused USM host storage on GPU | Row maps, offsets and write controls remain runtime inputs |
|
||||
| Protected source rows | Private device scratch | Staged before writes to protect cycles, duplicate sources and overlaps |
|
||||
| Inactive sampling results | Private device-USM scratch | Discarded without touching inactive GGML output buffers |
|
||||
|
||||
Attention context capacity does not set recurrent-cache size. Each cache input Parameter binds a stable writable allocation; recurrent cache Results are omitted. There are no rotation banks or pointer swaps.
|
||||
|
||||
The GPU plugin uses two stages: first preserve required original rows in scratch while applying logical reset and precision rounding; then compute and perform ordered writes. This avoids overwriting a source before another destination has read it. Identity reorder sources need no scratch writes. In-place execution removes the spare full output bank, not every cache read or internal memory transaction.
|
||||
|
||||
`RecurrentConvCacheUpdate` has nine inputs, including depthwise weights, and returns convolution `[1,B,T,C]`. `RecurrentGatedDeltaNetUpdate` returns attention and writes the newest `min(T,snapshot_count)` states into its cache input, newest first. Neither op returns a cache tensor.
|
||||
|
||||
## Convolution cache stores
|
||||
|
||||
`build_conv_state_store` emits one CPY for the final history when rollback is disabled. With rollback enabled, it emits one SET_ROWS from overlapping `[history, window, channel, sequence]` views into strided snapshot planes. Window starts advance in time while destination planes descend, so plane zero holds the newest history. The fixed ARANGE/scale/cast chain supplies this descending slot order.
|
||||
|
||||
The backend recognizes this structure and refreshes source starts, cache heads, sequence counts, window counts and plane strides on every batch. Native fusion passes runtime `[source column, destination slot]` pairs to the existing convolution op; the pair-count dimension is dynamic. Functional translation gathers the same windows, scatters them into the full cache, and preserves unwritten slots and features. The native op spec is unchanged.
|
||||
|
||||
## Generic fallback and restrictions
|
||||
|
||||
Writable cache inputs must be GPU RemoteTensors with exclusive graph readers, optionally through Reshape aliases. The GPU plugin rejects shared cache readers and computed cache banks. Other inputs and the computation output must not overlap the cache. The frontend skips fusion for unmatched cache reads and retains functional cache writeback. Restore or clear state after a failed inference before retrying.
|
||||
|
||||
`GGML_OPENVINO_DISABLE_FUSED_RECURRENCE=1` retains ordinary cache decomposition. Naive graphs skip full-model recurrence fusion. Unmatched snapshot GDN uses ordinary GDN token prefixes to preserve state outputs. The fused path requires the matching native runtime; otherwise dynamic CPU and GPU execution uses ordinary GDN. Stateful and NPU MTP are outside this change.
|
||||
|
||||
## Attention KV layout
|
||||
|
||||
`GGML_OPENVINO_ENABLE_KV_4D=1` enables eligible native stateless in-place attention updates. The current eligibility covers one physical stream, including unified KV. Explicit split multi-slot KV retains the full-cache path and can be much slower even with only one active slot. Use `--kv-unified` for multi-slot OV GPU serving.
|
||||
|
||||
Leave `GGML_OPENVINO_KV_SLICE` at its default (`1`). Live-prefix slicing applies to eligible ordinary attention K/V roots, not recurrent caches. In-place KV fusion can bind the full allocation while exposing a live prefix internally. KV-slice on/off is a runtime binding choice and does not require different compiled blobs; incompatible graph layouts/capacities can still do so.
|
||||
|
||||
## Stable MTP sampling
|
||||
|
||||
The [graph walkthrough](MTP-SAMPLING-GRAPH.md) describes the fixed sampling interface. The backend keeps pure top-k branches for all configured slots, packs runtime row indices, and uses scratch for inactive outputs. It does not change the original cgraph or execute inactive stateful sampler chains. Only the small sampling branches stay at configured capacity; the expensive draft layer processes active rows.
|
||||
|
||||
One compatible draft graph is shared across active subsets. Trunk and genuinely different graph stages/configurations remain separate. Prepare a compatible revision-15 cache once, then use cache-only mode. Exhaustive active-subset priming is no longer required, though first-use model import and runtime preparation still have a cost.
|
||||
|
||||
## Validation
|
||||
|
||||
Native OV tests cover shape/attribute serialization, reset, cycles, duplicates, overlapping writes, mixed precision, changing shapes, writable cache input rebinding, exclusive Reshapes, shared readers, host/device USM and export/import. Convolution is checked against a host reference; GDN uses an independent cache mapping around ordinary GDN. See `docs/dev/qwen35-llamacpp-recurrent-cache.md` and `recurrent_cache_update.cpp` in the OV checkout for contracts and tests.
|
||||
|
||||
## Debugging and source map
|
||||
|
||||
- `ggml-decoder.cpp`: dynamic dimensions, control inputs, output names and `cgraph_ov.txt` dump.
|
||||
- `utils.cpp`: compute-branch filtering, compiled reuse and input/output binding.
|
||||
- `openvino/recurrent_cache_fusion.cpp`: structural pattern matching and native Update op translation.
|
||||
- `openvino/translate_session.cpp`: runtime sampling aliases and model construction.
|
||||
- OV `recurrent_cache_update` implementation: writable-input validation, source staging and GPU computation.
|
||||
|
||||
`GGML_OPENVINO_DUMP_CGRAPH=1` dumps the raw graph before filtering. It includes COMPUTE/OUTPUT flags and byte offsets and overwrites `cgraph_ov.txt` on each call. Enable dumps only for diagnosis; use the captured examples for a static view. GPU profiles and OpenCL copy traces add overhead, and lack of an explicit copy call does not prove zero traffic inside a kernel.
|
||||
@@ -0,0 +1,221 @@
|
||||
# OpenVINO backend environment variables for development
|
||||
|
||||
The variables for users, and the rules for values, are in [docs/backend/OPENVINO.md](../../../../docs/backend/OPENVINO.md#runtime-configuration). This file lists the other `GGML_OPENVINO_*` variables. They are for debugging, comparisons and experiments, and they can change or be removed.
|
||||
|
||||
## Debug output
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_DUMP_CGRAPH` | all | Boolean | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. |
|
||||
| `GGML_OPENVINO_DUMP_IR` | all | Boolean | `0` | Write the OpenVINO IR files, with timestamps. |
|
||||
| `GGML_OPENVINO_DEBUG_INPUT` | all | Boolean | `0` | Print the input tensors. |
|
||||
| `GGML_OPENVINO_DEBUG_OUTPUT` | all | Boolean | `0` | Print the output tensors. |
|
||||
| `GGML_OPENVINO_DEBUG_NODE` | all | String | `not set` | Add the named graph nodes as outputs of the compiled model. Separate several names with commas. |
|
||||
| `GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS` | all | Boolean | `0` | Print the tensor address map once. |
|
||||
| `GGML_OPENVINO_LOG_UNSUPPORTED_OPS` | all | Boolean | `0` | Log each op that the backend does not support, with the tensor details and the reason. |
|
||||
| `GGML_OPENVINO_DISABLE_CACHE` | all | Boolean | `0` | Turn the in-process cache of compiled models and decoders off. |
|
||||
| `GGML_OPENVINO_FULL_GRAPH_KEY` | all | Boolean | `0` | Add the external inputs of a graph to the key of the in-process cache. By default the key is the node count and the names of the first and last nodes. Use it when two different graphs could share that key. |
|
||||
| `GGML_OPENVINO_LOG_SWA_LAYERS` | all | Boolean | `0` | Log the attention extent of each layer and which layers use sliding-window attention, on an `ov-swa:` line at the WARN level. |
|
||||
| `GGML_OPENVINO_NPUW_DUMP_SUBS` | NPU | String | `not set` | Dump NPUW subgraphs: `YES` (all), `MIN` (unique function bodies), `last`, or indices such as `0,2`. |
|
||||
| `GGML_OPENVINO_NPUW_DUMP_SUBS_DIR` | NPU | String | `not set` | Directory for the NPUW subgraph dumps. |
|
||||
| `GGML_OPENVINO_NPUW_DUMP_SUBS_ON_FAIL` | NPU | String | `not set` | Passed to OpenVINO unchanged as `NPUW_DUMP_SUBS_ON_FAIL`. |
|
||||
| `GGML_OPENVINO_NPUW_DUMP_IO` | NPU | String | `not set` | Passed to OpenVINO unchanged as `NPUW_DUMP_IO`. |
|
||||
|
||||
```bash
|
||||
# Dump the GGML graph and the OpenVINO IR of a run
|
||||
GGML_OPENVINO_DUMP_CGRAPH=1 GGML_OPENVINO_DUMP_IR=1 \
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512
|
||||
|
||||
# Log unsupported ops. The lines have the WARN level, which the default log verbosity shows (-lv 2 or higher).
|
||||
GGML_OPENVINO_LOG_UNSUPPORTED_OPS=1 ./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512
|
||||
|
||||
# NPUW dumps. They need an OpenVINO developer build (NPU_PLUGIN_DEVELOPER_BUILD).
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_NPUW_DUMP_SUBS=YES
|
||||
export GGML_OPENVINO_NPUW_DUMP_SUBS_DIR=/tmp/npuw_dump # optional
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512
|
||||
```
|
||||
|
||||
## Switches for default optimizations
|
||||
|
||||
Each variable turns off an optimization that is on by default, or sets its strength. Use them to find the cause of a wrong result or of a speed change.
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_KV_SLICE` | all | Boolean | `1` | Bind the KV cache, and size the KQ masks, at the attended length instead of the full context. CPU and GPU: a view over the cache. NPU: the prefill graph is compiled at the attended length, rounded up to a power of two, and replaced when that size changes. With `0`, `llama-bench -d` fails on the NPU. |
|
||||
| `GGML_OPENVINO_NPU_BLOB_PREBUILD` | NPU | Boolean | `1` | Decode attends the smallest `1024 * 2^k` rows that cover the next position, with one decode graph for each window. All windows are built, or read from the cache, together with the first prefill and decode pair. `0`: one decode graph attends the whole context. Use the same value in the run that writes the cache and in the runs that read it. |
|
||||
| `GGML_OPENVINO_NPU_FAST_MASK` | NPU | Boolean | `1` | Write the prefill attention mask directly into the input tensor, without an intermediate copy. The result is identical. |
|
||||
| `GGML_OPENVINO_NPU_L0_HOST_TENSORS` | NPU | Boolean | `1` | Keep the KV cache and the compute buffers in Level Zero host memory, so the plugin reads them without an extra copy. Falls back to plain host memory if the allocation is not available. |
|
||||
| `GGML_OPENVINO_GDN_Q_SCALE` | NPU | Float | `64` | Factor of the GDN q pre-scaling, which keeps the GDN output above the FP16 flush-to-zero floor of the NPU. `0` or a negative value turns it off. Values below `64` are raised to `64`. |
|
||||
| `GGML_OPENVINO_MOE_OP` | GPU | Boolean | `1` | `0` keeps the unfused GatherMatmul path for mixture-of-experts layers. |
|
||||
| `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK` | CPU, GPU | Boolean | `0` | By default the model reads the sequence count and the attended KV length from the shape of the KQ mask, so the GPU plugin can select its fused attention kernel. `1` passes them as separate inputs. |
|
||||
| `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | CPU, GPU | Boolean | `0` | Stateful execution only. By default the sequence axis of the KV state is moved so that new tokens are appended in place. `1` keeps the original layout. |
|
||||
| `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS` | GPU | Boolean | `0` | By default an output tensor in a GPU buffer is bound directly to the infer request when its shape and type match. `1` turns this off. |
|
||||
| `GGML_OPENVINO_DISABLE_FUSED_RECURRENCE` | GPU | Boolean | `0` | Stateless execution only, with an OpenVINO build newer than 2026.4.1 (2026.4.1 does not build the fusion). By default the recurrent-state cache updates (Qwen3.5, LFM2) are fused into the OpenVINO model. `1` keeps the separate copies. |
|
||||
| `GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN` | GPU | Boolean | `0` | By default the operands of an element-wise op get the same rank, which works around a GPU plugin defect when the op is fused into RMS norm (gemma-4 in stateful execution gave wrong output). `1` turns the workaround off. |
|
||||
|
||||
```bash
|
||||
# Find out whether an optimization causes a wrong result: turn it off and compare the output
|
||||
GGML_OPENVINO_DEVICE=NPU GGML_OPENVINO_NPU_FAST_MASK=0 \
|
||||
./build/ReleaseOV/bin/llama-completion -m model.gguf -c 512 -n 32 --temp 0 -no-cnv -p "The capital of France is"
|
||||
```
|
||||
|
||||
## Experimental options
|
||||
|
||||
These options are off by default. They are not validated for general use: some help on one model or device and give a wrong or slower result on another.
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_NPU_DECODE_UNFOLD` | NPU | Boolean | `0` | Set `NPUW_UNFOLD_IREQS=YES` on the decode graph only. An explicit `GGML_OPENVINO_NPUW_UNFOLD_IREQS` or `GGML_OPENVINO_NPU_CONFIG` value takes precedence. |
|
||||
| `GGML_OPENVINO_NPU_PARALLEL_COMPILE` | NPU | Boolean | `0` | Compile the prefill and decode graphs in parallel. The load is shorter and the host memory peak during compilation is higher. |
|
||||
| `GGML_OPENVINO_NPU_KV_ON_DEMAND` | NPU | Boolean | `0` | Keep the ggml KV buffer non-resident and bind storage for each layer that grows with the rows in use, so a large `-c` costs memory only for the context that is reached. K is stored head-major and V transposed, as OpenVINO GenAI does. `llama-perplexity` fails with it. |
|
||||
| `GGML_OPENVINO_NPU_REQUANT_POLICY` | NPU | String | `group-128` | Layout of requantized weights: `group-128` (`Q4_0_128`) or `channel-wise` (`Q4_0_C`, lower accuracy). |
|
||||
| `GGML_OPENVINO_GDN_DECOMPOSE_CHUNK` | all | Integer | `16` | Tokens in one chunk when `GGML_OPENVINO_GDN_DECOMPOSE` is on. A power of 2 from 2 to 128; other values use `16`. |
|
||||
| `GGML_OPENVINO_GDN_NATIVE_STATE` | NPU | Boolean | `0` | Keep the `GATED_DELTA_NET` state in the layout of the OpenVINO op instead of the GGML layout. Every `GATED_DELTA_NET` must then run on the OpenVINO backend; the load aborts if one would run on the CPU backend. Saved states and `GGML_OPENVINO_DEBUG_NODE` dumps hold the OpenVINO layout. `test-backend-ops` fails with it. |
|
||||
| `GGML_OPENVINO_FORCE_STATIC` | CPU, GPU | Boolean | `0` | Run the static-shape path of the NPU on the CPU or the GPU, to separate translation bugs from NPU plugin or driver problems. Cannot be used with `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY`. |
|
||||
| `GGML_OPENVINO_ENABLE_FALLBACK` | all | Boolean | `0` | Detect graphs that the scheduler splits between OpenVINO and another backend, and translate them as partial graphs, without the compiled model cache. By default every graph is treated as a complete model graph. Cannot be used with `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY`. |
|
||||
| `GGML_OPENVINO_NATIVE_SOFTPLUS` | all | Boolean | `0` | Translate `SOFTPLUS` to the OpenVINO `SoftPlus` op instead of the default `relu(x) + log(1 + exp(-abs(x)))`. |
|
||||
| `GGML_OPENVINO_KV_SCATTER_ELEMENTS` | all | Boolean | `0` | Use `ScatterElementsUpdate` instead of `ScatterUpdate` for single-row KV writes in stateless execution. |
|
||||
| `GGML_OPENVINO_ENABLE_KV_4D` | GPU | Boolean | `0` | Bind the KV cache as `[1, ctx, n_heads_kv, head_size]`, so the GPU plugin can update and read it in place. Needs an OpenVINO build newer than 2026.4.1; do not use it with 2026.4.1. |
|
||||
| `GGML_OPENVINO_MEMORY_OPTIMIZE` | all | Boolean | `0` | Turn on `GGML_OPENVINO_REDUCE_COMPILE_MEM` and, on GPU, `GGML_OPENVINO_RELEASE_WEIGHTS`, unless they are set. |
|
||||
| `GGML_OPENVINO_REDUCE_COMPILE_MEM` | all | Boolean | value of `GGML_OPENVINO_MEMORY_OPTIMIZE` | Create fewer weight nodes during compilation, to lower the host memory in use. |
|
||||
| `GGML_OPENVINO_RELEASE_WEIGHTS` | GPU | Boolean | value of `GGML_OPENVINO_MEMORY_OPTIMIZE` | Release the host weight buffers when the compiled model holds its own copy. Needs stable graph shapes; do not use it for workloads that compile again. |
|
||||
| `GGML_OPENVINO_SPILL_DIR` | all | String | `not set` | Directory for a disk-backed weight buffer, so its pages can be reclaimed under memory pressure. It must be on real storage, not on tmpfs (`/tmp` on many systems). It is not part of the key of the compiled model cache and is ignored in cache-only mode. |
|
||||
| `GGML_OPENVINO_SELF_CONTAINED_BLOB` | NPU | Boolean | `0` | Requires `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR`. Write the prefill and decode graphs as blobs with the weights inside; each blob takes about the size of the model on disk. Decode then attends the whole context. `GGML_OPENVINO_NPU_WEIGHTLESS_BANK` is the recommended cache layout for the NPU. |
|
||||
| `GGML_OPENVINO_NPU_SHARED_BANK` | NPU | Boolean | `0` | With `GGML_OPENVINO_SELF_CONTAINED_BLOB=1`: one weights bank for the prefill and decode blobs. With OpenVINO 2026.4.1 the blobs cannot be read back, so decode is compiled again. |
|
||||
| `GGML_OPENVINO_LAZY_HOST_THRESHOLD_MB` | NPU | Integer | `2048` | With `GGML_OPENVINO_SELF_CONTAINED_MMAP=1`: buffers of at least this size get demand-zero host memory instead of Level Zero host memory. |
|
||||
|
||||
```bash
|
||||
# NPU cache with the weights inside the blobs (experimental). Keep all other NPU settings the same in both runs.
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache
|
||||
export GGML_OPENVINO_SELF_CONTAINED_BLOB=1
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512 -lv 4 # first run: log shows "model cache WROTE"
|
||||
|
||||
export GGML_OPENVINO_SELF_CONTAINED_MMAP=1
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512 -lv 4 # later runs: log shows "model cache HIT"
|
||||
```
|
||||
|
||||
## Tips for Memory Optimization
|
||||
|
||||
With the default settings, the GPU and CPU plugins keep their own copy of the weights next to the backend's copy while a model compiles. On integrated GPUs both copies are in system RAM. The settings below lower that peak. They do not change the output.
|
||||
|
||||
```bash
|
||||
# GPU or CPU, large model: back the weight buffer with a file, so the kernel can drop its pages.
|
||||
# Use a directory on a local disk (not tmpfs) with free space of about the model size. Can lower prompt speed.
|
||||
export GGML_OPENVINO_SPILL_DIR=$HOME/ov_spill
|
||||
mkdir -p $GGML_OPENVINO_SPILL_DIR
|
||||
```
|
||||
|
||||
```bash
|
||||
# GPU or CPU, the same model started again and again: compile once, then load the compiled model only.
|
||||
export GGML_OPENVINO_DEVICE=GPU # or CPU
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=$HOME/ov_cache
|
||||
GGML_OPENVINO_SPILL_DIR=$HOME/ov_spill ./build/ReleaseOV/bin/llama-cli -m model.gguf # first run writes the cache
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf # later runs skip the weight upload
|
||||
```
|
||||
|
||||
```bash
|
||||
# GPU, one long-running model (llama-server): release the host copy of the weights after compilation.
|
||||
# One model per process; a graph that must compile again later stops the run.
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
export GGML_OPENVINO_MEMORY_OPTIMIZE=1
|
||||
```
|
||||
|
||||
```bash
|
||||
# CPU, MoE model: build fewer weight nodes during compilation.
|
||||
export GGML_OPENVINO_REDUCE_COMPILE_MEM=1
|
||||
```
|
||||
|
||||
```bash
|
||||
# NPU: weightless cache, then use the memory-mapped GGUF as the model buffer.
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=$HOME/ov_cache
|
||||
export GGML_OPENVINO_NPU_WEIGHTLESS_BANK=1
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512 # first run writes the cache
|
||||
export GGML_OPENVINO_SELF_CONTAINED_MMAP=1
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512 # later runs
|
||||
```
|
||||
|
||||
- A cache hit without `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1` shortens the load, but it uses about as much memory as a run without the cache.
|
||||
- MoE models are not written to the cache on CPU, so cache-only mode does not work for them on CPU.
|
||||
- `GGML_OPENVINO_SPILL_DIR` is ignored on Windows and in cache-only mode.
|
||||
- `-lm none` (no mmap) does not lower the memory in use, and it turns cache-only mode off.
|
||||
|
||||
## NPU plugin properties
|
||||
|
||||
The backend sets the properties below for the NPU. `GGML_OPENVINO_NPU_CONFIG` can change them and add others.
|
||||
|
||||
- **Backend default**: the value that the backend passes to OpenVINO when the user sets nothing.
|
||||
- **OpenVINO default**: the value that OpenVINO uses when nobody sets the property.
|
||||
- Values are those of OpenVINO, for example `YES` and `NO`.
|
||||
|
||||
| Property | Type | Backend default | OpenVINO default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `NPU_USE_NPUW` | Boolean | `YES` | `NO` | Use NPUW, the OpenVINO layer that splits a model into parts and compiles them for the NPU. |
|
||||
| `NPUW_DEVICES` | String | `NPU` | `NPU,CPU` | Devices that NPUW tries for each part, in order. With `NPU` only, no part runs on the CPU. |
|
||||
| `NPUW_FOLD` | Boolean | `YES` | `NO` | Compile the repeating blocks of the model once and reuse them. |
|
||||
| `NPUW_FUNCALL_FOR_ALL` | Boolean | `YES` | `NO` | Make every part of the model a function, not only the repeating blocks. |
|
||||
| `NPUW_FUNCALL_ASYNC` | Boolean | `YES` | `NO` | Run a function and the preparation of the next one as a pipeline. |
|
||||
| `NPUW_WEIGHTS_BANK` | String | `shared` | empty | Name of the weights bank. Graphs with the same name share one copy of the weights. |
|
||||
| `NPUW_DQ` | Boolean | `YES` | `NO` | Apply the dynamic quantization transformations in the plugin. |
|
||||
| `NPUW_DQ_FULL` | Boolean | `NO` | `YES` | Apply the full dynamic quantization pipeline in the plugin. |
|
||||
| `NPU_COMPILER_DYNAMIC_QUANTIZATION` | Boolean | `YES` | `NO` | Dynamic quantization in the NPU compiler. |
|
||||
| `NPU_COMPILER_TYPE` | String | `DRIVER` | `PREFER_PLUGIN` | The NPU compiler: `DRIVER` (in the NPU driver), `PLUGIN` (in OpenVINO) or `PREFER_PLUGIN` (`PLUGIN` when it is available, `DRIVER` if not). Models with `GATED_DELTA_NET` layers (Qwen3.5) compile with `PLUGIN` unless the user sets this property. |
|
||||
| `NPUW_CACHE_DIR` | String | value of `GGML_OPENVINO_CACHE_DIR` | empty | Directory for the compiled parts. |
|
||||
|
||||
Other properties that are useful with `GGML_OPENVINO_NPU_CONFIG`. The backend does not set them.
|
||||
|
||||
| Property | Type | Backend default | OpenVINO default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `COMPILATION_NUM_THREADS` | Integer | `not set` | number of CPU cores | Threads of the NPU compiler. A lower value lowers the host memory peak when a large model compiles. |
|
||||
| `NPUW_UNFOLD_IREQS` | Boolean | `not set` | `NO` | One infer request for each part, also for repeating blocks. |
|
||||
| `LOG_LEVEL` | String | `not set` | `LOG_ERROR` | Log level of the NPU plugin, for example `LOG_INFO`. |
|
||||
|
||||
```bash
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_NPU_CONFIG="NPU_COMPILER_TYPE=PLUGIN,COMPILATION_NUM_THREADS=1,LOG_LEVEL=LOG_INFO"
|
||||
```
|
||||
|
||||
```cpp
|
||||
// List the properties of the installed NPU plugin with their current values
|
||||
ov::Core core;
|
||||
for (const auto & name : core.get_property("NPU", ov::supported_properties)) {
|
||||
std::cout << name << " = " << core.get_property("NPU", name).as<std::string>() << "\n";
|
||||
}
|
||||
```
|
||||
|
||||
- All NPU properties: [OpenVINO NPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.html).
|
||||
- `NPUW_*` options, with type and default: `src/plugins/intel_npu/src/al/include/intel_npu/config/npuw_option_defs.inc` in the OpenVINO source.
|
||||
- Graphs without a KV cache compile without NPUW: the backend removes `NPU_USE_NPUW` and the `NPUW_*` keys for them.
|
||||
|
||||
## GPU and CPU plugin properties
|
||||
|
||||
The backend sets no GPU or CPU property, except `CACHE_DIR` and `CACHE_MODE=OPTIMIZE_SIZE` when `GGML_OPENVINO_CACHE_DIR` is set.
|
||||
|
||||
```bash
|
||||
# CPU in FP16, to compare with the GPU and the NPU, which compute in FP16
|
||||
GGML_OPENVINO_DEVICE=CPU GGML_OPENVINO_CPU_CONFIG="INFERENCE_PRECISION_HINT=f16" \
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512
|
||||
```
|
||||
|
||||
All properties: [OpenVINO GPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html), [OpenVINO CPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device.html).
|
||||
|
||||
## Older variable names
|
||||
|
||||
Each of these variables sets one NPU property. They still work. `GGML_OPENVINO_NPU_CONFIG` sets the same properties and takes precedence when both are set.
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_NPU_COMPILE_CONFIG` | NPU | String | `not set` | Sets `NPU_COMPILATION_MODE_PARAMS`. |
|
||||
| `GGML_OPENVINO_NPU_COMPILER_TYPE` | NPU | String | `not set` | Sets `NPU_COMPILER_TYPE`. |
|
||||
| `GGML_OPENVINO_NPUW_FUNCALL_FOR_ALL` | NPU | String | `not set` | Sets `NPUW_FUNCALL_FOR_ALL`. |
|
||||
| `GGML_OPENVINO_NPUW_UNFOLD_IREQS` | NPU | String | `not set` | Sets `NPUW_UNFOLD_IREQS`. |
|
||||
| `GGML_OPENVINO_COMPILATION_NUM_THREADS` | NPU | String | `not set` | Sets `COMPILATION_NUM_THREADS`. |
|
||||
|
||||
```bash
|
||||
# These two lines do the same. Use the first form.
|
||||
export GGML_OPENVINO_NPU_CONFIG="NPUW_UNFOLD_IREQS=YES"
|
||||
export GGML_OPENVINO_NPUW_UNFOLD_IREQS=YES
|
||||
```
|
||||
@@ -0,0 +1,8 @@
|
||||
# OpenVINO Qwen development documentation
|
||||
|
||||
The default supported Qwen3.5/Qwen3.6 GPU path is **dynamic stateless execution with in-place fused recurrent ops and stable MTP sampling**. Unified KV is the recommended multi-slot layout because it enables native in-place attention KV updates. No new inference switch is needed for the recurrent/sampling default.
|
||||
|
||||
- [Backend setup and cache usage](../README.md): runtime requirements, environment settings and restrictions.
|
||||
- [Implementation](OPENVINO-RECURRENT-CACHE.md): buffer ownership, fusion, rollback, fallback and source map.
|
||||
- [Multislot MTP graph](MTP-SAMPLING-GRAPH.md): why active-slot variants occurred and how stable translation works.
|
||||
- [Environment variables for development](OpenVINO-ENV-VARS.md): debug output, switches for default optimizations, experimental options, plugin properties and older variable names.
|
||||
@@ -0,0 +1,113 @@
|
||||
#include "npu_config.h"
|
||||
|
||||
#include "ggml-impl.h"
|
||||
#include "ggml-openvino/ggml-decoder.h"
|
||||
#include "ggml-openvino/ggml-openvino-extra.h"
|
||||
|
||||
#include <cstring>
|
||||
#include <openvino/runtime/properties.hpp>
|
||||
|
||||
// The environment does not set the NPU compiler type. Set by ggml_openvino_npu_build_compile_config().
|
||||
static bool npu_compiler_type_is_default = false;
|
||||
|
||||
ov::AnyMap ggml_openvino_npu_build_compile_config(const char * cache_dir) {
|
||||
ov::AnyMap compile_config = {
|
||||
{"NPU_COMPILER_DYNAMIC_QUANTIZATION", "YES" },
|
||||
{"NPU_USE_NPUW", "YES" },
|
||||
{"NPUW_DEVICES", "NPU" },
|
||||
{"NPUW_FOLD", "YES" },
|
||||
{"NPUW_WEIGHTS_BANK", "shared"},
|
||||
{"NPUW_FUNCALL_FOR_ALL", "YES" },
|
||||
{"NPUW_FUNCALL_ASYNC", "YES" },
|
||||
{"NPUW_DQ", "YES" },
|
||||
{"NPUW_DQ_FULL", "NO" },
|
||||
};
|
||||
if (cache_dir && strlen(cache_dir) > 0) {
|
||||
// Keep the default cache mode: CACHE_MODE=OPTIMIZE_SIZE makes the NPU plugin compile the NPUW submodels with
|
||||
// weights separation, which fails for them.
|
||||
compile_config["NPUW_CACHE_DIR"] = cache_dir;
|
||||
}
|
||||
// Copy an optional string env var into the compile config only when it is set and non-empty.
|
||||
auto set_compile_option_from_env = [&](const char * env_var, const char * config_key) {
|
||||
const char * value = ggml_openvino_getenv_str(env_var);
|
||||
if (value && strlen(value) > 0) {
|
||||
compile_config[config_key] = value;
|
||||
}
|
||||
};
|
||||
// Legacy alias for GGML_OPENVINO_NPU_CONFIG=NPU_COMPILATION_MODE_PARAMS=...
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPU_COMPILE_CONFIG", "NPU_COMPILATION_MODE_PARAMS");
|
||||
// PLUGIN | DRIVER | PREFER_PLUGIN. The in-plugin compiler and the driver compiler
|
||||
// can differ substantially in generated code quality for the same op. On the Intel
|
||||
// NPU the driver compiler emits markedly faster prefill kernels (~1940 vs ~1540 t/s
|
||||
// pp1024 on phi-4-mini), matching OpenVINO GenAI, so default to DRIVER below.
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPU_COMPILER_TYPE", "NPU_COMPILER_TYPE");
|
||||
// NPUW_FUNCALL_FOR_ALL=YES hangs the NPU (device lost via TDR) for context
|
||||
// lengths >= ~786 on the 2026.3 in-plugin compiler. Allow turning it off.
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPUW_FUNCALL_FOR_ALL", "NPUW_FUNCALL_FOR_ALL");
|
||||
// Unfolds function calls into separate infer requests, trading memory for the
|
||||
// per-call dispatch overhead that repeated funcalls otherwise pay.
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPUW_UNFOLD_IREQS", "NPUW_UNFOLD_IREQS");
|
||||
// The compiler runs one llvm worker per core by default; each carries its own
|
||||
// working set, so large graphs can exhaust host RAM. Capping the workers trades
|
||||
// compile time for peak memory.
|
||||
set_compile_option_from_env("GGML_OPENVINO_COMPILATION_NUM_THREADS", "COMPILATION_NUM_THREADS");
|
||||
// NPUW profiling dumps; need an OpenVINO build with NPU_PLUGIN_DEVELOPER_BUILD.
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPUW_DUMP_SUBS", "NPUW_DUMP_SUBS");
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPUW_DUMP_SUBS_DIR", "NPUW_DUMP_SUBS_DIR");
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPUW_DUMP_SUBS_ON_FAIL", "NPUW_DUMP_SUBS_ON_FAIL");
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPUW_DUMP_IO", "NPUW_DUMP_IO");
|
||||
// Comma-separated KEY=VALUE pairs appended last, so they override anything above.
|
||||
// Escape hatch for bisecting plugin options without a rebuild.
|
||||
ggml_openvino_apply_config_overrides(compile_config, ggml_openvino_getenv_str("GGML_OPENVINO_NPU_CONFIG"), "NPU");
|
||||
npu_compiler_type_is_default = compile_config.find("NPU_COMPILER_TYPE") == compile_config.end();
|
||||
if (npu_compiler_type_is_default) {
|
||||
compile_config["NPU_COMPILER_TYPE"] = "DRIVER";
|
||||
}
|
||||
return compile_config;
|
||||
}
|
||||
|
||||
bool ggml_openvino_npu_gdn_uses_plugin_compiler() {
|
||||
return ggml_openvino_is_npu() && npu_compiler_type_is_default;
|
||||
}
|
||||
|
||||
ov::AnyMap ggml_openvino_npu_gdn_config(const ov::AnyMap & base) {
|
||||
ov::AnyMap gdn_config = base;
|
||||
if (ggml_openvino_npu_gdn_uses_plugin_compiler()) {
|
||||
gdn_config["NPU_COMPILER_TYPE"] = "PLUGIN";
|
||||
}
|
||||
return gdn_config;
|
||||
}
|
||||
|
||||
ov::AnyMap ggml_openvino_npu_without_npuw(const ov::AnyMap & config) {
|
||||
ov::AnyMap out;
|
||||
for (const auto & kv : config) {
|
||||
if (kv.first.rfind("NPUW", 0) == 0 || kv.first == "NPU_USE_NPUW") {
|
||||
continue;
|
||||
}
|
||||
out.insert(kv);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
ov::AnyMap ggml_openvino_npu_prefill_config(const ov::AnyMap & base, const GgmlOvDecoder & decoder) {
|
||||
ov::AnyMap prefill_config = base;
|
||||
if (ggml_openvino_npu_hfa_prefill_enabled()) {
|
||||
prefill_config["NPUW_ATTN_HISTORY_SIZE_INPUT"] = decoder.get_hfa_history_input_name();
|
||||
prefill_config["NPUW_ONLINE_PIPELINE"] = "REP";
|
||||
prefill_config["NPUW_ONLINE_ISOLATE"] = "ATTN";
|
||||
prefill_config["NPUW_ONLINE_KEEP_BLOCK_SIZE"] = "4";
|
||||
prefill_config["NPUW_UNFOLD_IREQS"] = "NO";
|
||||
prefill_config["NPUW_ATTN"] = "HFA";
|
||||
prefill_config["NPUW_ATTN_HFA_FUSED"] = "YES";
|
||||
}
|
||||
return prefill_config;
|
||||
}
|
||||
|
||||
ov::AnyMap ggml_openvino_npu_decode_config(const ov::AnyMap & base, const std::string & device) {
|
||||
ov::AnyMap decode_config = base;
|
||||
if (device == "NPU" && ggml_openvino_npu_decode_unfold_enabled() &&
|
||||
decode_config.find("NPUW_UNFOLD_IREQS") == decode_config.end()) {
|
||||
decode_config["NPUW_UNFOLD_IREQS"] = "YES";
|
||||
}
|
||||
return decode_config;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
#pragma once
|
||||
|
||||
// NPU (NPUW) compile configuration. Everything that decides which OpenVINO properties an NPU graph is
|
||||
// compiled with lives here, so "what does the NPU compile with" is answered in one place:
|
||||
// * the backend-wide base config (NPUW defaults, named env aliases, GGML_OPENVINO_NPU_CONFIG overrides)
|
||||
// * the per-graph variants of it (cacheless graphs, GATED_DELTA_NET graphs, static prefill, static decode)
|
||||
|
||||
#include <openvino/core/any.hpp>
|
||||
#include <string>
|
||||
|
||||
class GgmlOvDecoder;
|
||||
|
||||
// Base config for every NPU graph, built once at backend init. GGML_OPENVINO_NPU_CONFIG ("K=V,...") is
|
||||
// applied last, so it overrides the defaults and the named aliases.
|
||||
ov::AnyMap ggml_openvino_npu_build_compile_config(const char * cache_dir);
|
||||
|
||||
// Cacheless (encoder) graphs compile on the plain NPU plugin: drop every NPUW key.
|
||||
ov::AnyMap ggml_openvino_npu_without_npuw(const ov::AnyMap & config);
|
||||
|
||||
// Graphs with GATED_DELTA_NET (Qwen3.5). The driver compiler gives wrong output for them and the compiler bundled
|
||||
// with OpenVINO does not, so they compile with NPU_COMPILER_TYPE=PLUGIN unless the environment sets the compiler
|
||||
// type (GGML_OPENVINO_NPU_COMPILER_TYPE, or NPU_COMPILER_TYPE in GGML_OPENVINO_NPU_CONFIG).
|
||||
ov::AnyMap ggml_openvino_npu_gdn_config(const ov::AnyMap & base);
|
||||
|
||||
// True when ggml_openvino_npu_gdn_config() switches the compiler. Part of the model cache key.
|
||||
bool ggml_openvino_npu_gdn_uses_plugin_compiler();
|
||||
|
||||
// Static prefill graph. With GGML_OPENVINO_NPU_HFA_PREFILL, switches NPUW to the HFA attention pipeline.
|
||||
ov::AnyMap ggml_openvino_npu_prefill_config(const ov::AnyMap & base, const GgmlOvDecoder & decoder);
|
||||
|
||||
// Static decode graph. With GGML_OPENVINO_NPU_DECODE_UNFOLD, unfolds NPUW function calls into separate infer
|
||||
// requests, unless NPUW_UNFOLD_IREQS is already set.
|
||||
ov::AnyMap ggml_openvino_npu_decode_config(const ov::AnyMap & base, const std::string & device);
|
||||
@@ -0,0 +1,50 @@
|
||||
#include "npu_quant_policy.h"
|
||||
|
||||
#include "ggml-impl.h"
|
||||
#include "ggml.h"
|
||||
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
ExtraQuantType ggml_openvino_get_npu_requant_type() {
|
||||
const std::string policy = ggml_openvino_getenv_str("GGML_OPENVINO_NPU_REQUANT_POLICY", "group-128");
|
||||
if (policy == "group-128") {
|
||||
return ExtraQuantType::Q4_0_128;
|
||||
}
|
||||
if (policy == "channel-wise") {
|
||||
return ExtraQuantType::Q4_0_C;
|
||||
}
|
||||
GGML_ABORT("Unknown GGML_OPENVINO_NPU_REQUANT_POLICY: %s", policy.c_str());
|
||||
}
|
||||
|
||||
std::optional<ExtraQuantType> ggml_openvino_npu_token_embd_requant_type(const ggml_tensor * tensor) {
|
||||
// A Q6_K embedding table is stored as q8_0_c by default on the NPU. The int4 layouts only pay off for a tied
|
||||
// embedding, where the table is also the lm_head weight and GET_ROWS gathers the packed rows
|
||||
// before dequantizing them (see gather_compressed_rows).
|
||||
if (tensor->type != GGML_TYPE_Q6_K) {
|
||||
return std::nullopt;
|
||||
}
|
||||
const std::string quant = ggml_openvino_getenv_str("GGML_OPENVINO_NPU_TOKEN_EMBD_QUANT", "q8_0_c");
|
||||
ExtraQuantType type;
|
||||
if (quant == "f16") {
|
||||
type = ExtraQuantType::F16;
|
||||
} else if (quant == "q8_0_c") {
|
||||
type = ExtraQuantType::Q8_0_C;
|
||||
} else if (quant == "q4_0_128") {
|
||||
type = ExtraQuantType::Q4_0_128;
|
||||
} else if (quant == "q4_0_c") {
|
||||
type = ExtraQuantType::Q4_0_C;
|
||||
} else {
|
||||
GGML_ABORT("Unknown GGML_OPENVINO_NPU_TOKEN_EMBD_QUANT: %s", quant.c_str());
|
||||
}
|
||||
static std::once_flag logged;
|
||||
std::call_once(logged, [&] {
|
||||
GGML_LOG_INFO("ggml-openvino: NPU token_embd.weight requant type: %s\n", quant.c_str());
|
||||
});
|
||||
return type;
|
||||
}
|
||||
|
||||
std::optional<ExtraQuantType> ggml_openvino_npu_weight_requant_type(const ggml_tensor * tensor) {
|
||||
GGML_UNUSED(tensor);
|
||||
return ggml_openvino_get_npu_requant_type();
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// NPU weight requantization rules. ggml_openvino_get_requant_type() (ggml-openvino-extra.cpp) delegates
|
||||
// the NPU-specific decisions here.
|
||||
|
||||
#include "ggml-openvino/ggml-openvino-extra.h"
|
||||
|
||||
#include <optional>
|
||||
|
||||
struct ggml_tensor;
|
||||
|
||||
// Layout selected by GGML_OPENVINO_NPU_REQUANT_POLICY (group-128 | channel-wise).
|
||||
ExtraQuantType ggml_openvino_get_npu_requant_type();
|
||||
|
||||
// token_embd.weight on NPU. nullopt: no NPU-specific rule, use the common default.
|
||||
std::optional<ExtraQuantType> ggml_openvino_npu_token_embd_requant_type(const ggml_tensor * tensor);
|
||||
|
||||
// Any other weight on NPU.
|
||||
std::optional<ExtraQuantType> ggml_openvino_npu_weight_requant_type(const ggml_tensor * tensor);
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user