mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-19 09:17:40 -06:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c7468107ef | ||
|
|
ac830a9cbb | ||
|
|
8af68d486a | ||
|
|
f8caa5ea30 | ||
|
|
4db8b7638f | ||
|
|
01592dc58b | ||
|
|
8bf2132c3f | ||
|
|
5812201a81 | ||
|
|
41d8297fe4 | ||
|
|
51a77c4f24 | ||
|
|
76f186df71 | ||
|
|
95b09b62be | ||
|
|
b5dbca1e44 | ||
|
|
80437488a4 | ||
|
|
39039aa27f | ||
|
|
7a4117eef9 | ||
|
|
573aa89de6 | ||
|
|
ecafc67145 | ||
|
|
d4f5c26049 | ||
|
|
08c252e221 | ||
|
|
e4b620af19 | ||
|
|
b001d7bb22 | ||
|
|
0a76757469 | ||
|
|
b044890408 | ||
|
|
78e05298f0 | ||
|
|
785f6bac00 | ||
|
|
edec462cc0 | ||
|
|
4483b2b987 | ||
|
|
bafa0420c1 | ||
|
|
2cbfc3cf34 | ||
|
|
6f311d7ad6 | ||
|
|
ff6b8d1761 | ||
|
|
98d67c8d53 | ||
|
|
387395aa8d | ||
|
|
a693a5f66f | ||
|
|
b069e526de | ||
|
|
ffc87c3a79 | ||
|
|
10f173b253 | ||
|
|
97069ff6c4 | ||
|
|
bba8408821 | ||
|
|
48a32194ee | ||
|
|
6da36139e8 | ||
|
|
61f182fcde | ||
|
|
03ec5b4326 | ||
|
|
7144124a5b | ||
|
|
43bd7a2a42 | ||
|
|
65758d01f5 | ||
|
|
a300a9c334 | ||
|
+4 |
4577932fab | ||
|
|
398f00d657 | ||
|
|
9e93cd8001 | ||
|
|
aaa807c568 |
@@ -3,11 +3,11 @@ ARG OPENVINO_VERSION_FULL=2026.4.1.22982.07f9c262b05
|
||||
ARG UBUNTU_VERSION=24.04
|
||||
|
||||
# Intel GPU driver versions. https://github.com/intel/compute-runtime/releases
|
||||
ARG IGC_VERSION=v2.41.5
|
||||
ARG IGC_VERSION_FULL=2_2.41.5+22716
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.35.39758.10
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.35.39758.10-0
|
||||
ARG IGDGMM_VERSION=22.10.0
|
||||
ARG IGC_VERSION=v2.42.6
|
||||
ARG IGC_VERSION_FULL=2_2.42.6+23070
|
||||
ARG COMPUTE_RUNTIME_VERSION=26.40.40196.8
|
||||
ARG COMPUTE_RUNTIME_VERSION_FULL=26.40.40196.8-0
|
||||
ARG IGDGMM_VERSION=22.10.2
|
||||
|
||||
# Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases
|
||||
ARG NPU_DRIVER_VERSION=v1.38.0
|
||||
|
||||
@@ -25,7 +25,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-11.7.91-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-11.7.101-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-11.7.91-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-11.7.91-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvml_dev-windows-x86_64-11.7.91-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_cudart-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvcc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvrtc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
@@ -54,7 +56,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-12.4.127-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvml_dev-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_cudart-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvcc-windows-x86_64-12.4.131-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvrtc-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
@@ -85,7 +89,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.1.80-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.1.68-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-13.1.78-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.1.68-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvml_dev-windows-x86_64-13.1.68-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_crt-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_cudart-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvcc-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
@@ -116,7 +122,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.5.37-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
@@ -143,7 +151,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.10.0.20-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-arm64/cuda_nvml_dev-windows-arm64-13.5.37-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-arm64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
|
||||
@@ -140,6 +140,11 @@ jobs:
|
||||
source .venv/Scripts/activate
|
||||
pip install -r requirements/requirements-convert_hf_to_gguf.txt
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
shell: cmd
|
||||
|
||||
@@ -187,6 +187,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
|
||||
@@ -61,6 +61,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Install OpenCL Headers and Libs
|
||||
id: install_opencl
|
||||
run: |
|
||||
|
||||
@@ -213,6 +213,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
|
||||
@@ -900,6 +900,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
@@ -1146,6 +1151,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
# TODO: these jobs need to use llvm toolchain in order to utilize the ccache
|
||||
#- name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
|
||||
@@ -175,6 +175,11 @@ jobs:
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
shell: cmd
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
name: UI Chat Publish
|
||||
|
||||
# Publishes Llama Chat to npm: @ggml-org/llama-chat (ui/apps/chat, the app and its CLI) and
|
||||
# llama-chat (ui/apps/chat/alias, the short name for npx), always at the same version.
|
||||
#
|
||||
# The version in ui/apps/chat/package.json is the release, set by hand, as for @ggml-org/llama-ui
|
||||
# in ui-lib-publish.yml. A commit that reaches master with a version npm does not have builds the
|
||||
# app and publishes both packages; any other commit publishes nothing. Bump both package.json
|
||||
# files in the PR that should ship.
|
||||
#
|
||||
# Auth: no npm token is stored here. The job exchanges its OIDC identity for one (npm trusted
|
||||
# publishing) and signs the tarballs with provenance. One-time setup, before this can publish: on
|
||||
# each of the two packages, Publishing settings, Trusted Publisher, GitHub Actions, with
|
||||
# organization ggml-org, repository llama.cpp, workflow file ui-chat-publish.yml and Environment
|
||||
# left blank. Until then the publish step fails on master; re-running the job publishes it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: 'Build and pack, but do not publish'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-chat-publish.yml',
|
||||
'ui/apps/chat/package.json',
|
||||
'ui/apps/chat/alias/package.json'
|
||||
]
|
||||
# a dry run for PRs that change what goes into the packages
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ui-chat-publish.yml',
|
||||
'ui/apps/chat/package.json',
|
||||
'ui/apps/chat/alias/**',
|
||||
'ui/apps/chat/bin/**',
|
||||
'ui/apps/chat/cli/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
# never cancel a run mid-publish: a version half-published cannot be taken back
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
APP_DIR: ui/apps/chat
|
||||
ALIAS_DIR: ui/apps/chat/alias
|
||||
VERSION_CHECK: ui/lib/ui/scripts/npm-version-check.mjs
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish @ggml-org/llama-chat and llama-chat
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: ubuntu-26.04-arm
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Check npm can publish with an OIDC token
|
||||
run: |
|
||||
node -e '
|
||||
const v = process.argv[1];
|
||||
const [major, minor] = v.split(".").map(Number);
|
||||
if (major < 11 || (major === 11 && minor < 5)) {
|
||||
console.error(`npm ${v} cannot publish with trusted publishing: 11.5.1 or newer is needed`);
|
||||
process.exit(1);
|
||||
}
|
||||
' "$(npm -v)"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Build the app
|
||||
# the same build.json version as the UI that llama-server embeds, for the app's update check.
|
||||
# This folder's .npmrc sets ignore-scripts, so npm never builds it: this step does.
|
||||
run: LLAMA_BUILD_NUMBER="b$(git rev-list --count HEAD)" npm run build
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Test the CLI and the alias version
|
||||
run: npx vitest --project=unit --run tests/unit/cli.test.ts
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Check what goes into the package
|
||||
run: |
|
||||
npm pack --dry-run --json > "$RUNNER_TEMP/pack.json"
|
||||
node -e '
|
||||
const [{ files }] = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
|
||||
const paths = files.map((file) => file.path);
|
||||
const maps = paths.filter((path) => path.endsWith(".map"));
|
||||
if (!paths.includes("dist/index.html")) {
|
||||
console.error("dist/index.html is not in the package: nothing would be served");
|
||||
process.exit(1);
|
||||
}
|
||||
if (maps.length > 0) {
|
||||
console.error(`source maps in the package: ${maps.join(", ")}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`${paths.length} files, no source maps`);
|
||||
' "$RUNNER_TEMP/pack.json"
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Compare @ggml-org/llama-chat with npm
|
||||
id: app
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node "$VERSION_CHECK" "$APP_DIR"
|
||||
|
||||
- name: Compare llama-chat with npm
|
||||
id: alias
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node "$VERSION_CHECK" "$ALIAS_DIR"
|
||||
|
||||
# master only: npm's trusted publisher matches the repository and the workflow file, not the
|
||||
# branch, so this condition is what keeps a manual run on a PR branch from publishing.
|
||||
# The app goes first, since the alias depends on it at the same version.
|
||||
- name: Publish @ggml-org/llama-chat
|
||||
if: github.ref == 'refs/heads/master' && steps.app.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
TAG: ${{ steps.app.outputs.tag }}
|
||||
run: npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Publish llama-chat
|
||||
if: github.ref == 'refs/heads/master' && steps.alias.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
TAG: ${{ steps.alias.outputs.tag }}
|
||||
run: npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.ALIAS_DIR }}
|
||||
|
||||
- name: Report
|
||||
if: always()
|
||||
# values reach the shell through env, so a name from a PR's package.json cannot inject here
|
||||
env:
|
||||
APP: ${{ steps.app.outputs.name }}@${{ steps.app.outputs.version }}
|
||||
APP_VERDICT: ${{ steps.app.outputs.verdict || 'not reached' }}
|
||||
APP_NOTE: ${{ steps.app.outputs.note }}
|
||||
ALIAS: ${{ steps.alias.outputs.name }}@${{ steps.alias.outputs.version }}
|
||||
ALIAS_VERDICT: ${{ steps.alias.outputs.verdict || 'not reached' }}
|
||||
ALIAS_NOTE: ${{ steps.alias.outputs.note }}
|
||||
DRY: ${{ inputs.dry_run }}
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
{
|
||||
echo "### Llama Chat on npm"
|
||||
echo
|
||||
echo "| package | verdict | note |"
|
||||
echo "| --- | --- | --- |"
|
||||
echo "| \`$APP\` | \`$APP_VERDICT\` | $APP_NOTE |"
|
||||
echo "| \`$ALIAS\` | \`$ALIAS_VERDICT\` | $ALIAS_NOTE |"
|
||||
echo
|
||||
if [ "$EVENT" = "pull_request" ] || [ "$DRY" = "true" ]; then echo "Dry run: nothing was published."; fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Mind the verdict
|
||||
# a version going backwards is a mistake; an unchanged version is a package.json edit that
|
||||
# is not a release, such as a dependency update
|
||||
if: always() && github.event_name != 'pull_request'
|
||||
env:
|
||||
VERDICTS: ${{ steps.app.outputs.verdict }} ${{ steps.alias.outputs.verdict }}
|
||||
run: |
|
||||
case " $VERDICTS " in
|
||||
*" stale "*)
|
||||
echo "::error::a version in package.json is older than the one on npm"
|
||||
exit 1 ;;
|
||||
esac
|
||||
@@ -0,0 +1,84 @@
|
||||
name: UI Storybook
|
||||
|
||||
# Builds Llama UI and Llama Chat's Storybook (ui/apps/chat) and publishes it to its Hugging Face Space
|
||||
# whenever the UI changes on master, so the Space always shows the latest build.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-storybook.yml',
|
||||
'ui/apps/chat/**',
|
||||
'ui/lib/ui/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# only the newest build matters: a newer push replaces a deploy still in progress
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
name: Deploy to Hugging Face Spaces
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: hf-jobs-cpu-upgrade:ubuntu26_04
|
||||
|
||||
env:
|
||||
HF_SPACE: ${{ vars.HF_SPACE_UI_STORYBOOK || 'ggml-org/llama-ui-storybook' }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Build Storybook
|
||||
run: npm run build-storybook
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Write the Space card
|
||||
run: |
|
||||
cat > ui/apps/chat/storybook-static/README.md << EOF
|
||||
---
|
||||
title: Llama UI Storybook
|
||||
emoji: 🦙
|
||||
colorFrom: gray
|
||||
colorTo: gray
|
||||
sdk: static
|
||||
pinned: false
|
||||
---
|
||||
|
||||
The Storybook of Llama UI and Llama Chat, built from [ggml-org/llama.cpp](https://github.com/ggml-org/llama.cpp) (\`ui/apps/chat\`) at ${GITHUB_SHA}.
|
||||
EOF
|
||||
|
||||
- name: Install Hugging Face Hub CLI
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y python3 python3-venv
|
||||
python3 -m venv .venv-hf
|
||||
.venv-hf/bin/pip install -U huggingface_hub==1.28.0
|
||||
|
||||
- name: Upload to the Space
|
||||
env:
|
||||
# a token with write access to the Space; HF_TOKEN_UI_STATIC_OUTPUT if that one has it
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_UI_STORYBOOK || secrets.HF_TOKEN_UI_STATIC_OUTPUT }}
|
||||
run: |
|
||||
# --delete '*' removes files the new build no longer has; the Space's .gitattributes is always kept
|
||||
.venv-hf/bin/hf upload "$HF_SPACE" ui/apps/chat/storybook-static . \
|
||||
--repo-type space \
|
||||
--delete '*' \
|
||||
--commit-message "Storybook from ggml-org/llama.cpp@${GITHUB_SHA::10}"
|
||||
@@ -93,6 +93,11 @@ jobs:
|
||||
run: npm run lint
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Check DESIGN.md is current with the tokens
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
run: npm run check:design
|
||||
working-directory: ui/apps/chat
|
||||
|
||||
- name: Install Playwright browsers
|
||||
id: playwright
|
||||
if: ${{ always() && steps.setup.conclusion == 'success' }}
|
||||
@@ -132,6 +137,15 @@ jobs:
|
||||
run: npm run check
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Install Playwright browsers
|
||||
run: npx playwright install --with-deps chromium
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
# the unit tests and the Storybook stories (ui/lib/ui/tests/stories), which run as tests in Chromium
|
||||
- name: Run tests
|
||||
run: npm test
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
- name: Build the package
|
||||
run: npm run prepack
|
||||
working-directory: ui/lib/ui
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<b>LLM inference in C/C++</b>
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v1)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v1)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=b)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
|
||||
@@ -1313,6 +1313,11 @@ common_init_result::common_init_result(common_params & params, bool model_only)
|
||||
LOG_INF("%s", "decision model reads the embeddings output, enabling embedding mode\n");
|
||||
}
|
||||
|
||||
// 0 is n_batch
|
||||
if ((decision_type == COMMON_DECISION_TYPE_LFM3_D1 || decision_type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) && cparams.n_outputs_max != 0) {
|
||||
cparams.n_outputs_max = std::max(cparams.n_outputs_max, (uint32_t) COMMON_DECISION_LFM3_D1_SET_SIZE);
|
||||
}
|
||||
|
||||
if (llama_model_classifier_free_guidance(model)) {
|
||||
params.kv_unified = true;
|
||||
cparams.kv_unified = true;
|
||||
@@ -2334,6 +2339,9 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
|
||||
if (t.decision_order != 0) {
|
||||
llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order);
|
||||
}
|
||||
if (t.decision_group != 0 && !llama_batch_ext_set_decision_group(res, idx, t.decision_group)) {
|
||||
GGML_ABORT("%s: failed to set the decision group of the entry at index %d\n", __func__, i);
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
|
||||
@@ -1002,6 +1002,9 @@ enum common_decision_type {
|
||||
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
|
||||
};
|
||||
|
||||
// lfm3-d1: max questions packed in one prompt
|
||||
#define COMMON_DECISION_LFM3_D1_SET_SIZE 16
|
||||
|
||||
common_decision_type common_get_decision_type(const struct llama_model * model);
|
||||
|
||||
// metadata of a GGUF file, read without loading the model
|
||||
@@ -1117,6 +1120,7 @@ struct common_batch {
|
||||
llama_embd state; // non-owning view of the data passed to set_embd_state(), data == NULL if none
|
||||
std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
|
||||
int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
|
||||
int32_t decision_group = 0; // see llama_batch_ext_set_decision_group()
|
||||
int32_t embd_row = -1; // see add_embd_row(), -1 if none
|
||||
int32_t state_row = -1; // see set_embd_state_row(), -1 if none
|
||||
};
|
||||
|
||||
@@ -287,6 +287,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"TinyVocaModel": "tiny_voca",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"VoxCPM2Model": "voxcpm",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
@@ -340,6 +341,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"XverseForCausalLM": "xverse",
|
||||
"YoutuForCausalLM": "deepseek",
|
||||
"YoutuVLForConditionalGeneration": "deepseek",
|
||||
"ZayaForCausalLM": "zaya1",
|
||||
"fish_qwen3_omni": "fish_s2",
|
||||
"modeling_grove_moe.GroveMoeForCausalLM": "grovemoe",
|
||||
"modeling_sarvam_moe.SarvamMoEForCausalLM": "bailingmoe",
|
||||
@@ -366,6 +368,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"FishOmniModel": "drama3",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"VoxCPM2Model": "voxcpm",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Gemma3ForConditionalGeneration": "gemma",
|
||||
"Gemma3nForConditionalGeneration": "gemma",
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, MmprojModel, TextModel, LazyTorchTensor, gguf, logger
|
||||
from .drama3 import fold_snake
|
||||
from .minicpm import MiniCPMModel
|
||||
|
||||
# sample rate of the AudioVAE decoder output, picks the row of the sample-rate conditioning
|
||||
VAE_OUT_SAMPLE_RATE = 48000
|
||||
|
||||
|
||||
def _load_config(dir_model: Path) -> dict[str, Any] | None:
|
||||
path = dir_model / "config.json"
|
||||
if not path.is_file():
|
||||
return None
|
||||
with open(path, encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
return config if config.get("architecture") == "voxcpm2" else None
|
||||
|
||||
|
||||
def _is_voxcpm2_checkpoint(dir_model: Path) -> bool:
|
||||
return _load_config(dir_model) is not None
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_voxcpm2_checkpoint)
|
||||
def _load_voxcpm2_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected VoxCPM2 checkpoint")
|
||||
config = _load_config(dir_model)
|
||||
assert config is not None
|
||||
if config["lm_config"].get("use_mup"):
|
||||
raise NotImplementedError("VoxCPM2: use_mup = true is not supported")
|
||||
if config["dit_config"].get("mean_mode"):
|
||||
raise NotImplementedError("VoxCPM2: dit mean_mode = true is not supported")
|
||||
if not config.get("residual_lm_no_rope"):
|
||||
raise NotImplementedError("VoxCPM2: residual LM with RoPE is not supported")
|
||||
for key in ("hidden_dim", "ffn_dim", "num_heads", "num_layers", "kv_channels"):
|
||||
if config["encoder_config"][key] != config["dit_config"][key]:
|
||||
raise NotImplementedError(f"VoxCPM2: LocEnc and LocDiT differ in {key}")
|
||||
return {**config, "architectures": ["VoxCPM2Model"], "text_config": config["lm_config"], "audio_config": config["dit_config"]}
|
||||
|
||||
|
||||
@ModelBase.register("VoxCPM2Model")
|
||||
@ModelBase.example("openbmb/VoxCPM2")
|
||||
class VoxCPM2TextModel(MiniCPMModel):
|
||||
model_arch = gguf.MODEL_ARCH.MINICPM
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
# the text-semantic LM is a plain MiniCPM, the rest goes to the mmproj
|
||||
if not name.startswith("base_lm."):
|
||||
return None
|
||||
return super().filter_tensors(("model." + name.removeprefix("base_lm."), gen))
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
TextModel.set_gguf_parameters(self)
|
||||
# use_mup = false: no embedding, residual or logit scaling
|
||||
self.gguf_writer.add_embedding_scale(1.0)
|
||||
self.gguf_writer.add_residual_scale(1.0)
|
||||
self.gguf_writer.add_logit_scale(1.0)
|
||||
|
||||
def set_vocab(self):
|
||||
# the checkpoint has no tokenizer.model and its tokenizer class is remote code, so read tokenizer.json
|
||||
# SPM-style BPE: a merged token scores minus its merge rank, so the SPM tokenizer merges in the BPE order
|
||||
with open(self.dir_model / "tokenizer.json", encoding="utf-8") as f:
|
||||
tok = json.load(f)
|
||||
with open(self.dir_model / "tokenizer_config.json", encoding="utf-8") as f:
|
||||
tok_cfg = json.load(f)
|
||||
model = tok["model"]
|
||||
if model["type"] != "BPE" or not model.get("byte_fallback"):
|
||||
raise NotImplementedError("VoxCPM2: unexpected tokenizer type")
|
||||
|
||||
vocab: dict[str, int] = model["vocab"]
|
||||
vocab_size = self.hparams["vocab_size"]
|
||||
tokens = [f"[PAD{i}]" for i in range(vocab_size)]
|
||||
scores = [-1e9] * vocab_size
|
||||
toktypes = [gguf.TokenType.UNUSED] * vocab_size
|
||||
for text, i in vocab.items():
|
||||
tokens[i] = text
|
||||
toktypes[i] = gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-Fa-f]{2}>", text) else gguf.TokenType.NORMAL
|
||||
for rank, merge in enumerate(model["merges"]):
|
||||
a, b = merge.split(" ") if isinstance(merge, str) else merge
|
||||
i = vocab[a + b]
|
||||
scores[i] = max(scores[i], -float(rank))
|
||||
|
||||
# the special tokens (e.g. <|audio_start|>) are only listed in tokenizer_config.json
|
||||
specials = {int(i): t["content"] for i, t in tok_cfg.get("added_tokens_decoder", {}).items() if t.get("special")}
|
||||
specials.update({t["id"]: t["content"] for t in tok["added_tokens"] if t.get("special")})
|
||||
for i, text in specials.items():
|
||||
tokens[i] = text
|
||||
toktypes[i] = gguf.TokenType.CONTROL
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("llama")
|
||||
self.gguf_writer.add_tokenizer_pre("default")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores(scores)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
|
||||
@ModelBase.register("VoxCPM2Model")
|
||||
@ModelBase.example("openbmb/VoxCPM2")
|
||||
class VoxCPM2MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True # AudioVAE encoder, for voice cloning
|
||||
|
||||
_LAYER_MAP = {
|
||||
"input_layernorm.weight": "ln1",
|
||||
"self_attn.q_proj.weight": "attn_q",
|
||||
"self_attn.k_proj.weight": "attn_k",
|
||||
"self_attn.v_proj.weight": "attn_v",
|
||||
"self_attn.o_proj.weight": "attn_out",
|
||||
"post_attention_layernorm.weight": "ln2",
|
||||
"mlp.gate_proj.weight": "ffn_gate",
|
||||
"mlp.up_proj.weight": "ffn_up",
|
||||
"mlp.down_proj.weight": "ffn_down",
|
||||
}
|
||||
|
||||
# libllama names, mtmd loads the residual LM as a text model
|
||||
_RES_LAYER_MAP = {
|
||||
"input_layernorm.weight": "attn_norm",
|
||||
"self_attn.q_proj.weight": "attn_q",
|
||||
"self_attn.k_proj.weight": "attn_k",
|
||||
"self_attn.v_proj.weight": "attn_v",
|
||||
"self_attn.o_proj.weight": "attn_output",
|
||||
"post_attention_layernorm.weight": "ffn_norm",
|
||||
"mlp.gate_proj.weight": "ffn_gate",
|
||||
"mlp.up_proj.weight": "ffn_up",
|
||||
"mlp.down_proj.weight": "ffn_down",
|
||||
}
|
||||
|
||||
_PROJ_MAP = {
|
||||
"fsq_layer.in_proj": "a.gen.fsq.in",
|
||||
"fsq_layer.out_proj": "a.gen.fsq.out",
|
||||
"enc_to_lm_proj": "a.gen.enc_to_lm",
|
||||
"lm_to_dit_proj": "a.gen.lm_to_dit",
|
||||
"res_to_dit_proj": "a.gen.res_to_dit",
|
||||
"fusion_concat_proj": "a.gen.fusion",
|
||||
"stop_proj": "a.gen.stop_proj",
|
||||
"stop_head": "a.gen.stop_head",
|
||||
"feat_encoder.in_proj": "a.gen.enc.in_proj",
|
||||
"feat_decoder.estimator.in_proj": "a.gen.dit.in_proj",
|
||||
"feat_decoder.estimator.cond_proj": "a.gen.dit.cond_proj",
|
||||
"feat_decoder.estimator.out_proj": "a.gen.dit.out_proj",
|
||||
"feat_decoder.estimator.time_mlp.linear_1": "a.gen.dit.time_mlp.0",
|
||||
"feat_decoder.estimator.time_mlp.linear_2": "a.gen.dit.time_mlp.1",
|
||||
}
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
lm = self.global_config["lm_config"]
|
||||
dit = self.global_config["dit_config"]
|
||||
vae = self.global_config["audio_vae_config"]
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
|
||||
# LocEnc + LocDiT + residual LM + AudioVAE decoder, LocEnc has the hparams of LocDiT
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.VOXCPM2_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(dit["hidden_dim"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(dit["ffn_dim"])
|
||||
self.gguf_writer.add_gen_audio_block_count(dit["num_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(dit["num_heads"])
|
||||
self.gguf_writer.add_gen_audio_head_count_kv(lm["num_key_value_heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(lm["rms_norm_eps"])
|
||||
|
||||
# AudioVAE encoder: raw waveform -> latents, LocEnc in the gen model then makes the backbone rows
|
||||
self.gguf_writer.add_clip_has_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.VOXCPM2_SPKENC)
|
||||
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_audio_num_mel_bins(1)
|
||||
self.gguf_writer.add_audio_block_count(0)
|
||||
self.gguf_writer.add_audio_embedding_length(vae["encoder_dim"])
|
||||
self.gguf_writer.add_audio_feed_forward_length(vae["encoder_dim"])
|
||||
self.gguf_writer.add_audio_head_count(1)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(lm["rms_norm_eps"])
|
||||
# the ODE solver, the RoPE factors and the VAE layout are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# the AudioVAE convs and the 1D tensors stay F32
|
||||
if new_name.startswith(("a.vae.", "a.gen.vae.")) or n_dims < 2:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
if item[0].startswith("base_lm."):
|
||||
return None
|
||||
return item
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# already named by generate_extra_tensors()
|
||||
if name.startswith("a."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
# dt is always 0 (mean_mode = false), delta_time_mlp gives a constant that is folded into the time_mlp bias
|
||||
if name.startswith("feat_decoder.estimator.delta_time_mlp."):
|
||||
return
|
||||
if name == "feat_decoder.estimator.time_mlp.linear_2.bias":
|
||||
data_torch = LazyTorchTensor.to_eager(data_torch).float() + self._dtime_const()
|
||||
|
||||
if name == "feat_encoder.special_token":
|
||||
yield ("a.gen.enc.special_token", data_torch.reshape(-1))
|
||||
return
|
||||
|
||||
for src, dst in (("residual_lm.", "a.gen.res"), ("feat_encoder.encoder.", "a.gen.enc"),
|
||||
("feat_decoder.estimator.decoder.", "a.gen.code")):
|
||||
if not name.startswith(src):
|
||||
continue
|
||||
key = name.removeprefix(src)
|
||||
if key == "norm.weight":
|
||||
yield (f"{dst}.output_norm.weight" if dst == "a.gen.res" else f"{dst}.post_ln.weight", data_torch)
|
||||
return
|
||||
assert bid is not None, name
|
||||
key = key.removeprefix(f"layers.{bid}.")
|
||||
layer_map = self._RES_LAYER_MAP if dst == "a.gen.res" else self._LAYER_MAP
|
||||
yield (f"{dst}.blk.{bid}.{layer_map[key]}.weight", data_torch)
|
||||
return
|
||||
|
||||
base, _, suffix = name.rpartition(".")
|
||||
yield (f"{self._PROJ_MAP[base]}.{suffix}", data_torch)
|
||||
|
||||
# delta_time_mlp(SinusoidalPosEmb(0)), see VoxCPMLocDiTV2.forward()
|
||||
def _dtime_const(self) -> Tensor:
|
||||
d = {k: LazyTorchTensor.to_eager(self.model_tensors[f"feat_decoder.estimator.delta_time_mlp.{k}"]()).float()
|
||||
for k in ("linear_1.weight", "linear_1.bias", "linear_2.weight", "linear_2.bias")}
|
||||
dim = d["linear_1.weight"].shape[1]
|
||||
emb = torch.cat([torch.zeros(dim // 2), torch.ones(dim // 2)]) # sin(0), cos(0)
|
||||
h = torch.nn.functional.silu(d["linear_1.weight"] @ emb + d["linear_1.bias"])
|
||||
return d["linear_2.weight"] @ h + d["linear_2.bias"]
|
||||
|
||||
@staticmethod
|
||||
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
|
||||
return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# LongRoPE factors of LocEnc and LocDiT, picked as in MiniCPMLongRoPE._set_cos_sin_cache()
|
||||
lm = self.global_config["lm_config"]
|
||||
rope = lm["rope_scaling"]
|
||||
if rope.get("type") != "longrope":
|
||||
raise NotImplementedError(f"VoxCPM2: unexpected rope_scaling {rope.get('type')}")
|
||||
if lm["max_position_embeddings"] > rope["original_max_position_embeddings"]:
|
||||
raise NotImplementedError("VoxCPM2: the long RoPE factors also scale cos/sin, not supported")
|
||||
yield ("a.gen.rope_factors", torch.tensor(rope["short_factor"], dtype=torch.float32))
|
||||
|
||||
path = self.dir_model / "audiovae.pth"
|
||||
logger.info(f"gguf: loading AudioVAE from '{path}'")
|
||||
sd = torch.load(str(path), map_location="cpu", mmap=True, weights_only=True)
|
||||
sd = sd.get("state_dict", sd)
|
||||
vae = self.global_config["audio_vae_config"]
|
||||
|
||||
def conv(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
yield (f"{dst}.weight", self._wn(sd, src))
|
||||
yield (f"{dst}.bias", sd[src + ".bias"].float())
|
||||
|
||||
def snake(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
alpha, beta = fold_snake(sd[src + ".alpha"])
|
||||
yield (f"{dst}.alpha", alpha)
|
||||
yield (f"{dst}.beta", beta)
|
||||
|
||||
def res_units(dst: str, src: str, first: int) -> Iterable[tuple[str, Tensor]]:
|
||||
# dilation 1, 3, 9: Snake, depthwise conv k = 7, Snake, conv k = 1
|
||||
for r in range(3):
|
||||
s = f"{src}.{first + r}.block"
|
||||
d = f"{dst}.res.{r}"
|
||||
yield from snake(f"{d}.snake1", f"{s}.0")
|
||||
yield from conv(f"{d}.conv1", f"{s}.1")
|
||||
yield from snake(f"{d}.snake2", f"{s}.2")
|
||||
yield from conv(f"{d}.conv2", f"{s}.3")
|
||||
|
||||
# encoder: conv, then per stride (3 residual units, Snake, strided conv), then fc_mu
|
||||
p = "a.vae.enc"
|
||||
yield from conv(f"{p}.conv_in", "encoder.block.0")
|
||||
for b in range(len(vae["encoder_rates"])):
|
||||
s = f"encoder.block.{b + 1}.block"
|
||||
yield from res_units(f"{p}.blk.{b}", s, 0)
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.3")
|
||||
yield from conv(f"{p}.blk.{b}.down", f"{s}.4")
|
||||
yield from conv(f"{p}.fc_mu", "encoder.fc_mu")
|
||||
|
||||
# decoder: depthwise conv, conv k = 1, then per stride (sample-rate scale and bias, Snake, transposed conv, 3 residual units), then Snake, conv
|
||||
bounds = vae["sr_bin_boundaries"]
|
||||
sr_idx = sum(1 for b in bounds if b < VAE_OUT_SAMPLE_RATE) # torch.bucketize
|
||||
if vae["out_sample_rate"] != VAE_OUT_SAMPLE_RATE:
|
||||
raise NotImplementedError(f"VoxCPM2: unexpected out_sample_rate {vae['out_sample_rate']}")
|
||||
p = "a.gen.vae"
|
||||
yield from conv(f"{p}.conv_in_dw", "decoder.model.0")
|
||||
yield from conv(f"{p}.conv_in", "decoder.model.1")
|
||||
n_up = len(vae["decoder_rates"])
|
||||
for b in range(n_up):
|
||||
c = f"decoder.sr_cond_model.{b + 2}"
|
||||
yield (f"{p}.blk.{b}.sr_scale", sd[c + ".scale_embed.weight"][sr_idx].float())
|
||||
yield (f"{p}.blk.{b}.sr_bias", sd[c + ".bias_embed.weight"][sr_idx].float())
|
||||
s = f"decoder.model.{b + 2}.block"
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.0")
|
||||
yield from conv(f"{p}.blk.{b}.up", f"{s}.1")
|
||||
yield from res_units(f"{p}.blk.{b}", s, 2)
|
||||
yield from snake(f"{p}.post_snake", f"decoder.model.{n_up + 2}")
|
||||
yield from conv(f"{p}.conv_out", f"decoder.model.{n_up + 3}")
|
||||
|
||||
assert math.prod(vae["encoder_rates"]) * VAE_OUT_SAMPLE_RATE == math.prod(vae["decoder_rates"]) * vae["sample_rate"]
|
||||
@@ -0,0 +1,144 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import torch
|
||||
|
||||
from typing import Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("ZayaForCausalLM")
|
||||
@ModelBase.example("Zyphra/ZAYA1-8B")
|
||||
class ZayaModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.ZAYA1
|
||||
|
||||
# checkpoint names without a .weight/.bias suffix
|
||||
_RENAME = {
|
||||
"model.input_hidden_states_scale": "model.input_hidden_states.weight",
|
||||
"model.input_hidden_states_bias": "model.input_hidden_states.bias",
|
||||
}
|
||||
|
||||
# the two kv heads of attn_v: head 0 = current token, head 1 = previous token
|
||||
_V_PARTS = (
|
||||
"self_attn.qkv_proj.v_proj_current.weight",
|
||||
"self_attn.qkv_proj.v_proj_delayed.weight",
|
||||
)
|
||||
|
||||
# (hs + b_hs) * s_hs + (res + b_res) * s_res = hs * s_hs + res * s_res + (b_hs * s_hs + b_res * s_res)
|
||||
_RES_SCALES = {
|
||||
"post_attention_residual_scale.": (gguf.MODEL_TENSOR.ATTN_RES_SCALE_HS, gguf.MODEL_TENSOR.ATTN_RESID_SCALE),
|
||||
"post_mlp_residual_scale.": (gguf.MODEL_TENSOR.FFN_RES_SCALE_HS, gguf.MODEL_TENSOR.FFN_RESID_SCALE),
|
||||
}
|
||||
_RES_PARTS = ("hidden_states_scale", "hidden_states_bias", "residual_scale", "residual_bias")
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
# added tokens overlap the base vocab, so count the token list itself
|
||||
self._vocab = gguf.LlamaHfVocab(self.dir_model)
|
||||
self._n_vocab = self._vocab.vocab_size
|
||||
self._parts: dict[tuple[int, str], dict[str, Tensor]] = {}
|
||||
self.rope_parameters = self.hparams["rope_parameters"]["hybrid"]
|
||||
|
||||
def set_vocab(self):
|
||||
# added special tokens overlap the base vocab, so mark them as control here
|
||||
with open(self.dir_model / "tokenizer.json", "r", encoding="utf-8") as f:
|
||||
added = {t["id"]: t["special"] for t in json.load(f)["added_tokens"]}
|
||||
|
||||
tokens, scores, toktypes = [], [], []
|
||||
for i, (text, score, toktype) in enumerate(self._vocab.all_tokens()):
|
||||
tokens.append(text)
|
||||
scores.append(score)
|
||||
if added.get(i):
|
||||
toktype = gguf.TokenType.CONTROL
|
||||
toktypes.append(toktype)
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("gemma4")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores(scores)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
self.gguf_writer.add_add_space_prefix(False)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hparams = self.hparams
|
||||
super().set_gguf_parameters()
|
||||
|
||||
self.gguf_writer.add_rope_dimension_count(int(self.rope_parameters["partial_rotary_factor"] * hparams["head_dim"]))
|
||||
self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
|
||||
self.gguf_writer.add_ssm_conv_kernel(hparams["cca_time0"])
|
||||
self.gguf_writer.add_router_hidden_size(hparams["router_hidden_size"])
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
# skip the draft head tensors now, they will be added as part of the TiDAR changes
|
||||
if name.startswith(("diffusion_output_layer.", "diffusion_markov_head.")):
|
||||
return None
|
||||
|
||||
return super().filter_tensors(item)
|
||||
|
||||
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
|
||||
# the router picks one expert, so its matmuls stay F32 like ffn_gate_inp
|
||||
if any(self.match_model_tensor_name(new_name, key, bid) for key in (
|
||||
gguf.MODEL_TENSOR.CCA_CONV_GRP,
|
||||
gguf.MODEL_TENSOR.ROUTER_FC1,
|
||||
gguf.MODEL_TENSOR.ROUTER_FC2,
|
||||
gguf.MODEL_TENSOR.ROUTER_OUT)):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if bid is not None:
|
||||
local = name.removeprefix(f"model.layers.{bid}.")
|
||||
if local in self._V_PARTS:
|
||||
p = self._parts.setdefault((bid, "v"), {})
|
||||
p[local] = data_torch
|
||||
if len(p) == len(self._V_PARTS):
|
||||
del self._parts[(bid, "v")]
|
||||
v = torch.cat([p[n] for n in self._V_PARTS], dim=0)
|
||||
yield from super().modify_tensors(v, f"model.layers.{bid}.{self._V_PARTS[0]}", bid)
|
||||
return
|
||||
for prefix, (key_hs, key_res) in self._RES_SCALES.items():
|
||||
if local.startswith(prefix) and local[len(prefix):] in self._RES_PARTS:
|
||||
p = self._parts.setdefault((bid, prefix), {})
|
||||
p[local[len(prefix):]] = data_torch
|
||||
if len(p) == len(self._RES_PARTS):
|
||||
del self._parts[(bid, prefix)]
|
||||
s_hs, b_hs, s_res, b_res = (p[n].float() for n in self._RES_PARTS)
|
||||
yield self.format_tensor_name(key_hs, bid, ".weight"), s_hs
|
||||
yield self.format_tensor_name(key_res, bid, ".weight"), s_res
|
||||
yield self.format_tensor_name(key_res, bid, ".bias"), b_hs * s_hs + b_res * s_res
|
||||
return
|
||||
|
||||
name = self._RENAME.get(name, name)
|
||||
if name.endswith("balancing_biases"):
|
||||
name += ".bias"
|
||||
elif not name.endswith((".weight", ".bias")):
|
||||
name += ".weight"
|
||||
new_name = self.map_tensor_name(name)
|
||||
|
||||
def is_tensor(key: gguf.MODEL_TENSOR) -> bool:
|
||||
return self.match_model_tensor_name(new_name, key, bid)
|
||||
|
||||
if any(is_tensor(key) for key in (gguf.MODEL_TENSOR.ATTN_Q, gguf.MODEL_TENSOR.ATTN_K)):
|
||||
# through the base class, which fuses q, k and v with --fuse-qkv
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
return
|
||||
if is_tensor(gguf.MODEL_TENSOR.TOKEN_EMBD):
|
||||
data_torch = data_torch[:self._n_vocab]
|
||||
elif is_tensor(gguf.MODEL_TENSOR.SSM_CONV1D):
|
||||
data_torch = data_torch.squeeze(1)
|
||||
elif is_tensor(gguf.MODEL_TENSOR.CCA_CONV_GRP):
|
||||
# [n_group*out, in, kernel] -> [kernel, n_group, out, in]
|
||||
n_group = self.hparams["num_attention_heads"] + self.hparams["num_key_value_heads"]
|
||||
n_out, n_in, n_k = data_torch.shape
|
||||
data_torch = data_torch.reshape(n_group, n_out // n_group, n_in, n_k).permute(3, 0, 1, 2).contiguous()
|
||||
|
||||
yield new_name, data_torch
|
||||
@@ -100,6 +100,12 @@ The OpenCL backend has the following CMake options that control the behavior of
|
||||
| `GGML_OPENCL_USE_ADRENO_KERNELS` | `ON` | Use kernels optimized for Adreno. |
|
||||
| `GGML_OPENCL_USE_ADRENO_BIN_KERNELS` | `OFF` | Allow using binary kernel lib for Adreno. |
|
||||
|
||||
## Decomposed Flash Attention
|
||||
|
||||
The P8 path rounds scratch row pitches for P and transposed V up to a multiple of 32 KiB before adding the existing 64-byte padding. This reduces cache conflicts when the KV length exceeds 16384 and the V head size is at most 256. Larger V heads keep the previous layout. The extra scratch space is less than 32 KiB per row at the default granule.
|
||||
|
||||
`GGML_OPENCL_FA_KV_PITCH_ROUND` sets the rounding granule in bytes to a positive power of two; `0` restores the previous layout. `GGML_OPENCL_FA_KV_PITCH_ROUND_MIN` sets the KV-length threshold (default `16384`), and `GGML_OPENCL_FA_KV_PITCH_ROUND_MAX_DV` sets the largest V head size to round (default `256`). These settings do not change the existing attention dispatch conditions.
|
||||
|
||||
## Program Binary Cache
|
||||
|
||||
Compiled `cl_program` binaries are cached on disk, so subsequent runs skip the expensive
|
||||
|
||||
+413
-193
@@ -3,9 +3,9 @@
|
||||
> [!NOTE]
|
||||
> Performance and memory optimizations, accuracy validation, broader quantization coverage, broader operator and model support are work in progress.
|
||||
|
||||
[OpenVINO](https://docs.openvino.ai/) is an open-source toolkit for optimizing and deploying high-performance AI inference, specifically designed for Intel hardware, including CPUs, GPUs, and NPUs, in the cloud, on-premises, and on the edge. [OpenVINO backend for llama.cpp](../../ggml/src/ggml-openvino) enables hardware-accelerated inference on **Intel® CPUs, GPUs, and NPUs** while remaining compatible with the existing **GGUF model ecosystem**. The backend translates GGML compute graphs into OpenVINO graphs and leverages graph compilation, kernel fusion, and device-specific optimizations to improve inference performance on supported Intel hardware.
|
||||
[OpenVINO](https://docs.openvino.ai/) is an open-source toolkit for optimizing and deploying high-performance AI inference, specifically designed for Intel hardware, including CPUs, GPUs (integrated and discrete), and NPUs, in the cloud, on-premises, and on the edge. The [OpenVINO backend](../../ggml/src/ggml-openvino) runs the same GGUF model files on **Intel® CPUs, GPUs and NPUs**, without changes to the model or the rest of llama.cpp.
|
||||
|
||||
The OpenVINO backend is implemented in `ggml/src/ggml-openvino` and provides a translation layer for core GGML operations. The OpenVINO backend replaces the standard GGML graph execution path with Intel's OpenVINO inference engine. This approach allows the same GGUF model file to run on Intel CPUs, Intel GPUs (integrated and discrete), and Intel NPUs without changes to the model or the rest of the llama.cpp stack. When a `ggml_cgraph` is dispatched to OpenVINO backend, it:
|
||||
When a `ggml_cgraph` is dispatched to the OpenVINO backend, it:
|
||||
|
||||
- Walks the GGML graph and identifies inputs, outputs, weights, and KV cache tensors.
|
||||
- Translates the GGML operations into an `ov::Model` using OpenVINO's frontend API.
|
||||
@@ -16,9 +16,10 @@ For guidance on contributing to the OpenVINO backend, see the [OpenVINO Backend
|
||||
|
||||
## Contents
|
||||
|
||||
- [Quick Start](#quick-start)
|
||||
- [Supported Devices](#supported-devices)
|
||||
- [Supported Model Precisions](#supported-model-precisions)
|
||||
- [Supported Llama.cpp Tools](#supported-llamacpp-tools)
|
||||
- [Supported llama.cpp Tools](#supported-llamacpp-tools)
|
||||
- [Validated Models](#validated-models)
|
||||
- [Build Instructions](#build-instructions)
|
||||
- [0. Prerequisites](#0-prerequisites)
|
||||
@@ -29,10 +30,44 @@ For guidance on contributing to the OpenVINO backend, see the [OpenVINO Backend
|
||||
- [3. Download Sample Model](#3-download-sample-model)
|
||||
- [4. Run Inference with OpenVINO Backend](#4-run-inference-with-openvino-backend)
|
||||
- [5. Docker Build](#5-docker-build)
|
||||
- [GGML OpenVINO Backend Runtime Configurations](#ggml-openvino-backend-runtime-configurations)
|
||||
- [Runtime Configuration](#runtime-configuration)
|
||||
- [Basic](#basic)
|
||||
- [Plugin properties](#plugin-properties)
|
||||
- [Advanced tuning](#advanced-tuning)
|
||||
- [Compiled model cache](#compiled-model-cache)
|
||||
- [Deprecated](#deprecated)
|
||||
- [Debugging and development](#debugging-and-development)
|
||||
- [Example Usage](#example-usage)
|
||||
- [GPU Inference with Profiling](#gpu-inference-with-profiling)
|
||||
- [Known Limitations](#known-limitations)
|
||||
- [Work in Progress](#work-in-progress)
|
||||
|
||||
## Quick Start
|
||||
|
||||
On Ubuntu, with the [prerequisites](#0-prerequisites) and the [OpenVINO Runtime](#1-install-openvino-runtime) installed:
|
||||
|
||||
```bash
|
||||
# 1. Build
|
||||
git clone https://github.com/ggml-org/llama.cpp
|
||||
cd llama.cpp
|
||||
source /opt/intel/openvino/setupvars.sh
|
||||
cmake -B build/ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON
|
||||
cmake --build build/ReleaseOV --parallel
|
||||
|
||||
# 2. Download a model
|
||||
mkdir -p ~/models/
|
||||
wget https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf \
|
||||
-O ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
|
||||
# 3. Run on CPU, GPU or NPU
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024
|
||||
```
|
||||
|
||||
The [Ubuntu Build Script](#ubuntu-build-script) and the [Windows Build Script](#windows-build-script) do the setup in one step: they install the prerequisites and the OpenVINO Runtime, and build llama.cpp with the OpenVINO backend.
|
||||
|
||||
See [Build Instructions](#build-instructions) for Windows and Docker, and [Runtime Configuration](#runtime-configuration) for the environment variables.
|
||||
|
||||
## Supported Devices
|
||||
|
||||
OpenVINO backend supports the following hardware:
|
||||
@@ -41,7 +76,7 @@ OpenVINO backend supports the following hardware:
|
||||
- Intel GPUs (integrated and discrete)
|
||||
- Intel NPUs
|
||||
|
||||
Although OpenVINO supports a wide range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), the llama.cpp OpenVINO backend has been validated specifically on AI PCs such as the Intel® Core™ Ultra Series 1 and Series 2.
|
||||
Although OpenVINO supports a wide range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), the llama.cpp OpenVINO backend has been validated specifically on AI PCs such as the Intel® Core™ Ultra Series 2 (Lunar Lake) and Series 3 (Panther Lake).
|
||||
|
||||
## Supported Model Precisions
|
||||
|
||||
@@ -51,101 +86,123 @@ Although OpenVINO supports a wide range of [Intel hardware](https://docs.openvin
|
||||
- `Q4_0`
|
||||
- `Q4_1`
|
||||
- `Q4_K`
|
||||
- `Q4_K_M`
|
||||
- `Q5_K` (converted to `Q8_0_C` at runtime by default)
|
||||
- `Q6_K` (converted to `Q8_0_C` at runtime by default)
|
||||
- `Q5_0` (converted at runtime, see the details below)
|
||||
- `Q5_1`
|
||||
- `Q5_K` (converted at runtime, see the details below)
|
||||
- `Q6_K` (converted at runtime, see the details below)
|
||||
|
||||
> [!NOTE]
|
||||
> Accuracy validation and performance optimizations for quantized models are a work in progress.
|
||||
`Q4_K_M` files mix `Q4_K` tensors with `Q6_K` tensors, and sometimes `Q5_K` tensors.
|
||||
|
||||
**CPU and GPU Quantization Details:**
|
||||
- `Q5_K` and `Q6_K` tensors are converted to `Q8_0_C`
|
||||
<details>
|
||||
<summary>Quantization details for each device</summary>
|
||||
|
||||
**NPU Quantization Details:**
|
||||
- Primary supported quantization scheme is `Q4_0`
|
||||
- `Q6_K` tensors are requantized to `Q4_0_128` in general. For embedding weights, `Q6_K` tensors are requantized to `Q8_0_C` except for the token embedding matrix which is dequantized to fp16
|
||||
**All devices:**
|
||||
- `FP16` and `BF16` tensors are used as they are.
|
||||
- A quantized token embedding (`token_embd.weight`) and output weight (`output.weight`) are converted to `Q8_0_C`.
|
||||
|
||||
**CPU and GPU:**
|
||||
- `Q4_0`, `Q4_1`, `Q4_K`, `Q5_1` and `Q8_0` tensors keep their layout: weights, scales and zero points are extracted directly.
|
||||
- `Q5_K` and `Q6_K` tensors are converted to `Q8_0_C`. `GGML_OPENVINO_REQUANT_KQUANT` selects a 4-bit layout instead.
|
||||
- `Q5_0` tensors are converted to `Q4_0_64`.
|
||||
|
||||
**GPU:**
|
||||
- `Q4_K`, `Q5_K` and `Q6_K` MoE expert tensors are converted to `Q4_1_64`. This lets the GPU plugin use its fused MoE op, which is much faster for prompt processing. `GGML_OPENVINO_REQUANT_KQUANT` does not change this. `GGML_OPENVINO_MOE_OP=0` turns the fused MoE op off.
|
||||
|
||||
**NPU:**
|
||||
- All other quantized tensors are requantized to `Q4_0_128`.
|
||||
- For a `Q6_K` token embedding, `GGML_OPENVINO_NPU_TOKEN_EMBD_QUANT` selects another layout.
|
||||
- The validated models use `Q4_K_M` files.
|
||||
|
||||
**Layout names:**
|
||||
- `Q8_0_C`: 8-bit, one scale per row.
|
||||
- `Q4_0_128` and `Q4_0_64`: 4-bit, symmetric, one scale per 128 or 64 weights.
|
||||
|
||||
**Additional Notes:**
|
||||
- Both `Q4_0` and `Q4_1` models use `Q6_K` for the token embedding tensor and the final matmul weight tensor (often the same tensor)
|
||||
- `Q4_0` models may produce some `Q4_1` tensors if an imatrix is provided during quantization using `llama-quantize`
|
||||
- `Q4_K_M` models may include both `Q6_K` and `Q5_K` tensors (observed in Phi-3)
|
||||
- `Q5_1` tensors are dequantized natively (weights, scales, and zero-points extracted directly)
|
||||
|
||||
## Supported Llama.cpp Tools
|
||||
</details>
|
||||
|
||||
The OpenVINO backend integrates with the standard llama.cpp tools listed below.
|
||||
However, all the tools coverage across all devices is not uniform and exhaustive validation is work in progress.
|
||||
## Supported llama.cpp Tools
|
||||
|
||||
The OpenVINO backend works with the standard llama.cpp tools listed below. Tool coverage is not uniform across devices.
|
||||
|
||||
- llama-bench
|
||||
- llama-cli
|
||||
- llama-completion
|
||||
- llama-embedding
|
||||
- llama-perplexity
|
||||
- llama-run
|
||||
- llama-server
|
||||
- llama-simple
|
||||
|
||||
## Validated Models
|
||||
|
||||
Although, the validated models below were tested with `llama-cli` using the `Q4_K_M` quantization format on Intel® Core™ Ultra Series 2 (Lunar Lake), the OpenVINO backend is expected to work across a broader range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), [supported model precisions](#supported-model-precisions), [supported llama.cpp tools](#supported-llamacpp-tools) and additional model architectures.
|
||||
|
||||
> [!NOTE]
|
||||
> Extensive accuracy validation, performance optimizations, and broader architecture coverage are work in progress.
|
||||
The models below were tested with `llama-cli` and `Q4_K_M` files on an Intel® Core™ Ultra Series 2 (Lunar Lake). The OpenVINO backend is expected to work on a broader range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), [model precisions](#supported-model-precisions), [llama.cpp tools](#supported-llamacpp-tools) and model architectures.
|
||||
|
||||
**Legend & Test Configuration:**
|
||||
- **Status:** ✓ = Passed | ~ = Accuracy issues | ✗ = Failed or Unsupported
|
||||
- **Execution Modes:**
|
||||
- **SL** = Stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`)
|
||||
- **SF** = Stateful (`GGML_OPENVINO_STATEFUL_EXECUTION=1`)
|
||||
- Note: The NPU operates in stateless mode only.
|
||||
- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel Graphics Compiler 2.41.5 | Intel OpenCL GPU Driver 26.35.39758.10-0 | Intel NPU Driver 1.38.0.
|
||||
- **Execution mode:** stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`) on every device.
|
||||
- **Validation system:** Intel® Core™ Ultra 7 258V (Lunar Lake) | 32 GB RAM | Ubuntu 26.04.1 | Intel Graphics Compiler 2.42.6 | Intel OpenCL GPU Driver 26.40.40196.8-0 | Intel NPU Driver 1.38.0.
|
||||
- See [Known Limitations](#known-limitations) for context on observed failures.
|
||||
|
||||
| Model | CPU (SL / SF) | GPU (SL / SF) | NPU (SL) |
|
||||
<details open>
|
||||
<summary><b>Validated models</b></summary>
|
||||
|
||||
| Model | CPU | GPU | NPU |
|
||||
| :--- | :---: | :---: | :---: |
|
||||
| [bartowski/Llama-3.2-1B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Llama-3.2-3B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Meta-Llama-3.1-8B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Llama-3.2-1B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Llama-3.2-3B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Meta-Llama-3.1-8B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3.5-0.8B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-0.8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [bartowski/Qwen_Qwen3.5-2B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [bartowski/Qwen_Qwen3.5-4B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-4B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3.5-0.8B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-0.8B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3.5-2B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Qwen_Qwen3.5-4B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-4B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ~ |
|
||||
| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✓ | ✗ / ✗ | ✓ |
|
||||
| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ |
|
||||
| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ | ✓ | ✗ |
|
||||
| | | | |
|
||||
| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/microsoft_Phi-4-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/microsoft_Phi-4-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/microsoft_Phi-4-mini-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [bartowski/Mistral-7B-Instruct-v0.3-Q4_K_M](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [QuantFactory/Ministral-3b-instruct.Q4_K_M](https://huggingface.co/QuantFactory/Ministral-3b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Ministral-8B-Instruct-2410-Q4_K_M](https://huggingface.co/bartowski/Ministral-8B-Instruct-2410-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/Mistral-7B-Instruct-v0.3-Q4_K_M](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [QuantFactory/Ministral-3b-instruct.Q4_K_M](https://huggingface.co/QuantFactory/Ministral-3b-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/Ministral-8B-Instruct-2410-Q4_K_M](https://huggingface.co/bartowski/Ministral-8B-Instruct-2410-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ~ / ~ | ✓ |
|
||||
| [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ / ✓ | ~ / ~ | ~ |
|
||||
| [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ | ✓ | ~ |
|
||||
| [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| | | | |
|
||||
| [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ |
|
||||
| [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ | ✓ | ✓ |
|
||||
| [openbmb/MiniCPM-V-4_5-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-4_5-gguf) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [LiquidAI/LFM2.5-2.6B-Q4_K_M](https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF) | ✓ | ✓ | ✗ |
|
||||
| [bartowski/inclusionAI_Ling-mini-2.0-Q4_K_M](https://huggingface.co/bartowski/inclusionAI_Ling-mini-2.0-GGUF) | ✓ | ✓ | ✗ |
|
||||
| | | | |
|
||||
| [gpustack/bge-m3-Q4_K_M.gguf](https://huggingface.co/gpustack/bge-m3-GGUF) | ✓ | ✗ | ✗ |
|
||||
| [Qwen/Qwen3-Embedding-0.6B-Q8_0](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B-GGUF) | ✓ | ✓ | ✗ |
|
||||
| [ggml-org/bge-m3-Q8_0](https://huggingface.co/ggml-org/bge-m3-Q8_0-GGUF) | ✓ | ✓ | ✗ |
|
||||
| [nomic-ai/nomic-embed-text-v1.5-Q8_0](https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF) | ✓ | ✓ | ✗ |
|
||||
|
||||
The embedding models were tested with `llama-embedding` and `Q8_0` files.
|
||||
|
||||
</details>
|
||||
|
||||
## Build Instructions
|
||||
|
||||
@@ -154,39 +211,51 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
|
||||
- Linux or Windows system with Intel hardware (CPU, GPU, or NPU)
|
||||
- **For Intel GPU or NPU Usage**: Install the appropriate hardware drivers for your Intel GPU or NPU. For detailed instructions, see: [Additional Configurations for Hardware Acceleration](https://docs.openvino.ai/2026/get-started/install-openvino/configurations.html).
|
||||
|
||||
- **Linux:**
|
||||
- Git, CMake, and Ninja software tools are needed for building.
|
||||
```bash
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential libcurl4-openssl-dev libtbb12 cmake ninja-build python3-pip curl wget tar
|
||||
```
|
||||
- OpenCL
|
||||
```bash
|
||||
sudo apt install ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd
|
||||
```
|
||||
<details open>
|
||||
<summary><b>Linux</b></summary>
|
||||
|
||||
- **Windows:**
|
||||
- Download and install [Microsoft Visual Studio 2022 Build Tools](https://aka.ms/vs/17/release/vs_BuildTools.exe). During installation, select the **"Desktop development with C++"** workload.
|
||||
Build tools (Git, CMake, Ninja):
|
||||
|
||||
- Install required tools:
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
winget install Git.Git
|
||||
winget install GNU.Wget
|
||||
winget install Ninja-build.Ninja
|
||||
```
|
||||
```bash
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y build-essential libcurl4-openssl-dev libtbb12 cmake ninja-build python3-pip curl wget tar
|
||||
```
|
||||
|
||||
- Install **OpenCL** using **vcpkg**:
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
cd C:\
|
||||
git clone https://github.com/microsoft/vcpkg
|
||||
cd vcpkg
|
||||
.\bootstrap-vcpkg.bat
|
||||
.\vcpkg install opencl
|
||||
# Optional but recommended: Integrate vcpkg with Visual Studio / CMake:
|
||||
.\vcpkg integrate install
|
||||
```
|
||||
OpenCL:
|
||||
|
||||
```bash
|
||||
sudo apt install ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Windows</b></summary>
|
||||
|
||||
1. Download and install [Microsoft Visual Studio 2022 Build Tools](https://aka.ms/vs/17/release/vs_BuildTools.exe). During installation, select the **"Desktop development with C++"** workload.
|
||||
2. Install the required tools:
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
winget install Git.Git
|
||||
winget install GNU.Wget
|
||||
winget install Ninja-build.Ninja
|
||||
```
|
||||
|
||||
3. Install **OpenCL** using **vcpkg**:
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
cd C:\
|
||||
git clone https://github.com/microsoft/vcpkg
|
||||
cd vcpkg
|
||||
.\bootstrap-vcpkg.bat
|
||||
.\vcpkg install opencl
|
||||
# Optional but recommended: Integrate vcpkg with Visual Studio / CMake:
|
||||
.\vcpkg integrate install
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
### 1. Install OpenVINO Runtime
|
||||
|
||||
@@ -199,21 +268,25 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_
|
||||
|
||||
### 2. Build llama.cpp with OpenVINO Backend
|
||||
|
||||
Clone llama.cpp repo and build :
|
||||
Clone the llama.cpp repository:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/ggml-org/llama.cpp
|
||||
cd llama.cpp
|
||||
```
|
||||
|
||||
- **Linux:**
|
||||
Build on Linux:
|
||||
|
||||
```bash
|
||||
source /opt/intel/openvino/setupvars.sh
|
||||
cmake -B build/ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON
|
||||
cmake --build build/ReleaseOV --parallel
|
||||
```
|
||||
|
||||
- **Windows:** Open **x64 Native Tools Command Prompt for VS** (so the MSVC toolchain is on `PATH`), then run:
|
||||
<details>
|
||||
<summary><b>Build on Windows</b></summary>
|
||||
|
||||
Open **x64 Native Tools Command Prompt for VS** (so the MSVC toolchain is on `PATH`), then run:
|
||||
|
||||
```cmd
|
||||
C:\Intel\openvino\setupvars.bat
|
||||
@@ -224,9 +297,11 @@ cmake --build build\ReleaseOV --parallel
|
||||
> [!NOTE]
|
||||
> The Windows install path is `C:\Intel\openvino` (no spaces) to avoid quoting problems some CMake/Ninja toolchains have with `C:\Program Files (x86)\...`. Adjust to wherever you installed OpenVINO Runtime. From `cmd`, run `C:\Intel\openvino\setupvars.bat`; from PowerShell, run `& "C:\Intel\openvino\setupvars.ps1"` instead. Once the build is finished you can launch the binaries from any `cmd` or `PowerShell` window after sourcing the matching `setupvars` script for that shell.
|
||||
|
||||
</details>
|
||||
|
||||
#### Ubuntu Build Script
|
||||
|
||||
For Ubuntu24 users, the following shell script automates the prerequisite installs (build tools, OpenCL ICD), the OpenVINO Runtime download/extract/setup, and the Ninja-based llama.cpp build.
|
||||
For Ubuntu users, the following shell script automates the prerequisite installs (build tools, OpenCL ICD), the OpenVINO Runtime download/extract/setup, and the Ninja-based llama.cpp build.
|
||||
Save the following as `build-llamacpp-ov.sh` next to where you want the `llama.cpp` folder to land, then run it:
|
||||
|
||||
```bash
|
||||
@@ -556,81 +631,102 @@ endlocal
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
### 3. Download Sample Model
|
||||
|
||||
Download sample model for testing.
|
||||
Download a sample model for testing.
|
||||
|
||||
```bash
|
||||
# Linux
|
||||
mkdir -p ~/models/
|
||||
wget https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf \
|
||||
-O ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary><b>Windows</b></summary>
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
mkdir C:\models
|
||||
Invoke-WebRequest -Uri https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf -OutFile C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
```
|
||||
|
||||
# Windows Command Line
|
||||
```cmd
|
||||
REM Windows Command Prompt
|
||||
mkdir C:\models
|
||||
curl -L https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf -o C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
### 4. Run Inference with OpenVINO Backend
|
||||
|
||||
When using the OpenVINO backend, the first inference token may have slightly higher latency due to on-the-fly conversion to the OpenVINO graph. Subsequent tokens and runs will be faster.
|
||||
The first run of a model converts and compiles it for the device. This takes seconds on CPU and GPU, and can take minutes on the NPU for large models. Later tokens are faster, and the [compiled model cache](#compiled-model-cache) makes later runs start faster.
|
||||
|
||||
> [!NOTE]
|
||||
> Default context size is set to the model training context, which may be very large. For example, 131072 for Llama 3.2 1B, which may result in lower performance, especially on edge/laptop devices. Use `-c` to limit context size in supported llama.cpp tools for better performance. For example, `-c 512`.
|
||||
> Default context size is set to the model training context, which may be very large. For example, 131072 for Llama 3.2 1B, which may result in lower performance, especially on edge/laptop devices. Use `-c` to limit context size in supported llama.cpp tools for better performance:
|
||||
> - CPU and GPU: for example `-c 1024`.
|
||||
> - NPU: keep the context small, for example `-c 512`, or `-c 256` for larger models. Decode may slow down as the context grows.
|
||||
|
||||
```bash
|
||||
# If device is unset or unavailable, defaults to CPU.
|
||||
# If the system has multiple GPUs, use GPU.0 or GPU.1 to explicitly target a specific GPU.
|
||||
|
||||
# Linux
|
||||
# If the device is unset, the CPU is used.
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
# Optional: enable stateful execution for improved GPU performance (recommended).
|
||||
export GGML_OPENVINO_STATEFUL_EXECUTION=1
|
||||
# To run llama-simple:
|
||||
./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -n 50 "The story of AI is "
|
||||
# To run in chat mode:
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024
|
||||
# To run llama-bench, -fa 1 is needed
|
||||
GGML_OPENVINO_STATEFUL_EXECUTION=1 GGML_OPENVINO_DEVICE=GPU ./build/ReleaseOV/bin/llama-bench -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -fa 1
|
||||
# To run llama-bench:
|
||||
GGML_OPENVINO_DEVICE=GPU ./build/ReleaseOV/bin/llama-bench -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
|
||||
# NPU: keep context small to avoid failures from very large model context windows.
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 512
|
||||
```
|
||||
|
||||
# Windows Command Line
|
||||
<details>
|
||||
<summary><b>Windows</b></summary>
|
||||
|
||||
```cmd
|
||||
REM Windows Command Prompt
|
||||
set GGML_OPENVINO_DEVICE=GPU
|
||||
# Optional: enable stateful execution for improved GPU performance (recommended).
|
||||
set GGML_OPENVINO_STATEFUL_EXECUTION=1
|
||||
REM To run llama-simple:
|
||||
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
|
||||
REM To run in chat mode:
|
||||
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 1024
|
||||
REM To run llama-bench:
|
||||
build\ReleaseOV\bin\llama-bench.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf"
|
||||
|
||||
REM NPU: keep the context small.
|
||||
set GGML_OPENVINO_DEVICE=NPU
|
||||
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 512
|
||||
```
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
$env:GGML_OPENVINO_DEVICE = "GPU"
|
||||
$env:GGML_OPENVINO_STATEFUL_EXECUTION = "1"
|
||||
|
||||
# To run llama-simple
|
||||
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
|
||||
# To run in chat mode:
|
||||
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 1024
|
||||
# To run llama-bench, -fa 1 is needed
|
||||
build\ReleaseOV\bin\llama-bench.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -fa 1
|
||||
|
||||
# NPU: keep context small to avoid failures from very large model context windows.
|
||||
# Windows Command Line
|
||||
set GGML_OPENVINO_DEVICE=NPU
|
||||
# Windows PowerShell
|
||||
# NPU: keep the context small.
|
||||
$env:GGML_OPENVINO_DEVICE = "NPU"
|
||||
build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 512
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
> [!NOTE]
|
||||
> On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU. A device that is not available is an error (no fallback to CPU), and the error message lists the available OpenVINO devices with their names. Run `llama-cli --list-devices` to see the valid values: each OpenVINO device shows the `GGML_OPENVINO_DEVICE=<value>` to set, and `(selected)` marks the active one. Select the OpenVINO device with this variable, not with `-dev`. See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html) for more details.
|
||||
> - Select the OpenVINO device with `GGML_OPENVINO_DEVICE`, not with `-dev`.
|
||||
> - On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU.
|
||||
> - To split a model across several GPUs, set a comma-separated list, e.g. `GPU.0,GPU.1`, and use `-sm layer`.
|
||||
> - A device that is not available is an error (no fallback to CPU). The error message lists the available OpenVINO devices with their names.
|
||||
> - Run `llama-cli --list-devices` to see the valid values: each OpenVINO device shows the `GGML_OPENVINO_DEVICE=<value>` to set, and `(selected)` marks the active one.
|
||||
> - See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html) for more details.
|
||||
|
||||
### 5. Docker Build
|
||||
|
||||
You can build and run llama.cpp with OpenVINO backend using Docker.
|
||||
|
||||
<details>
|
||||
<summary><b>Build the images</b></summary>
|
||||
|
||||
```bash
|
||||
# Build the base runtime image with compiled shared libraries and minimal dependencies.
|
||||
docker build -t llama-openvino:base -f .devops/openvino.Dockerfile .
|
||||
@@ -648,9 +744,12 @@ docker build --target=server -t llama-openvino:server -f .devops/openvino.Docker
|
||||
docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --target=server -t llama-openvino:server -f .devops/openvino.Dockerfile .
|
||||
```
|
||||
|
||||
Run llama.cpp with OpenVINO backend Docker container.
|
||||
Save sample models in `~/models` as [shown above](#3-download-sample-model). It will be mounted to the container in the examples below.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Run a container</b> (CPU, GPU, NPU)</summary>
|
||||
|
||||
Run llama.cpp with OpenVINO backend Docker container. Save sample models in `~/models` as [shown above](#3-download-sample-model). It will be mounted to the container in the examples below.
|
||||
|
||||
```bash
|
||||
# Run Docker container
|
||||
@@ -659,19 +758,20 @@ docker run --rm -it -v ~/models:/models llama-openvino:light --no-warmup -c 1024
|
||||
# With Intel GPU access (iGPU or dGPU)
|
||||
docker run --rm -it -v ~/models:/models \
|
||||
--device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
|
||||
--env=GGML_OPENVINO_DEVICE=GPU --env=GGML_OPENVINO_STATEFUL_EXECUTION=1 \
|
||||
--env=GGML_OPENVINO_DEVICE=GPU \
|
||||
llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
|
||||
# With Intel NPU access
|
||||
docker run --rm -it -v ~/models:/models \
|
||||
--device=/dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
|
||||
--device=/dev/accel --group-add=$(stat -c "%g" /dev/accel/accel* | head -n 1) -u $(id -u):$(id -g) \
|
||||
--env=GGML_OPENVINO_DEVICE=NPU \
|
||||
llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
llama-openvino:light --no-warmup -c 512 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
|
||||
```
|
||||
|
||||
Run Llama.cpp Server with OpenVINO Backend.
|
||||
> [!NOTE]
|
||||
> `llama-server` with OpenVINO backend supports only one chat session/thread, when `GGML_OPENVINO_STATEFUL_EXECUTION=1` is enabled.
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Run llama-server</b> (container or local build) and test it</summary>
|
||||
|
||||
```bash
|
||||
# Run the llama-openvino:server Docker container (CPU)
|
||||
@@ -685,9 +785,9 @@ llama-openvino:server --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_
|
||||
|
||||
# Run the llama-openvino:server Docker container with Intel NPU access
|
||||
docker run --rm -it -v ~/models:/models \
|
||||
--device=/dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \
|
||||
--device=/dev/accel --group-add=$(stat -c "%g" /dev/accel/accel* | head -n 1) -u $(id -u):$(id -g) \
|
||||
-p 8080:8080 --env=GGML_OPENVINO_DEVICE=NPU \
|
||||
llama-openvino:server --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --host 0.0.0.0
|
||||
llama-openvino:server --no-warmup -c 512 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --host 0.0.0.0
|
||||
|
||||
# Or Using llama-server executable
|
||||
./build/ReleaseOV/bin/llama-server -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --port 8080 -c 1024
|
||||
@@ -706,101 +806,221 @@ curl -X POST "http://localhost:8080/v1/chat/completions" -H "Content-Type: appli
|
||||
-d '{"messages":[{"role":"user","content":"Write a poem about OpenVINO"}],"max_tokens":100}' | jq .
|
||||
```
|
||||
|
||||
## GGML OpenVINO Backend Runtime Configurations
|
||||
</details>
|
||||
|
||||
The OpenVINO backend can be configured using the following environment variables at runtime to control device selection, caching, debugging, and profiling behavior.
|
||||
Boolean flags follow a uniform convention: set to a **positive integer** (e.g. `1`) to enable; unset, empty, `0`, negative, or non-numeric values are treated as disabled.
|
||||
## Runtime Configuration
|
||||
|
||||
| Variable | Type | Default | Description |
|
||||
|-----------------------------------|-----------|------------|-------------------------------------------------------------------------------------------------------------|
|
||||
| `GGML_OPENVINO_DEVICE` | String | `CPU` | Specify the target device (CPU, GPU, NPU). On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU. A device that is not available is an error (no fallback to CPU), and the error message lists the available OpenVINO devices with their names. See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html). When set to **NPU**, static compilation mode is enabled for optimal performance. |
|
||||
| `GGML_OPENVINO_CACHE_DIR` | String | `not set` | Directory for OpenVINO's separate plugin cache. On NPU, this sets `NPUW_CACHE_DIR`. |
|
||||
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` | String | `not set` | Directory for standalone compiled blobs with weights. Dynamic CPU/GPU graphs can import matching blobs on later runs. |
|
||||
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY` | Boolean | `0` | Require an existing compiled blob and skip weight uploads and compilation. Requires Linux or Windows mmap loading and a full dynamic CPU/GPU graph on OpenVINO. |
|
||||
| `GGML_OPENVINO_PREFILL_CHUNK_SIZE`| Integer | `256` | Token chunk size for **NPU** prefill (NPU-only; ignored on CPU/GPU). Must be a positive integer; otherwise the default is used. |
|
||||
| `GGML_OPENVINO_NPU_COMPILE_CONFIG` | String | `not set` | NPU-only compiler mode parameters forwarded to OpenVINO as `NPU_COMPILATION_MODE_PARAMS`, for example `optimization-level=3`. |
|
||||
| `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Keep KV and supported recurrent caches inside the model. Single-slot CPU/GPU execution only. |
|
||||
| `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. |
|
||||
| `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. |
|
||||
| `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | Boolean | `0` | Disable the stateful KV-state sequence-axis relayout (relayout is on by default). It moves the KV state sequence axis from dim 1 to dim 2, so the GPU plugin can append new tokens in place instead of copying the whole state every token, and the reader side no longer transposes the whole accumulated state. Set to `1` to disable. |
|
||||
| `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. |
|
||||
| `GGML_OPENVINO_MEMORY_OPTIMIZE` | Boolean | `0` | Umbrella switch for compile-time memory reductions. Enables `GGML_OPENVINO_REDUCE_COMPILE_MEM` and, on GPU, `GGML_OPENVINO_RELEASE_WEIGHTS` unless those fine-grained variables are explicitly set. |
|
||||
| `GGML_OPENVINO_REDUCE_COMPILE_MEM`| Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` | Reduce compile-time host memory use by streaming weight requantization and avoiding extra weight-node materialization where possible. Set explicitly to override the umbrella switch. |
|
||||
| `GGML_OPENVINO_RELEASE_WEIGHTS` | Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` on GPU | GPU-only. Release host weight buffers after the compiled model cache can reuse the device/plugin copy. Requires stable graph shapes; dynamic workloads that need recompilation should leave this disabled. |
|
||||
| `GGML_OPENVINO_SPILL_DIR` | String | `not set` | Directory for a disk-backed weight buffer. When set, the repacked weight buffer is mapped from an unlinked file on this path instead of anonymous memory, so its pages are reclaimable under memory pressure instead of staying pinned, cutting the load-time host memory peak. Must point at real storage; a tmpfs mount (e.g. `/tmp` on many systems) backs it with RAM and makes the peak worse. |
|
||||
| `GGML_OPENVINO_REQUANT_KQUANT` | String | `not set` | Requantize Q6_K/Q5_K weights (and matching MoE expert weights) to a 4-bit target instead of the default Q8_0_C, trading accuracy for less memory traffic. One of `q4_asym64` (Q6_K/Q5_K only, keeps a real zero point at group 64), `q4_asym64_all` (also requantizes Q4_K), `q4_sym128` (Q6_K/Q5_K only), `q4_sym128_all` (Q4_K too, drops its per-group zero point), or `native` (no requantization). |
|
||||
| `GGML_OPENVINO_PROFILING` | Integer | `0` | `1` logs execution timing; `2` or higher also enables OpenVINO and OpenCL profiling. |
|
||||
| `GGML_OPENVINO_DEBUG_NODE` | String | `not set` | Add the named graph nodes as compiled outputs for debugging. Separate multiple names with commas. |
|
||||
| `GGML_OPENVINO_MOE_OP` | Boolean | `1` | On GPU, set to `0` to keep the unfused GatherMatmul path. |
|
||||
| `GGML_OPENVINO_DUMP_CGRAPH` | Boolean | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. |
|
||||
| `GGML_OPENVINO_DUMP_IR` | Boolean | `0` | Serialize OpenVINO IR files with timestamps. |
|
||||
| `GGML_OPENVINO_DEBUG_INPUT` | Boolean | `0` | Enable input debugging and print input tensor info. |
|
||||
| `GGML_OPENVINO_DEBUG_OUTPUT` | Boolean | `0` | Enable output debugging and print output tensor info. |
|
||||
| `GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS` | Boolean | `0` | Print tensor address map once. |
|
||||
| `GGML_OPENVINO_LOG_UNSUPPORTED_OPS`| Boolean | `0` | Log warning messages with tensor details and rejection reasons for any ops not supported by the OpenVINO backend. Emits at `WARN` level (requires `--log-verbosity >= 2`, enabled by default). |
|
||||
The backend needs no environment variable: it runs on the CPU with its defaults. Set `GGML_OPENVINO_DEVICE` to use the GPU or the NPU. All other variables are optional.
|
||||
|
||||
> [!NOTE]
|
||||
> - `GGML_OPENVINO_STATEFUL_EXECUTION` is an **Experimental** feature for managing caches internally inside the OpenVINO model on CPUs and GPUs. Use a single slot (`-np 1`). KV caches retain the append-based state layout and sequence-axis optimization. Qwen3.5 adds recurrent cache states in their GGML layouts. Qwen3.5 requires an unsplit graph with model caching enabled and no recurrent rollback. A prompt starting at position 0 resets all states. State save/restore, sequence rewind, context shift, and mid-sequence graph replacement are unsupported. Stateful execution is not effective on NPUs.
|
||||
> - `GGML_OPENVINO_LOG_UNSUPPORTED_OPS` emits logs at `WARN` level (`GGML_LOG_WARN`), which requires application log verbosity `--log-verbosity >= 2` (or `-lv 2`).
|
||||
> - With `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1`, use the same compilation settings as the export run. One directory can hold blobs for different models and settings; `GGML_OPENVINO_SPILL_DIR` does not affect the cache key and is ignored in cache-only mode. See [Compiled model cache](../../ggml/src/ggml-openvino/README.md) for the workflow and restrictions.
|
||||
```bash
|
||||
export GGML_OPENVINO_DEVICE=GPU # String
|
||||
export GGML_OPENVINO_PROFILING=1 # Integer
|
||||
export GGML_OPENVINO_EXACT_ZERO_POINT=1 # Boolean: 1 (on) or 0 (off)
|
||||
|
||||
# Show the variables in effect (log verbosity 4)
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024 -lv 4 2>&1 | grep "settings from the environment"
|
||||
```
|
||||
|
||||
- The backend reads the variables once, at start. Unset or empty means the default.
|
||||
- To lower the memory a model needs, see [Tips for Memory Optimization](../../ggml/src/ggml-openvino/dev_doc/OpenVINO-ENV-VARS.md#tips-for-memory-optimization).
|
||||
- A value that is not valid stops the backend. A `GGML_OPENVINO_*` name that the backend does not know gives a warning:
|
||||
|
||||
```text
|
||||
ggml-openvino: GGML_OPENVINO_EXACT_ZERO_POINT=yes is not valid, use 1 (on) or 0 (off)
|
||||
ggml-openvino: unknown environment variable GGML_OPENVINO_DEVCE is ignored
|
||||
```
|
||||
|
||||
### Basic
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_DEVICE` | all | String | `CPU` | Target device: `CPU`, `GPU` or `NPU`. For several GPUs, see the [device notes](#4-run-inference-with-openvino-backend). Examples: `GGML_OPENVINO_DEVICE=NPU`, `GGML_OPENVINO_DEVICE=GPU.0,GPU.1`. |
|
||||
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` | all | String | `not set` | Directory for the backend's cache of compiled models. A later start with the same model and settings loads faster. See [Compiled model cache](#compiled-model-cache). Example: `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache`. |
|
||||
| `GGML_OPENVINO_NPU_CONFIG` | NPU | String | `not set` | OpenVINO properties for the NPU plugin, as comma-separated `KEY=VALUE` pairs. See [Plugin properties](#plugin-properties). Example: `GGML_OPENVINO_NPU_CONFIG="NPU_COMPILER_TYPE=PLUGIN"`. |
|
||||
| `GGML_OPENVINO_GPU_CONFIG` | GPU | String | `not set` | OpenVINO properties for the GPU plugin, as comma-separated `KEY=VALUE` pairs. See [Plugin properties](#plugin-properties). Example: `GGML_OPENVINO_GPU_CONFIG="INFERENCE_PRECISION_HINT=f16"`. |
|
||||
| `GGML_OPENVINO_CPU_CONFIG` | CPU | String | `not set` | OpenVINO properties for the CPU plugin, as comma-separated `KEY=VALUE` pairs. See [Plugin properties](#plugin-properties). Example: `GGML_OPENVINO_CPU_CONFIG="INFERENCE_PRECISION_HINT=f16"`. |
|
||||
| `GGML_OPENVINO_PROFILING` | all | Integer | `0` | `1` logs execution timing. `2` or higher also turns on OpenVINO and OpenCL profiling. Example: `GGML_OPENVINO_PROFILING=1`. |
|
||||
|
||||
### Plugin properties
|
||||
|
||||
`GGML_OPENVINO_NPU_CONFIG`, `GGML_OPENVINO_GPU_CONFIG` and `GGML_OPENVINO_CPU_CONFIG` pass OpenVINO plugin properties to the device in their name.
|
||||
|
||||
```bash
|
||||
# Format: comma-separated KEY=VALUE pairs. The backend uses the variable of the selected device only.
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_NPU_CONFIG="NPU_COMPILER_TYPE=PLUGIN,COMPILATION_NUM_THREADS=4"
|
||||
|
||||
# CPU: compute in FP16, as the GPU and the NPU do. This helps to debug the accuracy of a quantized model.
|
||||
export GGML_OPENVINO_DEVICE=CPU
|
||||
export GGML_OPENVINO_CPU_CONFIG="INFERENCE_PRECISION_HINT=f16"
|
||||
```
|
||||
|
||||
- Property names and values are those of OpenVINO, for example `YES` and `NO`. They are not the `1` and `0` of the `GGML_OPENVINO_*` variables.
|
||||
- The backend applies the variable last, so it overrides the properties that the backend sets.
|
||||
- OpenVINO checks the properties when the model loads. If it rejects one, the load fails and the error names the variable.
|
||||
- All properties and their values are in the OpenVINO documentation: [NPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.html), [GPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html), [CPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device.html).
|
||||
- NPU: the backend sets several properties by default. They are listed in the [developer documentation](../../ggml/src/ggml-openvino/dev_doc/OpenVINO-ENV-VARS.md#npu-plugin-properties).
|
||||
- GPU and CPU: the backend sets no property, except `CACHE_DIR` and `CACHE_MODE=OPTIMIZE_SIZE` when `GGML_OPENVINO_CACHE_DIR` is set.
|
||||
|
||||
`NPU_COMPILER_TYPE` selects the NPU compiler. Both forms below work; use the first one. If both are set, `GGML_OPENVINO_NPU_CONFIG` takes precedence.
|
||||
|
||||
```bash
|
||||
# Values: DRIVER (the compiler in the NPU driver; backend default), PLUGIN (the compiler in OpenVINO),
|
||||
# PREFER_PLUGIN (OpenVINO default: PLUGIN when it is available, DRIVER if not).
|
||||
export GGML_OPENVINO_NPU_CONFIG="NPU_COMPILER_TYPE=PLUGIN" # recommended form
|
||||
export GGML_OPENVINO_NPU_COMPILER_TYPE=PLUGIN # older name, still works
|
||||
```
|
||||
|
||||
### Advanced tuning
|
||||
|
||||
The defaults fit most models. The variables below trade one property for another, for example speed for accuracy.
|
||||
|
||||
<details>
|
||||
<summary><b>Advanced tuning variables</b></summary>
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_PREFILL_CHUNK_SIZE` | NPU | Integer | `256` | Number of tokens that the NPU processes in one prefill step. A larger value can make long prompts faster, and makes the first load longer. Zero or a negative value means the default. Example: `GGML_OPENVINO_PREFILL_CHUNK_SIZE=512`. |
|
||||
| `GGML_OPENVINO_NPU_TOKEN_EMBD_QUANT` | NPU | String | `q8_0_c` | How a `Q6_K` `token_embd.weight` is stored for the NPU: `q8_0_c` (8-bit, one scale per row), `f16`, `q4_0_128` (4-bit, one scale per 128 weights) or `q4_0_c` (4-bit, one scale per row). The 4-bit layouts use less memory at some cost in accuracy; `f16` uses more memory. Keep the default unless memory is tight. |
|
||||
| `GGML_OPENVINO_GDN_DECOMPOSE` | all | Boolean | auto | Translate `GATED_DELTA_NET` (Qwen3.5) into MatMul operations instead of the OpenVINO `GatedDeltaNet` op. Unset: on for the Lunar Lake NPU, off elsewhere. `0` turns it off, `1` turns it on for any device. Try `1` for Qwen3.5 on the Panther Lake NPU. |
|
||||
| `GGML_OPENVINO_MANUAL_GQA_ATTN` | CPU, GPU | Boolean | on for GPU | Build grouped-query attention from separate MatMul operations. Unset: on for GPU, off for CPU and NPU. `1` turns it on, `0` turns it off. Try `1` on the CPU for faster decode. |
|
||||
| `GGML_OPENVINO_REQUANT_KQUANT` | CPU, GPU | String | `not set` | Requantize `Q6_K` and `Q5_K` weights to a 4-bit layout instead of `Q8_0_C`: faster decode for lower accuracy. One of `q4_asym64`, `q4_asym64_all` (also `Q4_K`), `q4_sym128`, `q4_sym128_all` (also `Q4_K`) or `native` (no requantization). On GPU with `Q4_K_M` files use `q4_asym64_all`, because `q4_asym64` does not compile there. |
|
||||
| `GGML_OPENVINO_EXACT_ZERO_POINT` | CPU, GPU | Boolean | `0` | Keep the exact zero point of asymmetric weights (`Q4_1`, `Q4_K`, `Q5_1`, `Q5_K`) instead of rounding it to an integer. This improves accuracy and lowers speed. Use it when accuracy matters more than speed. |
|
||||
|
||||
</details>
|
||||
|
||||
### Compiled model cache
|
||||
|
||||
The cache keeps compiled models on disk, so later starts of the same model load faster. Use the same model and the same settings in every run.
|
||||
|
||||
```bash
|
||||
# CPU or GPU: the first run compiles the model and writes the cache. Later runs read it.
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024
|
||||
|
||||
# CPU or GPU, later runs (optional): require the cached model and skip the weight upload and the compilation.
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1
|
||||
|
||||
# NPU: add the weightless bank.
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache
|
||||
export GGML_OPENVINO_NPU_WEIGHTLESS_BANK=1
|
||||
./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 512
|
||||
|
||||
# NPU, later runs (optional): use less memory. The run fails if a cached model is missing.
|
||||
export GGML_OPENVINO_SELF_CONTAINED_MMAP=1
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary><b>Cache variables</b></summary>
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY` | CPU, GPU | Boolean | `0` | Require an existing cached model and skip the weight upload and the compilation. Needs mmap model loading (the default on Linux and Windows) and a model that runs fully on OpenVINO. |
|
||||
| `GGML_OPENVINO_NPU_WEIGHTLESS_BANK` | NPU | Boolean | `0` | Requires `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR`. Write the prefill and decode models without weights, plus one shared weights file. Later runs read the weights from that file and keep one copy in memory. |
|
||||
| `GGML_OPENVINO_SELF_CONTAINED_MMAP` | NPU | Boolean | `0` | Later runs only. Requires `GGML_OPENVINO_NPU_WEIGHTLESS_BANK=1` and an existing cache. Use the memory-mapped GGUF file as the model buffer, which lowers the host memory peak. Fails if a cached model is missing; run once without it to build the cache. |
|
||||
| `GGML_OPENVINO_CACHE_DIR` | all | String | `not set` | Directory for the cache of the OpenVINO plugin itself. `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` is the recommended cache. |
|
||||
|
||||
- One directory can hold the cached models of different models and settings.
|
||||
- NPU: each prefill length enters the cache in a run without `GGML_OPENVINO_SELF_CONTAINED_MMAP`.
|
||||
- See [Compiled model cache](../../ggml/src/ggml-openvino/README.md#compiled-model-cache) for the workflow and the restrictions.
|
||||
|
||||
</details>
|
||||
|
||||
### Deprecated
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_STATEFUL_EXECUTION` | CPU, GPU | Boolean | `0` | **Deprecated**, to be removed. Keep the KV cache inside the OpenVINO model. Use the default, stateless execution (`0`). |
|
||||
|
||||
<details>
|
||||
<summary>Limits of stateful execution</summary>
|
||||
|
||||
- CPU and GPU only; it has no effect on the NPU.
|
||||
- One slot only (`-np 1`).
|
||||
- State save and restore, sequence rewind, context shift and recurrent rollback are not supported.
|
||||
- Qwen3.5 needs a graph that runs fully on OpenVINO, with model caching on.
|
||||
- A prompt that starts at position 0 resets all states.
|
||||
|
||||
</details>
|
||||
|
||||
### Debugging and development
|
||||
|
||||
Debug output, switches that turn a default optimization off, experimental options and older variable names are in the [developer documentation](../../ggml/src/ggml-openvino/dev_doc/OpenVINO-ENV-VARS.md).
|
||||
|
||||
### Example Usage
|
||||
|
||||
#### GPU Inference with Profiling
|
||||
|
||||
```bash
|
||||
# If the system has multiple GPUs, use GPU.0 or GPU.1 to explicitly target a specific GPU.
|
||||
|
||||
# Linux
|
||||
export GGML_OPENVINO_CACHE_DIR=/tmp/ov_cache
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache
|
||||
export GGML_OPENVINO_PROFILING=1
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
export GGML_OPENVINO_STATEFUL_EXECUTION=1
|
||||
|
||||
./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -n 50 "The story of AI is "
|
||||
```
|
||||
|
||||
# Windows Command Line
|
||||
set GGML_OPENVINO_CACHE_DIR=C:\tmp\ov_cache
|
||||
<details>
|
||||
<summary><b>Windows</b></summary>
|
||||
|
||||
```cmd
|
||||
REM Windows Command Prompt
|
||||
set GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=C:\tmp\ov_cache
|
||||
set GGML_OPENVINO_PROFILING=1
|
||||
set GGML_OPENVINO_DEVICE=GPU
|
||||
set GGML_OPENVINO_STATEFUL_EXECUTION=1
|
||||
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
|
||||
```
|
||||
|
||||
```powershell
|
||||
# Windows PowerShell
|
||||
$env:GGML_OPENVINO_CACHE_DIR = "C:\tmp\ov_cache"
|
||||
$env:GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR = "C:\tmp\ov_cache"
|
||||
$env:GGML_OPENVINO_PROFILING = "1"
|
||||
$env:GGML_OPENVINO_DEVICE = "GPU"
|
||||
$env:GGML_OPENVINO_STATEFUL_EXECUTION = "1"
|
||||
|
||||
build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is "
|
||||
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
## Known Limitations
|
||||
|
||||
**General (all devices)**
|
||||
|
||||
- Llama.cpp OpenVINO backend currently supports a subset of GGML ops and text-only models. Unsupported ops or unsupported op shapes/cases fail during OpenVINO translation.
|
||||
- Multimodal features (audio/image/video) are a work in progress.
|
||||
- Limited Embedding and Reranking model support.
|
||||
- Embedding models: reranking models are not validated.
|
||||
- Llama.cpp tool coverage across CPU/GPU/NPU is not uniform.
|
||||
|
||||
**Tool-specific**
|
||||
|
||||
- `llama-bench`: requires `-fa 1` (flash-attention).
|
||||
- `llama-cli --context-shift`: stateless only (`GGML_OPENVINO_STATEFUL_EXECUTION=0`). In stateful mode the KV cache is owned by the OpenVINO model and cannot be shifted externally.
|
||||
- `llama-server`: only one chat session/thread when `GGML_OPENVINO_STATEFUL_EXECUTION=1`.
|
||||
|
||||
**GPU-specific**
|
||||
|
||||
- `llama-server -np > 1`: concurrent requests are batched together, which may slightly reduce per-request throughput.
|
||||
- Multiple GPUs (`GGML_OPENVINO_DEVICE=GPU.0,GPU.1` with `-sm layer`):
|
||||
- `-sm tensor` is not supported.
|
||||
- Copies between GPUs go through host memory. There is no asynchronous copy and no peer-to-peer copy.
|
||||
- Only GPUs can be combined. A list with several NPUs, or with a GPU and the CPU or the NPU, is rejected.
|
||||
- MoE models use the GPU plugin's fused MoE op, which runs in f16 only:
|
||||
- granitemoe models (e.g. `granite-3.1-1b-a400m`) give wrong output.
|
||||
- **Workaround:** `GGML_OPENVINO_MOE_OP=0`. Output is correct, but prompt processing is much slower.
|
||||
- `GGML_OPENVINO_GPU_CONFIG="INFERENCE_PRECISION_HINT=f32"` fails to compile.
|
||||
- `llama-perplexity` with MoE models: set `-b` to the same value as `-c`, so that each batch holds one sequence.
|
||||
|
||||
**NPU-specific**
|
||||
|
||||
- Default context resolves to the model's training context (e.g. 131072 for Llama 3.2 1B), which can OOM or fail or degrade performance on NPU. Inspect the resolved value with `-lv 3`.
|
||||
- **Workaround:** Pass an explicit `-c <N>`, e.g. `-c 1024`.
|
||||
- **Workaround:** Pass an explicit `-c <N>`, e.g. `-c 512`.
|
||||
- NPU device uses a static graph with a fixed prefill chunk size (defaults to 256), configurable with `GGML_OPENVINO_PREFILL_CHUNK_SIZE`. Large prefill/batch settings may need tuning.
|
||||
- `llama-server -np > 1` (multiple parallel sequences) is not supported.
|
||||
- `llama-perplexity`: requires `-b 512` or smaller.
|
||||
- Flash attention must stay on, which is the default (`-fa auto`). With `-fa off`, the run fails on the NPU.
|
||||
- `llama-embedding` fails on the NPU.
|
||||
- Lunar Lake NPU: decode slows down sharply when the context is large. The context size at which this starts depends on the model.
|
||||
- **Workaround:** Pass the smallest context that fits the use, e.g. `-c 512`, or `-c 256` for larger models.
|
||||
- Panther Lake NPU: the load of a Qwen3.5 model can end with a segmentation fault in the NPU compiler of OpenVINO 2026.4.1.
|
||||
- **Workaround:** Set `GGML_OPENVINO_NPU_CONFIG=COMPILATION_NUM_THREADS=1`.
|
||||
- Panther Lake NPU: `GGML_OPENVINO_NPU_TOKEN_EMBD_QUANT=f16` and `q4_0_128` can give wrong output. Keep the default.
|
||||
- Lunar Lake NPU: `granite-4.0-1b` gives wrong output. The activations of the model exceed the FP16 range, and the NPU computes in FP16. The model runs correctly on CPU and GPU.
|
||||
- Lunar Lake NPU: `gemma-4-12B` does not run. The first inference fails with `ZE_RESULT_ERROR_UNINITIALIZED` from the NPU driver. The model runs on CPU and GPU.
|
||||
- The first load of a model on the NPU compiles it for the device, which can take minutes for large models.
|
||||
- **Workaround:** Use the [compiled model cache](#compiled-model-cache).
|
||||
|
||||
> [!NOTE]
|
||||
> The OpenVINO backend is actively under development. Fixes and improvements are underway, and this document will continue to be updated.
|
||||
|
||||
@@ -17,6 +17,7 @@ Supported models:
|
||||
- [MiniMax Music 3](#minimax-music-3) (songs)
|
||||
- [Kokoro](#kokoro)
|
||||
- [tiny-voca](#tiny-voca)
|
||||
- [VoxCPM2](#voxcpm2)
|
||||
|
||||
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
|
||||
|
||||
@@ -467,3 +468,45 @@ pip install git+https://github.com/n4ze3m/dizione
|
||||
python convert_hf_to_gguf.py path/to/tiny-voca --outfile tiny-voca.gguf
|
||||
python convert_hf_to_gguf.py path/to/tiny-voca --mmproj --outfile mmproj-tiny-voca.gguf
|
||||
```
|
||||
|
||||
## VoxCPM2
|
||||
|
||||
[VoxCPM2](https://huggingface.co/openbmb/VoxCPM2): a MiniCPM4 backbone (2B) makes one hidden state per frame, without an audio token. The mmproj turns it into a patch of 4 continuous latents: FSQ, an 8-layer residual LM (run with libllama, one context per sequence), then the LocDiT flow matching. The causal AudioVAE decodes the latents to 48 kHz audio.
|
||||
|
||||
Available params:
|
||||
- The language is the one of the text, `--tts-lang` is not used. A voice description in parentheses at the start of the text designs the voice, for example `(A young woman, gentle and warm)Hello!`
|
||||
- Voice cloning: `--tts-speaker-file` with the reference clip (resampled to 16 kHz, truncated to the first 30 s). `--tts-speaker-text` is not used, the transcript of the clip is not needed
|
||||
- `--temp` scales the noise of the flow matching (default 1.0), `--top-k` and `--top-p` are not used
|
||||
- The flow matching uses 10 Euler steps and classifier-free guidance with scale 2.0, like the reference. `llama-server` takes `flow_steps` (2 to 32) and `cfg_scale` to change them
|
||||
- Each frame is 4 latents, 7680 samples (160 ms). Generation stops on the stop head of the model, at least 4 frames, at most 6 frames per text token + 10 (the limit of the reference)
|
||||
- Not supported yet: the continuation mode of the reference (prompt audio with its transcript)
|
||||
- Not supported yet: the bad case retry of the reference (`retry_badcase`, a new seed when the audio reaches 6 frames per text token). Short prompts may then run long and repeat some words, try another seed
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m VoxCPM2.gguf \
|
||||
-mm mmproj-VoxCPM2.gguf \
|
||||
-p "(A calm male narrator)The quick brown fox jumps over the lazy dog." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Voice cloning:
|
||||
|
||||
```sh
|
||||
llama-tts -m VoxCPM2.gguf \
|
||||
-mm mmproj-VoxCPM2.gguf \
|
||||
--tts-speaker-file reference.wav \
|
||||
-p "Hello, this is a cloned voice." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The mmproj holds LocEnc, LocDiT and the residual LM layers (loaded with libllama as a Granite text model with RoPE off) in the `--outtype` type, and the AudioVAE (`audiovae.pth`, encoder for voice cloning, decoder for the audio) always in F32:
|
||||
|
||||
```sh
|
||||
hf download openbmb/VoxCPM2 --local-dir VoxCPM2
|
||||
python convert_hf_to_gguf.py VoxCPM2 --outfile VoxCPM2.gguf
|
||||
python convert_hf_to_gguf.py VoxCPM2 --mmproj --outfile mmproj-VoxCPM2.gguf
|
||||
```
|
||||
|
||||
+1
-1
@@ -90,7 +90,7 @@ Legend:
|
||||
| PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ |
|
||||
| POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
|
||||
+128
-128
@@ -1155,134 +1155,134 @@
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[1,8,1,3],nr23=[1,1],r=2,v=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -8,10 +8,10 @@ extern "C" {
|
||||
|
||||
#define RPC_PROTO_MAJOR_VERSION 10
|
||||
#define RPC_PROTO_MINOR_VERSION 0
|
||||
#define RPC_PROTO_PATCH_VERSION 0
|
||||
#define RPC_PROTO_PATCH_VERSION 1
|
||||
|
||||
#ifdef __cplusplus
|
||||
static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
|
||||
static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
|
||||
#endif
|
||||
|
||||
#define GGML_RPC_MAX_SERVERS 16
|
||||
|
||||
@@ -429,6 +429,8 @@ extern "C" {
|
||||
|
||||
GGML_OP_GLU,
|
||||
|
||||
GGML_OP_SLEEP,
|
||||
|
||||
GGML_OP_COUNT,
|
||||
};
|
||||
|
||||
@@ -2599,6 +2601,12 @@ extern "C" {
|
||||
int n_tasks,
|
||||
void * userdata);
|
||||
|
||||
// Busy-wait operator for scheduling and backend testing.
|
||||
GGML_API struct ggml_tensor * ggml_sleep(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a, // wait after this tensor has executed, return value is simply a full view of this tensor
|
||||
int32_t us); // amount of time to wait in microseconds
|
||||
|
||||
// loss function
|
||||
|
||||
GGML_API struct ggml_tensor * ggml_cross_entropy_loss(
|
||||
|
||||
@@ -231,7 +231,7 @@ if (GGML_OPENMP_FETCH)
|
||||
message(FATAL_ERROR "GGML_OPENMP_FETCH currently requires Clang on Windows")
|
||||
endif()
|
||||
|
||||
set(GGML_OPENMP_LLVM_VERSION "20.1.8")
|
||||
set(GGML_OPENMP_LLVM_VERSION "22.1.8")
|
||||
string(REGEX MATCH "^[0-9]+" GGML_OPENMP_LLVM_VERSION_MAJOR "${GGML_OPENMP_LLVM_VERSION}")
|
||||
string(REGEX MATCH "^[0-9]+" GGML_OPENMP_COMPILER_VERSION_MAJOR "${CMAKE_C_COMPILER_VERSION}")
|
||||
if (NOT GGML_OPENMP_COMPILER_VERSION_MAJOR STREQUAL GGML_OPENMP_LLVM_VERSION_MAJOR)
|
||||
@@ -242,11 +242,11 @@ if (GGML_OPENMP_FETCH)
|
||||
if (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(amd64|x86_64)$")
|
||||
set(GGML_OPENMP_ARCH "x64")
|
||||
set(GGML_OPENMP_INSTALLER_SUFFIX "win64")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "3197846a2b19063687dd56e93e34cd941e3548d907f23a6131571321bdf9fe7b")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "16e5709785fef73c854646241c4a92c5cd574318d1b33c63330dd7721903e55c")
|
||||
elseif (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$")
|
||||
set(GGML_OPENMP_ARCH "arm64")
|
||||
set(GGML_OPENMP_INSTALLER_SUFFIX "woa64")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "7c4ac97eb2ae6b960ca5f9caf3ff6124c8d2a18cc07a7840a4d2ea15537bad8e")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "76f44ef1ba6eeb5a65904e9500f042f588fade49952778ce48f0374daa934396")
|
||||
else()
|
||||
message(FATAL_ERROR "GGML_OPENMP_FETCH does not support ${CMAKE_SYSTEM_PROCESSOR}")
|
||||
endif()
|
||||
|
||||
@@ -2584,6 +2584,7 @@ bool ggml_op_alloc_size_may_expand(enum ggml_op op) {
|
||||
case GGML_OP_ARGSORT:
|
||||
case GGML_OP_TOP_K:
|
||||
case GGML_OP_ARGMAX:
|
||||
case GGML_OP_CONV_2D:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -262,13 +262,22 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
v128_t v0_0 = wasm_v128_load(x0->qs);
|
||||
v128_t v0_0l = wasm_v128_and(v0_0, m4b);
|
||||
v128_t v0_0h = wasm_u8x16_shr(v0_0, 4);
|
||||
v128_t v0_0ls = wasm_i8x16_sub(v0_0l, s8b);
|
||||
v128_t v0_0hs = wasm_i8x16_sub(v0_0h, s8b);
|
||||
|
||||
// Load y0 vectors
|
||||
v128_t y0_l = wasm_v128_load(y0->qs);
|
||||
v128_t y0_h = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// unsigned nibbles (0..15) in the 7-bit operand, then subtract 8*sum(y)
|
||||
v128_t dp0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_l, v0_0l, wasm_i32x4_splat(0));
|
||||
dp0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_h, v0_0h, dp0);
|
||||
v128_t ys0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_l, s8b, wasm_i32x4_splat(0));
|
||||
ys0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_h, s8b, ys0);
|
||||
dp0 = wasm_i32x4_sub(dp0, ys0);
|
||||
# else
|
||||
v128_t v0_0ls = wasm_i8x16_sub(v0_0l, s8b);
|
||||
v128_t v0_0hs = wasm_i8x16_sub(v0_0h, s8b);
|
||||
|
||||
// Extend to i16x8 and compute dot products
|
||||
v128_t dx0l = wasm_i16x8_extend_low_i8x16(v0_0ls);
|
||||
v128_t dx0h = wasm_i16x8_extend_high_i8x16(v0_0ls);
|
||||
@@ -290,18 +299,27 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_dot_i16x8(dx0hh, dy0hh)
|
||||
)
|
||||
);
|
||||
# endif
|
||||
|
||||
// Load and process x1
|
||||
v128_t v0_1 = wasm_v128_load(x1->qs);
|
||||
v128_t v0_1l = wasm_v128_and(v0_1, m4b);
|
||||
v128_t v0_1h = wasm_u8x16_shr(v0_1, 4);
|
||||
v128_t v0_1ls = wasm_i8x16_sub(v0_1l, s8b);
|
||||
v128_t v0_1hs = wasm_i8x16_sub(v0_1h, s8b);
|
||||
|
||||
// Load y1 vectors
|
||||
v128_t y1_l = wasm_v128_load(y1->qs);
|
||||
v128_t y1_h = wasm_v128_load(y1->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// unsigned nibbles (0..15) in the 7-bit operand, then subtract 8*sum(y)
|
||||
v128_t dp1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_l, v0_1l, wasm_i32x4_splat(0));
|
||||
dp1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_h, v0_1h, dp1);
|
||||
v128_t ys1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_l, s8b, wasm_i32x4_splat(0));
|
||||
ys1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_h, s8b, ys1);
|
||||
dp1 = wasm_i32x4_sub(dp1, ys1);
|
||||
# else
|
||||
v128_t v0_1ls = wasm_i8x16_sub(v0_1l, s8b);
|
||||
v128_t v0_1hs = wasm_i8x16_sub(v0_1h, s8b);
|
||||
// Extend to i16x8 and compute dot products
|
||||
v128_t dx1l = wasm_i16x8_extend_low_i8x16(v0_1ls);
|
||||
v128_t dx1h = wasm_i16x8_extend_high_i8x16(v0_1ls);
|
||||
@@ -323,6 +341,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_dot_i16x8(dx1hh, dy1hh)
|
||||
)
|
||||
);
|
||||
# endif
|
||||
|
||||
// Accumulate results with scaling
|
||||
float scale0 = GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d);
|
||||
@@ -461,10 +480,18 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
// extract the 5th bit
|
||||
memcpy(&qh_, x0->qh, sizeof(qh_));
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// the high bit as is: unsigned 5-bit quants for the 7-bit operand
|
||||
tmp[0] = table_b2b_0[(qh_ >> 0) & 0xFF];
|
||||
tmp[1] = table_b2b_0[(qh_ >> 8) & 0xFF];
|
||||
tmp[2] = table_b2b_0[(qh_ >> 16) & 0xFF];
|
||||
tmp[3] = table_b2b_0[(qh_ >> 24) ];
|
||||
# else
|
||||
tmp[0] = table_b2b_1[(qh_ >> 0) & 0xFF];
|
||||
tmp[1] = table_b2b_1[(qh_ >> 8) & 0xFF];
|
||||
tmp[2] = table_b2b_1[(qh_ >> 16) & 0xFF];
|
||||
tmp[3] = table_b2b_1[(qh_ >> 24) ];
|
||||
# endif
|
||||
|
||||
const v128_t qhl = wasm_v128_load(tmp + 0);
|
||||
const v128_t qhh = wasm_v128_load(tmp + 2);
|
||||
@@ -475,14 +502,29 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
const v128_t v0l = wasm_v128_and (v0, m4b);
|
||||
const v128_t v0h = wasm_u8x16_shr(v0, 4);
|
||||
|
||||
// add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
|
||||
const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);
|
||||
const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);
|
||||
|
||||
// load y
|
||||
const v128_t v1l = wasm_v128_load(y0->qs);
|
||||
const v128_t v1h = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// unsigned 5-bit quants (0..31) in the 7-bit operand, then subtract 16*sum(y)
|
||||
const v128_t s16b = wasm_i8x16_splat(0x10);
|
||||
const v128_t v0lu = wasm_v128_or(v0l, qhl);
|
||||
const v128_t v0hu = wasm_v128_or(v0h, qhh);
|
||||
|
||||
v128_t dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1l, v0lu, wasm_i32x4_splat(0));
|
||||
dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1h, v0hu, dp);
|
||||
v128_t ys = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1l, s16b, wasm_i32x4_splat(0));
|
||||
ys = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1h, s16b, ys);
|
||||
dp = wasm_i32x4_sub(dp, ys);
|
||||
|
||||
sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(dp),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# else
|
||||
// add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
|
||||
const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);
|
||||
const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);
|
||||
|
||||
// int8x16 -> int16x8
|
||||
const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);
|
||||
const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);
|
||||
@@ -502,6 +544,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),
|
||||
wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# endif
|
||||
}
|
||||
|
||||
sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
|
||||
@@ -578,6 +621,13 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
const v128_t v1l = wasm_v128_load(y0->qs);
|
||||
const v128_t v1h = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1l, v0lf, wasm_i32x4_splat(0));
|
||||
dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1h, v0hf, dp);
|
||||
|
||||
sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(dp),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# else
|
||||
// int8x16 -> int16x8
|
||||
const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);
|
||||
const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);
|
||||
@@ -597,6 +647,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),
|
||||
wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# endif
|
||||
}
|
||||
|
||||
sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
|
||||
@@ -642,6 +693,17 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
const v128_t y0_0 = wasm_v128_load(y0->qs);
|
||||
const v128_t y0_1 = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// |x| (0..127) in the 7-bit operand and y*sign(x) in the other; relies on
|
||||
// q8_0 values being in -127..127, as ggml's quantizers produce
|
||||
const v128_t zero = wasm_i8x16_splat(0);
|
||||
const v128_t sx0 = wasm_i8x16_lt(x0_0, zero);
|
||||
const v128_t sx1 = wasm_i8x16_lt(x0_1, zero);
|
||||
const v128_t ys0 = wasm_i8x16_sub(wasm_v128_xor(y0_0, sx0), sx0);
|
||||
const v128_t ys1 = wasm_i8x16_sub(wasm_v128_xor(y0_1, sx1), sx1);
|
||||
v128_t sum_dots = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(ys0, wasm_i8x16_abs(x0_0), wasm_i32x4_splat(0));
|
||||
sum_dots = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(ys1, wasm_i8x16_abs(x0_1), sum_dots);
|
||||
# else
|
||||
// Extend 8-bit to 16-bit
|
||||
const v128_t x0_0l = wasm_i16x8_extend_low_i8x16(x0_0);
|
||||
const v128_t x0_0h = wasm_i16x8_extend_high_i8x16(x0_0);
|
||||
@@ -661,7 +723,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
|
||||
// Sum all dot products
|
||||
const v128_t sum_dots = wasm_i32x4_add(wasm_i32x4_add(dx0_0, dx0_1), wasm_i32x4_add(dx1_0, dx1_1));
|
||||
|
||||
# endif
|
||||
// Convert to float and accumulate
|
||||
const float scale = GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d);
|
||||
sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(sum_dots), wasm_f32x4_splat(scale)));
|
||||
@@ -750,6 +812,10 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
v128_t q2_shift_1 = wasm_u8x16_shr(q2_1, shift);
|
||||
v128_t q2_bits_1 = wasm_v128_and(q2_shift_1, wasm_i8x16_splat(0x03));
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t p_sum0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_0, q2_bits_0, wasm_i32x4_splat(0));
|
||||
v128_t p_sum1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_1, q2_bits_1, wasm_i32x4_splat(0));
|
||||
# else
|
||||
// Calculate dot products
|
||||
v128_t p0 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q8_0),
|
||||
@@ -768,10 +834,14 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q2_bits_1)
|
||||
);
|
||||
|
||||
v128_t p_sum0 = wasm_i32x4_add(p0, p1);
|
||||
v128_t p_sum1 = wasm_i32x4_add(p2, p3);
|
||||
# endif
|
||||
|
||||
// Accumulate scaled results
|
||||
v128_t scaled = wasm_i32x4_add(
|
||||
wasm_i32x4_mul(wasm_i32x4_add(p0, p1), wasm_i32x4_splat(d0)),
|
||||
wasm_i32x4_mul(wasm_i32x4_add(p2, p3), wasm_i32x4_splat(d1))
|
||||
wasm_i32x4_mul(p_sum0, wasm_i32x4_splat(d0)),
|
||||
wasm_i32x4_mul(p_sum1, wasm_i32x4_splat(d1))
|
||||
);
|
||||
|
||||
isum_vec = wasm_i32x4_add(isum_vec, scaled);
|
||||
@@ -817,7 +887,84 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
|
||||
const int nb = n / QK_K;
|
||||
|
||||
#if defined __wasm_simd128__
|
||||
#if defined(__wasm_relaxed_simd__)
|
||||
// Unsigned 3-bit quants (0..7) in the relaxed dot's 7-bit operand, q8 in
|
||||
// the other; the -4 offset is taken off with the q8 block sums.
|
||||
const v128_t m3 = wasm_i8x16_splat(0x03);
|
||||
const v128_t m1 = wasm_i8x16_splat(0x01);
|
||||
const v128_t zero = wasm_i32x4_splat(0);
|
||||
uint32_t auxs[4];
|
||||
|
||||
float sumf = 0;
|
||||
|
||||
for (int i = 0; i < nb; ++i) {
|
||||
const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
|
||||
|
||||
const uint8_t * GGML_RESTRICT q3 = x[i].qs;
|
||||
const int8_t * GGML_RESTRICT q8 = y[i].qs;
|
||||
|
||||
// Extract scales
|
||||
memcpy(auxs, x[i].scales, 12);
|
||||
uint32_t tmp = auxs[2];
|
||||
auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);
|
||||
auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);
|
||||
auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);
|
||||
auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);
|
||||
|
||||
const v128_t vsc = wasm_i8x16_sub(wasm_v128_load(auxs), wasm_i8x16_splat(32));
|
||||
const v128_t vsc_l = wasm_i16x8_extend_low_i8x16(vsc);
|
||||
const v128_t vsc_h = wasm_i16x8_extend_high_i8x16(vsc);
|
||||
|
||||
// sum(scale * bsum), for the -4 offset
|
||||
const v128_t vmins = wasm_i32x4_add(
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums), vsc_l),
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums + 8), vsc_h));
|
||||
|
||||
const v128_t hm0 = wasm_v128_load(x[i].hmask);
|
||||
const v128_t hm1 = wasm_v128_load(x[i].hmask + 16);
|
||||
|
||||
v128_t acc = zero;
|
||||
v128_t acc2 = zero;
|
||||
|
||||
for (int j = 0; j < QK_K/128; ++j) {
|
||||
const v128_t q3_0 = wasm_v128_load(q3);
|
||||
const v128_t q3_1 = wasm_v128_load(q3 + 16);
|
||||
q3 += 32;
|
||||
|
||||
const v128_t s32a = wasm_i32x4_extend_low_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
const v128_t s32b = wasm_i32x4_extend_high_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
const v128_t s32[2] = { s32a, s32b };
|
||||
|
||||
for (int g = 0; g < 4; ++g) {
|
||||
// 2 low bits at shift 2g, high bit 4j+g of hmask moved to bit 2
|
||||
// (16-bit shifts are safe on masked bytes)
|
||||
const int hb = 4*j + g;
|
||||
const v128_t u0 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(q3_0, 2*g), m3),
|
||||
wasm_i16x8_shl(wasm_v128_and(wasm_u16x8_shr(hm0, hb), m1), 2));
|
||||
const v128_t u1 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(q3_1, 2*g), m3),
|
||||
wasm_i16x8_shl(wasm_v128_and(wasm_u16x8_shr(hm1, hb), m1), 2));
|
||||
|
||||
const v128_t p0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8), u0, zero);
|
||||
const v128_t p1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 16), u1, zero);
|
||||
q8 += 32;
|
||||
|
||||
const v128_t sv = s32[g >> 1];
|
||||
const v128_t sc0 = (g & 1) ? wasm_i32x4_shuffle(sv, sv, 2, 2, 2, 2) : wasm_i32x4_shuffle(sv, sv, 0, 0, 0, 0);
|
||||
const v128_t sc1 = (g & 1) ? wasm_i32x4_shuffle(sv, sv, 3, 3, 3, 3) : wasm_i32x4_shuffle(sv, sv, 1, 1, 1, 1);
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p0, sc0));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p1, sc1));
|
||||
}
|
||||
}
|
||||
|
||||
acc = wasm_i32x4_sub(wasm_i32x4_add(acc, acc2), wasm_i32x4_shl(vmins, 2));
|
||||
const int32_t isum = wasm_i32x4_extract_lane(acc, 0) + wasm_i32x4_extract_lane(acc, 1) +
|
||||
wasm_i32x4_extract_lane(acc, 2) + wasm_i32x4_extract_lane(acc, 3);
|
||||
sumf += d * isum;
|
||||
}
|
||||
|
||||
*s = sumf;
|
||||
|
||||
#elif defined __wasm_simd128__
|
||||
int8_t aux8[QK_K];
|
||||
float sums[8] = {0};
|
||||
uint32_t auxs[4];
|
||||
@@ -986,6 +1133,14 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
q8 += 64;
|
||||
|
||||
// Low nibble products
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t vacc1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x0, q4l0, wasm_i32x4_splat(0)
|
||||
);
|
||||
vacc1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x1, q4l1, vacc1
|
||||
);
|
||||
# else
|
||||
v128_t vacc1 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q4l0),
|
||||
wasm_i16x8_extend_low_i8x16(q8x0)
|
||||
@@ -1002,8 +1157,17 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q4l1),
|
||||
wasm_i16x8_extend_high_i8x16(q8x1)
|
||||
));
|
||||
# endif
|
||||
|
||||
// High nibble products
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t vacc2 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x2, q4h0, wasm_i32x4_splat(0)
|
||||
);
|
||||
vacc2 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x3, q4h1, vacc2
|
||||
);
|
||||
# else
|
||||
v128_t vacc2 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q4h0),
|
||||
wasm_i16x8_extend_low_i8x16(q8x2)
|
||||
@@ -1020,6 +1184,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q4h1),
|
||||
wasm_i16x8_extend_high_i8x16(q8x3)
|
||||
));
|
||||
# endif
|
||||
|
||||
// Accumulate scaled results
|
||||
int32_t vacc1_sum = wasm_i32x4_extract_lane(vacc1, 0) + wasm_i32x4_extract_lane(vacc1, 1) +
|
||||
@@ -1127,6 +1292,11 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
v128_t q8_3 = wasm_v128_load(q8 + 48);
|
||||
q8 += 64;
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// Process low quants using relaxed SIMD
|
||||
v128_t sum_low = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_0, q5l_0, wasm_i32x4_splat(0));
|
||||
sum_low = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_1, q5l_1, sum_low);
|
||||
# else
|
||||
// Process low quants
|
||||
v128_t pl0 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q5l_0),
|
||||
@@ -1145,7 +1315,13 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q8_1)
|
||||
));
|
||||
v128_t sum_low = wasm_i32x4_add(pl0, pl1);
|
||||
# endif
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// Process high quants using relaxed SIMD
|
||||
v128_t sum_high = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_2, q5h_0, wasm_i32x4_splat(0));
|
||||
sum_high = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_3, q5h_1, sum_high);
|
||||
# else
|
||||
// Process high quants
|
||||
v128_t ph0 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q5h_0),
|
||||
@@ -1164,6 +1340,7 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q8_3)
|
||||
));
|
||||
v128_t sum_high = wasm_i32x4_add(ph0, ph1);
|
||||
# endif
|
||||
|
||||
// Accumulate with scale factors
|
||||
int32_t sl = wasm_i32x4_extract_lane(sum_low, 0) + wasm_i32x4_extract_lane(sum_low, 1) +
|
||||
@@ -1204,7 +1381,78 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
|
||||
const int nb = n / QK_K;
|
||||
|
||||
#if defined __wasm_simd128__
|
||||
#if defined(__wasm_relaxed_simd__)
|
||||
const v128_t m4 = wasm_i8x16_splat(0x0F);
|
||||
const v128_t m2 = wasm_i8x16_splat(0x03);
|
||||
const v128_t zero = wasm_i32x4_splat(0);
|
||||
|
||||
float sumf = 0;
|
||||
|
||||
for (int i = 0; i < nb; ++i) {
|
||||
const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
|
||||
const uint8_t * GGML_RESTRICT ql = x[i].ql;
|
||||
const uint8_t * GGML_RESTRICT qh = x[i].qh;
|
||||
const int8_t * GGML_RESTRICT q8 = y[i].qs;
|
||||
|
||||
const v128_t vsc = wasm_v128_load(x[i].scales);
|
||||
const v128_t vsc_l = wasm_i16x8_extend_low_i8x16(vsc);
|
||||
const v128_t vsc_h = wasm_i16x8_extend_high_i8x16(vsc);
|
||||
const v128_t vmins = wasm_i32x4_add(
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums), vsc_l),
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums + 8), vsc_h));
|
||||
|
||||
v128_t acc = zero;
|
||||
v128_t acc2 = zero;
|
||||
for (int j = 0; j < QK_K/128; ++j) {
|
||||
const v128_t ql0 = wasm_v128_load(ql);
|
||||
const v128_t ql1 = wasm_v128_load(ql + 16);
|
||||
const v128_t ql2 = wasm_v128_load(ql + 32);
|
||||
const v128_t ql3 = wasm_v128_load(ql + 48);
|
||||
const v128_t qh0 = wasm_v128_load(qh);
|
||||
const v128_t qh1 = wasm_v128_load(qh + 16);
|
||||
ql += 64;
|
||||
qh += 32;
|
||||
|
||||
const v128_t q6_0 = wasm_v128_or(wasm_v128_and(ql0, m4), wasm_i16x8_shl(wasm_v128_and(qh0, m2), 4));
|
||||
const v128_t q6_1 = wasm_v128_or(wasm_v128_and(ql1, m4), wasm_i16x8_shl(wasm_v128_and(qh1, m2), 4));
|
||||
const v128_t q6_2 = wasm_v128_or(wasm_v128_and(ql2, m4), wasm_i16x8_shl(wasm_v128_and(qh0, wasm_i8x16_splat(0x0C)), 2));
|
||||
const v128_t q6_3 = wasm_v128_or(wasm_v128_and(ql3, m4), wasm_i16x8_shl(wasm_v128_and(qh1, wasm_i8x16_splat(0x0C)), 2));
|
||||
const v128_t q6_4 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql0, 4), m4), wasm_v128_and(qh0, wasm_i8x16_splat(0x30)));
|
||||
const v128_t q6_5 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql1, 4), m4), wasm_v128_and(qh1, wasm_i8x16_splat(0x30)));
|
||||
const v128_t q6_6 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql2, 4), m4), wasm_u16x8_shr(wasm_v128_and(qh0, wasm_i8x16_splat((int8_t)0xC0)), 2));
|
||||
const v128_t q6_7 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql3, 4), m4), wasm_u16x8_shr(wasm_v128_and(qh1, wasm_i8x16_splat((int8_t)0xC0)), 2));
|
||||
|
||||
const v128_t p0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 0), q6_0, zero);
|
||||
const v128_t p1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 16), q6_1, zero);
|
||||
const v128_t p2 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 32), q6_2, zero);
|
||||
const v128_t p3 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 48), q6_3, zero);
|
||||
const v128_t p4 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 64), q6_4, zero);
|
||||
const v128_t p5 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 80), q6_5, zero);
|
||||
const v128_t p6 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 96), q6_6, zero);
|
||||
const v128_t p7 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 112), q6_7, zero);
|
||||
q8 += 128;
|
||||
|
||||
const v128_t s32a = wasm_i32x4_extend_low_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
const v128_t s32b = wasm_i32x4_extend_high_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p0, wasm_i32x4_shuffle(s32a, s32a, 0, 0, 0, 0)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p1, wasm_i32x4_shuffle(s32a, s32a, 1, 1, 1, 1)));
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p2, wasm_i32x4_shuffle(s32a, s32a, 2, 2, 2, 2)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p3, wasm_i32x4_shuffle(s32a, s32a, 3, 3, 3, 3)));
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p4, wasm_i32x4_shuffle(s32b, s32b, 0, 0, 0, 0)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p5, wasm_i32x4_shuffle(s32b, s32b, 1, 1, 1, 1)));
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p6, wasm_i32x4_shuffle(s32b, s32b, 2, 2, 2, 2)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p7, wasm_i32x4_shuffle(s32b, s32b, 3, 3, 3, 3)));
|
||||
}
|
||||
|
||||
acc = wasm_i32x4_sub(wasm_i32x4_add(acc, acc2), wasm_i32x4_shl(vmins, 5));
|
||||
const int32_t isum = wasm_i32x4_extract_lane(acc, 0) + wasm_i32x4_extract_lane(acc, 1) +
|
||||
wasm_i32x4_extract_lane(acc, 2) + wasm_i32x4_extract_lane(acc, 3);
|
||||
sumf += d * isum;
|
||||
}
|
||||
|
||||
*s = sumf;
|
||||
|
||||
#elif defined __wasm_simd128__
|
||||
int8_t aux8[QK_K] __attribute__((aligned(16)));
|
||||
int32_t aux32[8] __attribute__((aligned(16))) = {0};
|
||||
float sums[8] __attribute__((aligned(16))) = {0};
|
||||
|
||||
@@ -861,8 +861,8 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
__m128 acc_3 = _mm_setzero_ps();
|
||||
|
||||
for (; ib + 1 < nb; ib += 2) {
|
||||
_mm_prefetch(&x[ib] + sizeof(block_q4_0), _MM_HINT_T0);
|
||||
_mm_prefetch(&y[ib] + sizeof(block_q8_0), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&x[ib] + sizeof(block_q4_0)), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&y[ib] + sizeof(block_q8_0)), _MM_HINT_T0);
|
||||
|
||||
// Compute combined scale for the block 0 and 1
|
||||
const __m128 d_0_1 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d) );
|
||||
@@ -879,8 +879,8 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
bx_1 = _mm_sub_epi8(bx_1, off);
|
||||
const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);
|
||||
|
||||
_mm_prefetch(&x[ib] + 2 * sizeof(block_q4_0), _MM_HINT_T0);
|
||||
_mm_prefetch(&y[ib] + 2 * sizeof(block_q8_0), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&x[ib] + 2 * sizeof(block_q4_0)), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&y[ib] + 2 * sizeof(block_q8_0)), _MM_HINT_T0);
|
||||
|
||||
// Compute combined scale for the block 2 and 3
|
||||
const __m128 d_2_3 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) );
|
||||
|
||||
@@ -2034,6 +2034,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
|
||||
{
|
||||
ggml_compute_forward_fill(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_SLEEP:
|
||||
{
|
||||
ggml_compute_forward_sleep(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_FLASH_ATTN_EXT:
|
||||
{
|
||||
ggml_compute_forward_flash_attn_ext(params, tensor);
|
||||
@@ -2306,6 +2310,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
|
||||
case GGML_OP_REPEAT:
|
||||
case GGML_OP_REPEAT_BACK:
|
||||
case GGML_OP_LEAKY_RELU:
|
||||
case GGML_OP_SLEEP:
|
||||
{
|
||||
n_tasks = 1;
|
||||
} break;
|
||||
@@ -2378,10 +2383,8 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
|
||||
} break;
|
||||
case GGML_OP_GET_ROWS:
|
||||
{
|
||||
// FIXME: get_rows can use additional threads, but the cost of launching additional threads
|
||||
// decreases performance with GPU offloading
|
||||
//n_tasks = n_threads;
|
||||
n_tasks = 1;
|
||||
// waking the other threads costs more than a small lookup, e.g. when this is the only CPU node of an offloaded model
|
||||
n_tasks = ggml_nelements(node) >= 262144 ? n_threads : 1;
|
||||
} break;
|
||||
case GGML_OP_SCALE:
|
||||
case GGML_OP_SET:
|
||||
|
||||
+100
-49
@@ -2238,9 +2238,23 @@ void ggml_compute_forward_fill(const ggml_compute_params * params, ggml_tensor *
|
||||
}
|
||||
}
|
||||
|
||||
// ggml_compute_forward_sleep
|
||||
|
||||
void ggml_compute_forward_sleep(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
if (params->ith != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// same clock as ggml_time_us() callers; on macOS steady_clock is CLOCK_MONOTONIC_RAW, which can run faster
|
||||
const int64_t t_end = ggml_time_us() + ggml_get_op_params_i32(dst, 0);
|
||||
|
||||
while (ggml_time_us() < t_end) {}
|
||||
}
|
||||
|
||||
// ggml_compute_tri
|
||||
|
||||
static void ggml_compute_forward_tri_f32(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_tri_t(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
|
||||
const ggml_tri_type ttype = (ggml_tri_type) ggml_get_op_params_i32(dst, 0);
|
||||
@@ -2266,11 +2280,11 @@ static void ggml_compute_forward_tri_f32(const ggml_compute_params * params, ggm
|
||||
const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
|
||||
const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);
|
||||
|
||||
const float * src_ptr = (const float *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
|
||||
float * dst_ptr = ( float *) (( char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);
|
||||
const T * src_ptr = (const T *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
|
||||
T * dst_ptr = ( T *) (( char *) dst->data + i03*nb3 + i02*nb2 + i01*nb1);
|
||||
|
||||
for (int i0 = 0; i0 < ne0; ++i0) {
|
||||
dst_ptr[i0] = bipred(i0, i01) ? src_ptr[i0] : 0.0f;
|
||||
dst_ptr[i0] = bipred(i0, i01) ? src_ptr[i0] : T(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2281,7 +2295,12 @@ void ggml_compute_forward_tri(const ggml_compute_params * params, ggml_tensor *
|
||||
switch (src0->type) {
|
||||
case GGML_TYPE_F32:
|
||||
{
|
||||
ggml_compute_forward_tri_f32(params, dst);
|
||||
ggml_compute_forward_tri_t<uint32_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_tri_t<uint16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -2708,7 +2727,8 @@ static void ggml_compute_forward_leaky_relu_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_leaky_relu_f16(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_leaky_relu_half(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -2728,13 +2748,16 @@ static void ggml_compute_forward_leaky_relu_f16(
|
||||
float negative_slope;
|
||||
memcpy(&negative_slope, dst->op_params, sizeof(float));
|
||||
|
||||
assert(dst->nb[0] == sizeof(ggml_fp16_t));
|
||||
assert(src0->nb[0] == sizeof(ggml_fp16_t));
|
||||
assert(dst->nb[0] == sizeof(T));
|
||||
assert(src0->nb[0] == sizeof(T));
|
||||
|
||||
for (int i = 0; i < n; i++) {
|
||||
ggml_vec_leaky_relu_f16(nc,
|
||||
(ggml_fp16_t *) ((char *) dst->data + i*( dst->nb[1])),
|
||||
(ggml_fp16_t *) ((char *) src0->data + i*(src0->nb[1])), negative_slope);
|
||||
T * y = (T *) ((char *) dst->data + i*( dst->nb[1]));
|
||||
const T * x = (const T *) ((const char *) src0->data + i*(src0->nb[1]));
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float v = type_conversion_table<T>::to_f32(x[k]);
|
||||
y[k] = type_conversion_table<T>::from_f32(((v > 0.f) ? v : 0.f) + negative_slope * ((v < 0.0f) ? v : 0.f));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2751,7 +2774,11 @@ void ggml_compute_forward_leaky_relu(
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
ggml_compute_forward_leaky_relu_f16(params, dst);
|
||||
ggml_compute_forward_leaky_relu_half<ggml_fp16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_leaky_relu_half<ggml_bf16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -3452,7 +3479,8 @@ static void ggml_compute_forward_swiglu_oai_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_swiglu_oai_f16(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_swiglu_oai_half(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -3492,9 +3520,9 @@ static void ggml_compute_forward_swiglu_oai_f16(
|
||||
const int ir1 = MIN(ir0 + dr, nr);
|
||||
|
||||
for (int i1 = ir0; i1 < ir1; i1++) {
|
||||
ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1*src0_o);
|
||||
ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1*src1_o);
|
||||
ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1*(dst->nb[1]));
|
||||
T * src0_p = (T *) (src0_d + i1*src0_o);
|
||||
T * src1_p = (T *) (src1_d + i1*src1_o);
|
||||
T * dst_p = (T *) ((char *) dst->data + i1*(dst->nb[1]));
|
||||
|
||||
if (!src1) {
|
||||
src0_p += swapped ? nc : 0;
|
||||
@@ -3502,15 +3530,15 @@ static void ggml_compute_forward_swiglu_oai_f16(
|
||||
}
|
||||
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float x = std::min(GGML_CPU_FP16_TO_FP32(src0_p[k]), limit);
|
||||
const float y = std::clamp(GGML_CPU_FP16_TO_FP32(src1_p[k]), -limit, limit);
|
||||
const float x = std::min(type_conversion_table<T>::to_f32(src0_p[k]), limit);
|
||||
const float y = std::clamp(type_conversion_table<T>::to_f32(src1_p[k]), -limit, limit);
|
||||
const float out_glu = x / (1.f + expf(alpha * (-x)));
|
||||
dst_p[k] = GGML_CPU_FP32_TO_FP16(out_glu * (y + 1.f));
|
||||
dst_p[k] = type_conversion_table<T>::from_f32(out_glu * (y + 1.f));
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float x = GGML_CPU_FP16_TO_FP32(dst_p[k]);
|
||||
const float x = type_conversion_table<T>::to_f32(dst_p[k]);
|
||||
GGML_UNUSED(x);
|
||||
assert(!isnan(x));
|
||||
assert(!isinf(x));
|
||||
@@ -3532,7 +3560,11 @@ static void ggml_compute_forward_swiglu_oai(
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
ggml_compute_forward_swiglu_oai_f16(params, dst);
|
||||
ggml_compute_forward_swiglu_oai_half<ggml_fp16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_swiglu_oai_half<ggml_bf16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -3602,7 +3634,8 @@ static void ggml_compute_forward_swiglu_clamp_f32(const ggml_compute_params * pa
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_swiglu_clamp_half(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const ggml_tensor * src1 = dst->src[1];
|
||||
char * src0_d = (char *) src0->data;
|
||||
@@ -3635,9 +3668,9 @@ static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * pa
|
||||
const int ir1 = MIN(ir0 + dr, nr);
|
||||
|
||||
for (int i1 = ir0; i1 < ir1; i1++) {
|
||||
ggml_fp16_t * src0_p = (ggml_fp16_t *) (src0_d + i1 * src0_o);
|
||||
ggml_fp16_t * src1_p = (ggml_fp16_t *) (src1_d + i1 * src1_o);
|
||||
ggml_fp16_t * dst_p = (ggml_fp16_t *) ((char *) dst->data + i1 * (dst->nb[1]));
|
||||
T * src0_p = (T *) (src0_d + i1 * src0_o);
|
||||
T * src1_p = (T *) (src1_d + i1 * src1_o);
|
||||
T * dst_p = (T *) ((char *) dst->data + i1 * (dst->nb[1]));
|
||||
|
||||
if (!src1) {
|
||||
src0_p += swapped ? nc : 0;
|
||||
@@ -3645,14 +3678,14 @@ static void ggml_compute_forward_swiglu_clamp_f16(const ggml_compute_params * pa
|
||||
}
|
||||
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float gate = std::min(GGML_FP16_TO_FP32(src0_p[k]), limit);
|
||||
const float up = std::clamp(GGML_FP16_TO_FP32(src1_p[k]), -limit, limit);
|
||||
dst_p[k] = GGML_FP32_TO_FP16(gate / (1.f + expf(-gate)) * up);
|
||||
const float gate = std::min(type_conversion_table<T>::to_f32(src0_p[k]), limit);
|
||||
const float up = std::clamp(type_conversion_table<T>::to_f32(src1_p[k]), -limit, limit);
|
||||
dst_p[k] = type_conversion_table<T>::from_f32(gate / (1.f + expf(-gate)) * up);
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
for (int k = 0; k < nc; k++) {
|
||||
const float x = GGML_FP16_TO_FP32(dst_p[k]);
|
||||
const float x = type_conversion_table<T>::to_f32(dst_p[k]);
|
||||
GGML_UNUSED(x);
|
||||
assert(!isnan(x));
|
||||
assert(!isinf(x));
|
||||
@@ -3667,7 +3700,10 @@ static void ggml_compute_forward_swiglu_clamp(const ggml_compute_params * params
|
||||
ggml_compute_forward_swiglu_clamp_f32(params, dst);
|
||||
break;
|
||||
case GGML_TYPE_F16:
|
||||
ggml_compute_forward_swiglu_clamp_f16(params, dst);
|
||||
ggml_compute_forward_swiglu_clamp_half<ggml_fp16_t>(params, dst);
|
||||
break;
|
||||
case GGML_TYPE_BF16:
|
||||
ggml_compute_forward_swiglu_clamp_half<ggml_bf16_t>(params, dst);
|
||||
break;
|
||||
default:
|
||||
GGML_ABORT("fatal error");
|
||||
@@ -4953,7 +4989,8 @@ static void ggml_compute_forward_scale_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_scale_bf16(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_scale_half(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -4989,10 +5026,10 @@ static void ggml_compute_forward_scale_bf16(
|
||||
const int64_t i2 = (i / ne01) % ne02;
|
||||
const int64_t i3 = i / (ne01*ne02);
|
||||
|
||||
const ggml_bf16_t * x = (const ggml_bf16_t *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
|
||||
ggml_bf16_t * y = (ggml_bf16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
const T * x = (const T *) ((const char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
|
||||
T * y = (T *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
for (int k = 0; k < nc; k++) {
|
||||
y[k] = GGML_FP32_TO_BF16(s*GGML_BF16_TO_FP32(x[k]) + b);
|
||||
y[k] = type_conversion_table<T>::from_f32(s*type_conversion_table<T>::to_f32(x[k]) + b);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5008,9 +5045,13 @@ void ggml_compute_forward_scale(
|
||||
{
|
||||
ggml_compute_forward_scale_f32(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
ggml_compute_forward_scale_half<ggml_fp16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_scale_bf16(params, dst);
|
||||
ggml_compute_forward_scale_half<ggml_bf16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -5694,7 +5735,8 @@ void ggml_compute_forward_get_rows_back(
|
||||
|
||||
// ggml_compute_forward_diag
|
||||
|
||||
static void ggml_compute_forward_diag_f32(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_diag_t(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -5714,14 +5756,14 @@ static void ggml_compute_forward_diag_f32(
|
||||
GGML_ASSERT(ne02 == ne2);
|
||||
GGML_ASSERT(ne03 == ne3);
|
||||
|
||||
GGML_ASSERT(nb00 == sizeof(float));
|
||||
GGML_ASSERT(nb0 == sizeof(float));
|
||||
GGML_ASSERT(nb00 == sizeof(T));
|
||||
GGML_ASSERT(nb0 == sizeof(T));
|
||||
|
||||
for (int i3 = 0; i3 < ne3; i3++) {
|
||||
for (int i2 = 0; i2 < ne2; i2++) {
|
||||
for (int i1 = 0; i1 < ne1; i1++) {
|
||||
float * d = (float *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
|
||||
float * s = (float *)((char *) src0->data + i3*nb03 + i2*nb02);
|
||||
T * d = (T *)((char *) dst->data + i3*nb3 + i2*nb2 + i1*nb1);
|
||||
T * s = (T *)((char *) src0->data + i3*nb03 + i2*nb02);
|
||||
for (int i0 = 0; i0 < i1; i0++) {
|
||||
d[i0] = 0;
|
||||
}
|
||||
@@ -5743,7 +5785,12 @@ void ggml_compute_forward_diag(
|
||||
switch (src0->type) {
|
||||
case GGML_TYPE_F32:
|
||||
{
|
||||
ggml_compute_forward_diag_f32(params, dst);
|
||||
ggml_compute_forward_diag_t<uint32_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_diag_t<uint16_t>(params, dst);
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
@@ -6026,7 +6073,8 @@ static void ggml_compute_forward_clamp_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_compute_forward_clamp_f16(
|
||||
template <typename T>
|
||||
static void ggml_compute_forward_clamp_half(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
@@ -6045,20 +6093,20 @@ static void ggml_compute_forward_clamp_f16(
|
||||
|
||||
GGML_TENSOR_UNARY_OP_LOCALS
|
||||
|
||||
GGML_ASSERT( nb0 == sizeof(ggml_fp16_t));
|
||||
GGML_ASSERT(nb00 == sizeof(ggml_fp16_t));
|
||||
GGML_ASSERT( nb0 == sizeof(T));
|
||||
GGML_ASSERT(nb00 == sizeof(T));
|
||||
|
||||
for (int j = ith; j < n; j += nth) {
|
||||
const int64_t i1 = j % ne01;
|
||||
const int64_t i2 = (j / ne01) % ne02;
|
||||
const int64_t i3 = j / (ne01*ne02);
|
||||
|
||||
ggml_fp16_t * dst_ptr = (ggml_fp16_t *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
ggml_fp16_t * src0_ptr = (ggml_fp16_t *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
|
||||
T * dst_ptr = (T *) ((char *) dst->data + i1*nb1 + i2*nb2 + i3*nb3);
|
||||
T * src0_ptr = (T *) ((char *) src0->data + i1*nb01 + i2*nb02 + i3*nb03);
|
||||
|
||||
for (int i = 0; i < nc; i++) {
|
||||
float v = GGML_CPU_FP16_TO_FP32(src0_ptr[i]);
|
||||
dst_ptr[i] = GGML_CPU_FP32_TO_FP16(MAX(MIN(v, max), min));
|
||||
float v = type_conversion_table<T>::to_f32(src0_ptr[i]);
|
||||
dst_ptr[i] = type_conversion_table<T>::from_f32(MAX(MIN(v, max), min));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -6076,9 +6124,12 @@ void ggml_compute_forward_clamp(
|
||||
} break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
ggml_compute_forward_clamp_f16(params, dst);
|
||||
ggml_compute_forward_clamp_half<ggml_fp16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_clamp_half<ggml_bf16_t>(params, dst);
|
||||
} break;
|
||||
case GGML_TYPE_Q1_0:
|
||||
case GGML_TYPE_Q2_0:
|
||||
case GGML_TYPE_Q4_0:
|
||||
|
||||
@@ -84,6 +84,7 @@ void ggml_compute_forward_top_k(const struct ggml_compute_params * params, struc
|
||||
void ggml_compute_forward_leaky_relu(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_fill(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_sleep(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_flash_attn_ext(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_flash_attn_back(
|
||||
const struct ggml_compute_params * params,
|
||||
|
||||
@@ -962,12 +962,6 @@ inline static void ggml_vec_relu_f16 (const int n, ggml_fp16_t * y, const ggml_f
|
||||
}
|
||||
}
|
||||
inline static void ggml_vec_leaky_relu_f32 (const int n, float * y, const float * x, const float ns) { for (int i = 0; i < n; ++i) y[i] = ((x[i] > 0.f) ? x[i] : 0.f) + ns * ((x[i] < 0.0f) ? x[i] : 0.f); }
|
||||
inline static void ggml_vec_leaky_relu_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x, const float ns) {
|
||||
for (int i = 0; i < n; ++i) {
|
||||
float v = GGML_CPU_FP16_TO_FP32(x[i]);
|
||||
y[i] = GGML_CPU_FP32_TO_FP16(((v > 0.f) ? v : 0.f) + ns * ((v < 0.0f) ? v : 0.f));
|
||||
}
|
||||
}
|
||||
inline static void ggml_vec_sigmoid_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = 1.f / (1.f + expf(-x[i])); }
|
||||
inline static void ggml_vec_sigmoid_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {
|
||||
for (int i = 0; i < n; ++i) {
|
||||
|
||||
@@ -133,6 +133,8 @@ if (CUDAToolkit_FOUND)
|
||||
${GGML_SOURCES_CUDA}
|
||||
)
|
||||
|
||||
target_link_libraries(ggml-cuda PRIVATE ${CMAKE_DL_LIBS})
|
||||
|
||||
if (GGML_CUDA_GRAPHS)
|
||||
add_compile_definitions(GGML_CUDA_USE_GRAPHS)
|
||||
endif()
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
#include "batch-get.cuh"
|
||||
#include "batch-copy.cuh"
|
||||
#include "common.cuh"
|
||||
#include "ggml-cpp.h"
|
||||
|
||||
#include <climits>
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
struct ggml_cuda_batch_get_descriptor {
|
||||
struct ggml_cuda_batch_copy_descriptor {
|
||||
const unsigned char * src;
|
||||
unsigned char * dst;
|
||||
size_t size;
|
||||
};
|
||||
|
||||
static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_descriptor * copies, size_t count) {
|
||||
static __global__ void ggml_cuda_batch_copy_kernel(const ggml_cuda_batch_copy_descriptor * copies, size_t count) {
|
||||
// Blocks along y cooperate on each descriptor.
|
||||
const size_t index = blockIdx.x;
|
||||
if (index >= count) {
|
||||
@@ -50,11 +50,11 @@ static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_desc
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_cuda_batch_get_mapping {
|
||||
struct ggml_cuda_batch_copy_mapping {
|
||||
void * host = nullptr;
|
||||
unsigned char * device = nullptr;
|
||||
|
||||
ggml_cuda_batch_get_mapping(void * host, size_t size) {
|
||||
ggml_cuda_batch_copy_mapping(void * host, size_t size) {
|
||||
const cudaError_t err = cudaHostRegister(host, size, cudaHostRegisterMapped | cudaHostRegisterPortable);
|
||||
if (err != cudaSuccess) {
|
||||
(void) cudaGetLastError();
|
||||
@@ -70,10 +70,10 @@ struct ggml_cuda_batch_get_mapping {
|
||||
}
|
||||
}
|
||||
|
||||
ggml_cuda_batch_get_mapping(const ggml_cuda_batch_get_mapping &) = delete;
|
||||
ggml_cuda_batch_get_mapping & operator=(const ggml_cuda_batch_get_mapping &) = delete;
|
||||
ggml_cuda_batch_copy_mapping(const ggml_cuda_batch_copy_mapping &) = delete;
|
||||
ggml_cuda_batch_copy_mapping & operator=(const ggml_cuda_batch_copy_mapping &) = delete;
|
||||
|
||||
~ggml_cuda_batch_get_mapping() {
|
||||
~ggml_cuda_batch_copy_mapping() {
|
||||
if (host) {
|
||||
// Release only the registration acquired by this object.
|
||||
CUDA_CHECK(cudaHostUnregister(host));
|
||||
@@ -82,11 +82,12 @@ struct ggml_cuda_batch_get_mapping {
|
||||
};
|
||||
#endif
|
||||
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
static bool ggml_cuda_copy_tensor_n(int device, cudaMemcpyKind direction,
|
||||
const ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
GGML_UNUSED(device);
|
||||
GGML_UNUSED(direction);
|
||||
GGML_UNUSED(tensors);
|
||||
GGML_UNUSED(data);
|
||||
GGML_UNUSED(offsets);
|
||||
@@ -94,14 +95,17 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
GGML_UNUSED(n_tensors);
|
||||
return false;
|
||||
#else
|
||||
const bool to_host = direction == cudaMemcpyDeviceToHost;
|
||||
// Provisional dispatch values based on experiments.
|
||||
constexpr size_t min_batch_reads = 128;
|
||||
constexpr size_t max_gather_read_bytes = 128 * 1024;
|
||||
// Collect non-empty CPU destination ranges.
|
||||
constexpr size_t max_kernel_read_bytes = 128 * 1024;
|
||||
// Collect destination ranges; repeated sources are allowed.
|
||||
std::vector<std::pair<uintptr_t, uintptr_t>> ranges;
|
||||
ranges.reserve(n_tensors);
|
||||
size_t payload_bytes = 0;
|
||||
// Number of non-empty reads whose size is at most max_gather_read_bytes.
|
||||
uintptr_t begin = UINTPTR_MAX;
|
||||
uintptr_t end = 0;
|
||||
// Number of non-empty reads whose size is at most max_kernel_read_bytes.
|
||||
size_t n_reads_small = 0;
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
if (sizes[i] == 0) {
|
||||
@@ -111,15 +115,19 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
GGML_ASSERT(address <= UINTPTR_MAX - sizes[i]);
|
||||
GGML_ASSERT(payload_bytes <= SIZE_MAX - sizes[i]);
|
||||
payload_bytes += sizes[i];
|
||||
n_reads_small += sizes[i] <= max_gather_read_bytes;
|
||||
ranges.emplace_back(address, address + sizes[i]);
|
||||
n_reads_small += sizes[i] <= max_kernel_read_bytes;
|
||||
begin = std::min(begin, address);
|
||||
end = std::max(end, address + sizes[i]);
|
||||
const uintptr_t destination = to_host ? address : (uintptr_t) tensors[i]->data + offsets[i];
|
||||
GGML_ASSERT(destination <= UINTPTR_MAX - sizes[i]);
|
||||
ranges.emplace_back(destination, destination + sizes[i]);
|
||||
}
|
||||
if (ranges.empty()) {
|
||||
return true;
|
||||
}
|
||||
// Total number of non-empty reads.
|
||||
const size_t n_reads = ranges.size();
|
||||
if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_get_descriptor) || n_reads > INT_MAX) {
|
||||
if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_copy_descriptor) || n_reads > INT_MAX) {
|
||||
return false;
|
||||
}
|
||||
if (n_reads < 2 || (n_reads < min_batch_reads && n_reads_small == n_reads)) {
|
||||
@@ -127,72 +135,85 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
}
|
||||
std::sort(ranges.begin(), ranges.end());
|
||||
for (size_t i = 1; i < ranges.size(); ++i) {
|
||||
// Preserve the ordered copy path when CPU destinations overlap.
|
||||
// Preserve the ordered copy path when destinations overlap.
|
||||
if (ranges[i].first < ranges[i - 1].second) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Find the CPU span to register, including gaps between destinations.
|
||||
const uintptr_t begin = ranges.front().first;
|
||||
const uintptr_t end = ranges.back().second;
|
||||
if (end - begin - payload_bytes > payload_bytes) {
|
||||
// Limit gaps in the CPU span without underflow when sources overlap.
|
||||
if (end - begin > payload_bytes && end - begin - payload_bytes > payload_bytes) {
|
||||
return false;
|
||||
}
|
||||
// Register and map the existing CPU destination so the kernel can write to it directly.
|
||||
ggml_cuda_batch_get_mapping mapping((void *) begin, end - begin);
|
||||
// Register and map the CPU span for direct kernel access.
|
||||
ggml_cuda_batch_copy_mapping mapping((void *) begin, end - begin);
|
||||
if (!mapping.host) {
|
||||
return false;
|
||||
}
|
||||
// Reads selected for gather; zero means all reads use DMA.
|
||||
size_t n_reads_gather = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0;
|
||||
// Reads selected for the kernel; zero means all reads use DMA.
|
||||
size_t n_reads_kernel = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0;
|
||||
// Keep descriptor storage alive until the final stream synchronization.
|
||||
constexpr size_t max_batch_descriptors = 128 * 1024;
|
||||
const size_t bytes = std::min(n_reads_gather, max_batch_descriptors) * sizeof(ggml_cuda_batch_get_descriptor);
|
||||
ggml_backend_buffer_ptr descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr);
|
||||
const size_t bytes = std::min(n_reads_kernel, max_batch_descriptors) * sizeof(ggml_cuda_batch_copy_descriptor);
|
||||
ggml_backend_buffer_ptr descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr);
|
||||
if (!descriptors) {
|
||||
n_reads_gather = 0;
|
||||
n_reads_kernel = 0;
|
||||
}
|
||||
const auto host_type = ggml_backend_cuda_host_buffer_type();
|
||||
ggml_backend_buffer_ptr host_descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr);
|
||||
ggml_backend_buffer_ptr host_descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr);
|
||||
if (!host_descriptors || ggml_backend_buffer_get_type(host_descriptors.get()) != host_type) {
|
||||
n_reads_gather = 0;
|
||||
n_reads_kernel = 0;
|
||||
}
|
||||
if (n_reads_gather) {
|
||||
auto * copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get()));
|
||||
auto * device_copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(descriptors.get()));
|
||||
if (n_reads_kernel) {
|
||||
auto * copies = static_cast<ggml_cuda_batch_copy_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get()));
|
||||
auto * device_copies = static_cast<ggml_cuda_batch_copy_descriptor *>(ggml_backend_buffer_get_base(descriptors.get()));
|
||||
const int threads = 256;
|
||||
const size_t nsm = ggml_cuda_info().devices[device].nsm;
|
||||
size_t i = 0;
|
||||
for (size_t first = 0; first < n_reads_gather; first += max_batch_descriptors) {
|
||||
const size_t n_batch = std::min(n_reads_gather - first, max_batch_descriptors);
|
||||
for (size_t first = 0; first < n_reads_kernel; first += max_batch_descriptors) {
|
||||
const size_t n_batch = std::min(n_reads_kernel - first, max_batch_descriptors);
|
||||
size_t count = 0;
|
||||
for (; i < n_tensors && count < n_batch; ++i) {
|
||||
if (sizes[i] == 0 || sizes[i] > max_gather_read_bytes) {
|
||||
if (sizes[i] == 0 || sizes[i] > max_kernel_read_bytes) {
|
||||
continue;
|
||||
}
|
||||
copies[count++] = {(const unsigned char *) tensors[i]->data + offsets[i],
|
||||
mapping.device + ((uintptr_t) data[i] - begin), sizes[i]};
|
||||
auto * host = mapping.device + ((uintptr_t) data[i] - begin);
|
||||
auto * tensor = (unsigned char *) tensors[i]->data + offsets[i];
|
||||
copies[count++] = {to_host ? tensor : host, to_host ? host : tensor, sizes[i]};
|
||||
}
|
||||
const size_t blocks_per_copy = std::min<size_t>(UINT16_MAX, std::max<size_t>(1, (nsm + n_batch - 1) / n_batch));
|
||||
const dim3 blocks((unsigned int) n_batch, (unsigned int) blocks_per_copy);
|
||||
// Stream ordering keeps each kernel ahead of the next descriptor overwrite.
|
||||
CUDA_CHECK(cudaMemcpyAsync(device_copies, copies, n_batch * sizeof(copies[0]), cudaMemcpyHostToDevice, cudaStreamPerThread));
|
||||
ggml_cuda_batch_get_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch);
|
||||
ggml_cuda_batch_copy_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch);
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
if (first + n_batch < n_reads_gather) {
|
||||
if (first + n_batch < n_reads_kernel) {
|
||||
// Wait before refilling the pinned host descriptor buffer.
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Keep the destination pinned for DMA too; overlapping destinations already declined above.
|
||||
// Keep the CPU span pinned for DMA too; overlapping destinations already declined above.
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
if (sizes[i] && (!n_reads_gather || sizes[i] > max_gather_read_bytes)) {
|
||||
CUDA_CHECK(cudaMemcpyAsync(data[i], (const char *) tensors[i]->data + offsets[i], sizes[i], cudaMemcpyDeviceToHost, cudaStreamPerThread));
|
||||
if (sizes[i] && (!n_reads_kernel || sizes[i] > max_kernel_read_bytes)) {
|
||||
void * host = const_cast<void *>(data[i]);
|
||||
void * tensor = (char *) tensors[i]->data + offsets[i];
|
||||
CUDA_CHECK(cudaMemcpyAsync(to_host ? host : tensor, to_host ? tensor : host, sizes[i], direction, cudaStreamPerThread));
|
||||
}
|
||||
}
|
||||
// Wait for GPU writes before releasing the CPU mapping and descriptor buffers.
|
||||
// Wait for transfers before releasing the CPU mapping and descriptor buffers.
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
return ggml_cuda_copy_tensor_n(device, cudaMemcpyDeviceToHost, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
|
||||
bool ggml_cuda_set_tensor_n(int device,
|
||||
ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
return ggml_cuda_copy_tensor_n(device, cudaMemcpyHostToDevice, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
@@ -5,3 +5,7 @@
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors);
|
||||
|
||||
bool ggml_cuda_set_tensor_n(int device,
|
||||
ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors);
|
||||
@@ -59,7 +59,7 @@
|
||||
// While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, see
|
||||
// https://docs.nvidia.com/cutlass/media/docs/cpp/blackwell_functionality.html#blackwell-sm120-gemms
|
||||
#define GGML_CUDA_CC_BLACKWELL 1200
|
||||
#define GGML_CUDA_CC_DGX_SPARK 1210
|
||||
#define GGML_CUDA_CC_GB10 1210
|
||||
#define GGML_CUDA_CC_RUBIN 1300
|
||||
#define GGML_CUDA_CC_OFFSET_AMD 0x1000000
|
||||
#define GGML_CUDA_CC_OFFSET_MTHREADS 0x0100000
|
||||
@@ -1293,6 +1293,7 @@ struct ggml_cuda_device_info {
|
||||
size_t total_vram;
|
||||
int warp_size; // Number of threads in a dispatch
|
||||
bool supports_cooperative_launch; // whether cooperative launch is supported
|
||||
bool prefer_fp32_acc; // prefer BF16 inputs with FP32 accumulation for quantized cuBLAS
|
||||
int physical_device; // backing physical CUDA device for this (virtual) device
|
||||
int physical_share_count; // number of (virtual) devices sharing this device's physical GPU
|
||||
int virtual_index; // index of this (virtual) device among those sharing its physical GPU
|
||||
|
||||
@@ -128,7 +128,9 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
|
||||
const half * __restrict__ weight,
|
||||
float * __restrict__ output,
|
||||
const conv_params P,
|
||||
const int split_k) {
|
||||
const int split_k,
|
||||
const uint3 kw_fd,
|
||||
const uint3 kwkh_fd) {
|
||||
using namespace ggml_cuda_mma;
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
constexpr int nthreads = 4 * warp_size;
|
||||
@@ -202,7 +204,10 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
|
||||
}
|
||||
for (int k = load_row; k < BK; k += nthreads / (BN / 2)) {
|
||||
const int ki = k0 + k;
|
||||
const int ci = ki / (kw * kh), ky = ki / kw % kh, kx = ki % kw;
|
||||
const int ci = KW && KH ? ki / (KW * KH) : int(fastdiv(uint32_t(ki), kwkh_fd));
|
||||
const int r = ki - ci * kw * kh;
|
||||
const int ky = KW ? r / KW : int(fastdiv(uint32_t(r), kw_fd));
|
||||
const int kx = r - ky * kw;
|
||||
const int offset = ki < k_total ? (n * ic + ci) * ih * iw + ky * dy * iw + kx * dx : 0;
|
||||
half lo = __float2half(0.0f), hi = lo;
|
||||
if (ki < k_total && spatial < ow * oh) {
|
||||
@@ -314,12 +319,17 @@ static void conv2d_launch_implicit_gemm(const half * input,
|
||||
dim3 grid,
|
||||
dim3 block,
|
||||
cudaStream_t stream) {
|
||||
const uint3 kw_fd = init_fastdiv_values(uint64_t(params.KW));
|
||||
const uint3 kwkh_fd = init_fastdiv_values(uint64_t(params.KW * params.KH));
|
||||
if (params.KW == 3 && params.KH == 3) {
|
||||
conv2d_implicit_gemm_f16<3, 3, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k);
|
||||
conv2d_implicit_gemm_f16<3, 3, use_mma>
|
||||
<<<grid, block, 0, stream>>>(input, weight, output, params, split_k, kw_fd, kwkh_fd);
|
||||
} else if (params.KW == 1 && params.KH == 1) {
|
||||
conv2d_implicit_gemm_f16<1, 1, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k);
|
||||
conv2d_implicit_gemm_f16<1, 1, use_mma>
|
||||
<<<grid, block, 0, stream>>>(input, weight, output, params, split_k, kw_fd, kwkh_fd);
|
||||
} else {
|
||||
conv2d_implicit_gemm_f16<0, 0, use_mma><<<grid, block, 0, stream>>>(input, weight, output, params, split_k);
|
||||
conv2d_implicit_gemm_f16<0, 0, use_mma>
|
||||
<<<grid, block, 0, stream>>>(input, weight, output, params, split_k, kw_fd, kwkh_fd);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1206,7 +1206,7 @@ void launch_fattn(
|
||||
dim3 blocks_num;
|
||||
if (stream_k) {
|
||||
// Stream-K splits the work before the mask scan is applied, so skipped KV tiles make the blocks uneven.
|
||||
const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK && async_kv_preload && scan_mask;
|
||||
const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10 && async_kv_preload && scan_mask;
|
||||
|
||||
auto should_use_stream_k = [prefer_whole_tiles](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) {
|
||||
const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks;
|
||||
|
||||
@@ -397,7 +397,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
|
||||
if (amd_mfma_available(cc) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 16)) {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
|
||||
} else if (gqa_ratio == 20 && GGML_CUDA_CC_IS_NVIDIA(cc)) { // GLM 4.7 Flash
|
||||
if (cc >= GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (cc >= GGML_CUDA_CC_GB10) {
|
||||
if (Q->ne[1] <= 8) {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
|
||||
break;
|
||||
|
||||
@@ -468,7 +468,7 @@ static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a
|
||||
// Enable only architectures benchmarked for this schedule. MMA support alone does not imply a speedup.
|
||||
if (!ampere_mma_available(info.cc) ||
|
||||
(info.cc != GGML_CUDA_CC_AMPERE_GA10X && info.cc != GGML_CUDA_CC_ADA_LOVELACE &&
|
||||
info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_DGX_SPARK)) {
|
||||
info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_GB10)) {
|
||||
return gdn_path::ar;
|
||||
}
|
||||
// Value splitting pays off when whole heads leave at least half the SMs idle.
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
#include "ggml-cuda/arange.cuh"
|
||||
#include "ggml-cuda/argmax.cuh"
|
||||
#include "ggml-cuda/argsort.cuh"
|
||||
#include "ggml-cuda/batch-get.cuh"
|
||||
#include "ggml-cuda/batch-copy.cuh"
|
||||
#include "ggml-cuda/binbcast.cuh"
|
||||
#include "ggml-cuda/clamp.cuh"
|
||||
#include "ggml-cuda/col2im-1d.cuh"
|
||||
@@ -48,6 +48,7 @@
|
||||
#include "ggml-cuda/rope.cuh"
|
||||
#include "ggml-cuda/roll.cuh"
|
||||
#include "ggml-cuda/scale.cuh"
|
||||
#include "ggml-cuda/sleep.cuh"
|
||||
#include "ggml-cuda/snake.cuh"
|
||||
#include "ggml-cuda/softcap.cuh"
|
||||
#include "ggml-cuda/softmax.cuh"
|
||||
@@ -96,6 +97,11 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
#include "ggml-backend-dl.h"
|
||||
#include <nvml.h>
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");
|
||||
|
||||
#define GGML_LOG_WARN_ONCE(str) \
|
||||
@@ -309,6 +315,64 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
||||
}
|
||||
total_vram = 0;
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
#if defined(_WIN32)
|
||||
// NVML is installed by the driver; its NVSMI directory is not added to PATH on Windows.
|
||||
dl_handle_ptr library(LoadLibraryExW(L"nvml.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32));
|
||||
if (!library) {
|
||||
const DWORD size = GetEnvironmentVariableW(L"ProgramW6432", nullptr, 0);
|
||||
if (size > 0) {
|
||||
std::wstring path(size, L'\0');
|
||||
const DWORD length = GetEnvironmentVariableW(L"ProgramW6432", &path[0], size);
|
||||
if (length > 0 && length < size) {
|
||||
path.resize(length);
|
||||
path += L"\\NVIDIA Corporation\\NVSMI\\nvml.dll";
|
||||
if (fs::path(path).is_absolute()) {
|
||||
library.reset(LoadLibraryExW(path.c_str(), nullptr, LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
dl_handle_ptr library(dlopen("libnvidia-ml.so.1", RTLD_NOW | RTLD_LOCAL));
|
||||
#endif // defined(_WIN32)
|
||||
const auto get_symbol = [&](const char * name) {
|
||||
#if defined(_WIN32)
|
||||
return library ? GetProcAddress(library.get(), name) : nullptr;
|
||||
#else
|
||||
return library ? dlsym(library.get(), name) : nullptr;
|
||||
#endif // defined(_WIN32)
|
||||
};
|
||||
const auto init = reinterpret_cast<decltype(&nvmlInit_v2)>(get_symbol("nvmlInit_v2"));
|
||||
const auto shutdown = reinterpret_cast<decltype(&nvmlShutdown)>(get_symbol("nvmlShutdown"));
|
||||
const auto get_device = reinterpret_cast<decltype(&nvmlDeviceGetHandleByPciBusId_v2)>(get_symbol("nvmlDeviceGetHandleByPciBusId_v2"));
|
||||
const auto get_brand = reinterpret_cast<decltype(&nvmlDeviceGetBrand)>(get_symbol("nvmlDeviceGetBrand"));
|
||||
const bool nvml_initialized = init && shutdown && get_device && get_brand && init() == NVML_SUCCESS;
|
||||
|
||||
const auto should_use_fp32_acc = [&](const cudaDeviceProp & prop, const int cc) {
|
||||
if (cc == GGML_CUDA_CC_GB10) {
|
||||
return true;
|
||||
}
|
||||
if (!fast_bf16_hardware_available(cc) || !nvml_initialized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// CUDA and NVML device indices can differ, e.g. with CUDA_VISIBLE_DEVICES.
|
||||
char pci_bus_id[32];
|
||||
snprintf(pci_bus_id, sizeof(pci_bus_id), "%04x:%02x:%02x.0",
|
||||
(unsigned int) prop.pciDomainID, (unsigned int) prop.pciBusID, (unsigned int) prop.pciDeviceID);
|
||||
nvmlDevice_t device = nullptr;
|
||||
nvmlBrandType_t brand = NVML_BRAND_UNKNOWN;
|
||||
bool prefer_fp32_acc = false;
|
||||
if (get_device(pci_bus_id, &device) == NVML_SUCCESS && get_brand(device, &brand) == NVML_SUCCESS) {
|
||||
prefer_fp32_acc = brand == NVML_BRAND_QUADRO ||
|
||||
brand == NVML_BRAND_QUADRO_RTX ||
|
||||
brand == NVML_BRAND_NVIDIA_RTX;
|
||||
}
|
||||
return prefer_fp32_acc;
|
||||
};
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
std::vector<std::pair<int, std::string>> turing_devices_without_mma;
|
||||
for (int id = 0; id < info.device_count; ++id) {
|
||||
const int physical_id = info.devices[id].physical_device;
|
||||
@@ -379,6 +443,11 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
||||
#else
|
||||
info.devices[id].smpbo = prop.sharedMemPerBlockOptin;
|
||||
info.devices[id].cc = 100*prop.major + 10*prop.minor;
|
||||
if (info.devices[id].virtual_index == 0) {
|
||||
info.devices[id].prefer_fp32_acc = should_use_fp32_acc(prop, info.devices[id].cc);
|
||||
} else {
|
||||
info.devices[id].prefer_fp32_acc = info.devices[physical_id].prefer_fp32_acc;
|
||||
}
|
||||
GGML_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s, VRAM: %zu MiB\n",
|
||||
id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no",
|
||||
device_vram_mib);
|
||||
@@ -403,6 +472,13 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
||||
#endif // defined(GGML_USE_HIP)
|
||||
}
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
if (nvml_initialized) {
|
||||
(void) shutdown();
|
||||
}
|
||||
library.reset();
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
if (ggml_cuda_highest_compiled_arch(GGML_CUDA_CC_TURING) >= GGML_CUDA_CC_TURING && !turing_devices_without_mma.empty()) {
|
||||
GGML_LOG_INFO("The following devices will have suboptimal performance due to a lack of tensor cores:\n");
|
||||
for (size_t device_pos = 0; device_pos < turing_devices_without_mma.size(); device_pos++) {
|
||||
@@ -1041,6 +1117,11 @@ static void ggml_backend_cuda_buffer_set_tensor_n(ggml_backend_buffer_t buffer,
|
||||
|
||||
ggml_cuda_set_device(ctx->device);
|
||||
|
||||
if (!ggml_cuda_pinned_memory_disabled() &&
|
||||
ggml_cuda_set_tensor_n(ctx->device, tensors, data, offsets, sizes, n_tensors)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
@@ -1641,7 +1722,7 @@ static void * ggml_backend_cuda_comm_init_rank(ggml_backend_t backend, int rank,
|
||||
// NCCL_MAX_CTAS still overrides this.
|
||||
ncclConfig_t config = NCCL_CONFIG_INITIALIZER;
|
||||
const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc;
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10) {
|
||||
config.maxCTAs = 8;
|
||||
}
|
||||
|
||||
@@ -2047,6 +2128,9 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml
|
||||
ggml_type compute_type = src0->type;
|
||||
if (ggml_is_quantized(compute_type) || compute_type == GGML_TYPE_F8_E4M3) {
|
||||
compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32;
|
||||
if (ggml_cuda_info().devices[ctx.device].prefer_fp32_acc) {
|
||||
compute_type = GGML_TYPE_BF16;
|
||||
}
|
||||
} else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) {
|
||||
compute_type = GGML_TYPE_F32;
|
||||
} else if (compute_type == GGML_TYPE_BF16 && !fast_bf16_hardware_available(cc)) {
|
||||
@@ -3067,6 +3151,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
|
||||
case GGML_OP_FILL:
|
||||
ggml_cuda_op_fill(ctx, dst);
|
||||
break;
|
||||
case GGML_OP_SLEEP:
|
||||
ggml_cuda_op_sleep(ctx, dst);
|
||||
break;
|
||||
case GGML_OP_LIGHTNING_INDEXER:
|
||||
ggml_cuda_lightning_indexer(ctx, dst);
|
||||
break;
|
||||
@@ -4169,6 +4256,26 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph,
|
||||
return false;
|
||||
}
|
||||
|
||||
if ((ops.size() == 2 || ops.size() == 3) && ops.begin()[0] == GGML_OP_NORM && ops.begin()[1] == GGML_OP_MUL) {
|
||||
const ggml_tensor * norm = cgraph->nodes[node_idx];
|
||||
const ggml_tensor * mul = cgraph->nodes[node_idx + 1];
|
||||
const ggml_tensor * weight = mul->src[0] == norm ? mul->src[1] : mul->src[0];
|
||||
if (norm->src[0]->type != GGML_TYPE_F32 || norm->type != GGML_TYPE_F32 || mul->type != GGML_TYPE_F32 ||
|
||||
weight->type != GGML_TYPE_F32 || !ggml_is_vector(weight) || weight->ne[0] != norm->ne[0] ||
|
||||
!ggml_is_contiguous(weight)) {
|
||||
return false;
|
||||
}
|
||||
if (ops.size() == 3) {
|
||||
const ggml_tensor * add = cgraph->nodes[node_idx + 2];
|
||||
const ggml_tensor * bias = add->src[0] == mul ? add->src[1] : add->src[0];
|
||||
if (ops.begin()[2] != GGML_OP_ADD || add->type != GGML_TYPE_F32 || bias->type != GGML_TYPE_F32 ||
|
||||
!ggml_is_vector(bias) || bias->ne[0] != norm->ne[0] || !ggml_is_contiguous(bias)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
if ((ops.size() == 2 || ops.size() == 3) && ops.begin()[0] == GGML_OP_RMS_NORM && ops.begin()[1] == GGML_OP_MUL) {
|
||||
const ggml_tensor *rms_norm = cgraph->nodes[node_idx];
|
||||
const ggml_tensor *mul = cgraph->nodes[node_idx+1];
|
||||
@@ -5136,6 +5243,16 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph
|
||||
return 2;
|
||||
}
|
||||
|
||||
if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) {
|
||||
ggml_cuda_op_norm(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]);
|
||||
return 2;
|
||||
}
|
||||
|
||||
if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_NORM, GGML_OP_MUL }, {})) {
|
||||
ggml_cuda_op_norm(*cuda_ctx, node, cgraph->nodes[i + 1]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (ggml_cuda_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ADD }, {})) {
|
||||
ggml_cuda_op_rms_norm_fused_add(*cuda_ctx, node, cgraph->nodes[i + 1], cgraph->nodes[i + 2]);
|
||||
return 2;
|
||||
@@ -6564,7 +6681,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16) && op->type == op->src[0]->type &&
|
||||
ggml_is_contiguous_rows(op->src[0]);
|
||||
case GGML_OP_CLAMP:
|
||||
return ggml_is_contiguous_rows(op->src[0]);
|
||||
return ggml_is_contiguous_rows(op->src[0]) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
|
||||
case GGML_OP_ADD:
|
||||
case GGML_OP_SUB:
|
||||
case GGML_OP_MUL:
|
||||
@@ -6664,7 +6781,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
case GGML_OP_PAD_REFLECT_1D:
|
||||
case GGML_OP_ARANGE:
|
||||
case GGML_OP_TIMESTEP_EMBEDDING:
|
||||
return true;
|
||||
case GGML_OP_LEAKY_RELU:
|
||||
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
|
||||
case GGML_OP_RWKV_WKV6:
|
||||
case GGML_OP_GATED_LINEAR_ATTN:
|
||||
case GGML_OP_RWKV_WKV7:
|
||||
@@ -6690,13 +6809,21 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
case GGML_OP_FILL:
|
||||
case GGML_OP_CUMSUM:
|
||||
case GGML_OP_TRI:
|
||||
return true;
|
||||
case GGML_OP_DIAG:
|
||||
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
|
||||
case GGML_OP_SOLVE_TRI:
|
||||
return true;
|
||||
case GGML_OP_LIGHTNING_INDEXER:
|
||||
return ggml_cuda_lightning_indexer_supported(dev_ctx->device, op);
|
||||
case GGML_OP_TOP_K_MASK:
|
||||
return op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32;
|
||||
case GGML_OP_SLEEP:
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
return false;
|
||||
#else
|
||||
return true;
|
||||
#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -2,86 +2,86 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
@@ -162,14 +162,14 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ2_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true);
|
||||
@@ -198,14 +198,14 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ3_S, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
@@ -222,26 +222,26 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false);
|
||||
|
||||
// ---------------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -584,7 +584,7 @@ static __global__ void mul_mat_q_few(
|
||||
ggml_cuda_pdl_sync();
|
||||
}
|
||||
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
// DGX Spark DRAM serves the short row pieces of the stage copies slowly, so the rows of the block are first prefetched into L2 in address order.
|
||||
{
|
||||
const int tid = threadIdx.y*warp_size + threadIdx.x;
|
||||
@@ -599,7 +599,7 @@ static __global__ void mul_mat_q_few(
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
|
||||
#pragma unroll
|
||||
for (int stage = 0; stage < nstages - 1; ++stage) {
|
||||
@@ -880,14 +880,14 @@ static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const
|
||||
return 0;
|
||||
}
|
||||
// on DGX Spark the kernel needs its L2 prefetch, which is only in the DGX Spark device code
|
||||
if (cc == GGML_CUDA_CC_DGX_SPARK && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (cc == GGML_CUDA_CC_GB10 && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_GB10) {
|
||||
return 0;
|
||||
}
|
||||
// with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there (MUL_MAT_ID splits the rows instead)
|
||||
if (mmq_few_unit_size(type) == 0 || (!has_ids && ne00 < 8*QK_K)) {
|
||||
return 0;
|
||||
}
|
||||
if (cc == GGML_CUDA_CC_DGX_SPARK) { // tuned on DGX Spark GB10, also for MUL_MAT_ID
|
||||
if (cc == GGML_CUDA_CC_GB10) { // tuned on DGX Spark GB10, also for MUL_MAT_ID
|
||||
return type == GGML_TYPE_IQ4_XS || type == GGML_TYPE_IQ4_NL ? 8 : 2;
|
||||
}
|
||||
if (has_ids) { // after MMVQ, which is faster up to its max. batch size
|
||||
|
||||
@@ -460,7 +460,7 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t
|
||||
#endif //GGML_CUDA_FORCE_MMQ
|
||||
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc)) {
|
||||
return !fp16_mma_hardware_available(cc) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE;
|
||||
return !fp16_mma_hardware_available(cc) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE || n_experts > 0;
|
||||
}
|
||||
|
||||
if (amd_mfma_available(cc)) {
|
||||
|
||||
@@ -1094,7 +1094,7 @@ static __global__ void mul_mat_q(
|
||||
int col_high = min(col_low + J, ncols_dst);
|
||||
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
||||
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
||||
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
||||
int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
|
||||
|
||||
if (ids_dst) {
|
||||
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
|
||||
@@ -1176,7 +1176,7 @@ static __global__ void mul_mat_q(
|
||||
int col_high = min(col_low + J, ncols_dst);
|
||||
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
||||
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
||||
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
||||
int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
|
||||
|
||||
if (ids_dst) {
|
||||
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
|
||||
@@ -1257,7 +1257,7 @@ static __global__ void mul_mat_q(
|
||||
int col_high = min(col_low + J, ncols_dst);
|
||||
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
||||
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
||||
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
||||
int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
|
||||
|
||||
if (ids_dst) {
|
||||
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
|
||||
@@ -1403,7 +1403,7 @@ static __global__ void mul_mat_q_stream_k_fixup(
|
||||
const int it = tmp2.x;
|
||||
|
||||
if (!ids_dst) {
|
||||
const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst + it*I;
|
||||
const int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst + it*I;
|
||||
dst += offset_dst;
|
||||
|
||||
const int i_max = nrows_x - it*I - 1;
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
// only enabled on DGX Spark, where it is a gain on every type below. On the higher-bandwidth parts the kernel
|
||||
// has little exposed latency left to hide and the extra requests cost more than they save.
|
||||
// For perf data, see https://github.com/ggml-org/llama.cpp/pull/26705#issuecomment-5569335031
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
// returns true only for those quants that benefit from prefetch and false otherwise
|
||||
static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) {
|
||||
switch (type) {
|
||||
@@ -110,7 +110,7 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
|
||||
return MMVQ_PARAMETERS_GCN;
|
||||
#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA && __CUDA_ARCH__ < GGML_CUDA_CC_AMPERE
|
||||
return MMVQ_PARAMETERS_TURING;
|
||||
#elif __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#elif __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
return MMVQ_PARAMETERS_GB10;
|
||||
#else
|
||||
return MMVQ_PARAMETERS_GENERIC;
|
||||
@@ -133,7 +133,7 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA && ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_AMPERE) {
|
||||
return MMVQ_PARAMETERS_TURING;
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_GB10) {
|
||||
return MMVQ_PARAMETERS_GB10;
|
||||
}
|
||||
return MMVQ_PARAMETERS_GENERIC;
|
||||
@@ -342,7 +342,7 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
|
||||
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
||||
}
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10) {
|
||||
switch (type) { // tuned on DGX Spark GB10
|
||||
case GGML_TYPE_Q2_K:
|
||||
return ne11 <= 6;
|
||||
@@ -751,7 +751,7 @@ static __global__ void mul_mat_vec_q(
|
||||
// x block quant index when casting the quants to int
|
||||
const int kqs = vdr * (tid % (qi/vdr));
|
||||
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
// start the next iterations' weight loads early
|
||||
if constexpr (mmvq_should_prefetch(type)) {
|
||||
// loop iterations, not blocks; q3_K and q6_K keep the distance in blocks they had with vdr 1
|
||||
|
||||
+58
-14
@@ -2,10 +2,18 @@
|
||||
#include "convert.cuh"
|
||||
#include <cstdint>
|
||||
|
||||
template <int block_size>
|
||||
static __global__ void norm_f32(
|
||||
const float * x, float * dst, const int ncols, const int nchannels, const int nsamples,
|
||||
const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps) {
|
||||
template <int block_size, bool do_multiply = false, bool do_add = false>
|
||||
static __global__ void norm_f32(const float * x,
|
||||
float * dst,
|
||||
const int ncols,
|
||||
const int nchannels,
|
||||
const int nsamples,
|
||||
const int64_t stride_row,
|
||||
const int64_t stride_channel,
|
||||
const int64_t stride_sample,
|
||||
const float eps,
|
||||
const float * mul,
|
||||
const float * add) {
|
||||
const int nrows = gridDim.x;
|
||||
const int row = blockIdx.x;
|
||||
const int tid = threadIdx.x;
|
||||
@@ -36,7 +44,14 @@ static __global__ void norm_f32(
|
||||
const float inv_std = rsqrtf(var + eps);
|
||||
|
||||
for (int col = tid; col < ncols; col += block_size) {
|
||||
dstc[col] = (xc[col] - mean) * inv_std;
|
||||
float v = (xc[col] - mean) * inv_std;
|
||||
if constexpr (do_multiply) {
|
||||
v *= mul[col];
|
||||
}
|
||||
if constexpr (do_add) {
|
||||
v += add[col];
|
||||
}
|
||||
dstc[col] = v;
|
||||
}
|
||||
|
||||
if constexpr (block_size > WARP_SIZE) {
|
||||
@@ -384,16 +399,33 @@ static __global__ void l2_norm_f32(
|
||||
}
|
||||
}
|
||||
|
||||
static void norm_f32_cuda(
|
||||
const float * x, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples,
|
||||
const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, cudaStream_t stream) {
|
||||
template <bool do_multiply = false, bool do_add = false>
|
||||
static void norm_f32_cuda(const float * x,
|
||||
float * dst,
|
||||
const int ncols,
|
||||
const int nrows,
|
||||
const int nchannels,
|
||||
const int nsamples,
|
||||
const int64_t stride_row,
|
||||
const int64_t stride_channel,
|
||||
const int64_t stride_sample,
|
||||
const float eps,
|
||||
cudaStream_t stream,
|
||||
const float * mul = nullptr,
|
||||
const float * add = nullptr) {
|
||||
const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX));
|
||||
if (ncols < 1024) {
|
||||
if (do_multiply && ncols <= 2048) {
|
||||
const dim3 block_dims(256, 1, 1);
|
||||
norm_f32<256, do_multiply, do_add><<<blocks_num, block_dims, 32 * sizeof(float2), stream>>>(
|
||||
x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, add);
|
||||
} else if (!do_multiply && ncols < 1024) {
|
||||
const dim3 block_dims(WARP_SIZE, 1, 1);
|
||||
norm_f32<WARP_SIZE><<<blocks_num, block_dims, 0, stream>>>(x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps);
|
||||
norm_f32<WARP_SIZE, do_multiply, do_add><<<blocks_num, block_dims, 0, stream>>>(
|
||||
x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, add);
|
||||
} else {
|
||||
const dim3 block_dims(1024, 1, 1);
|
||||
norm_f32<1024><<<blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float2): 0, stream>>>(x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps);
|
||||
norm_f32<1024, do_multiply, do_add><<<blocks_num, block_dims, 32 * sizeof(float2), stream>>>(
|
||||
x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, mul, add);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -591,10 +623,10 @@ static void l2_norm_f32_cuda(
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * mul, ggml_tensor * add) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const float * src0_d = (const float *) src0->data;
|
||||
float * dst_d = (float *) dst->data;
|
||||
float * dst_d = (float *) (add ? add : mul ? mul : dst)->data;
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||
@@ -612,7 +644,19 @@ void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const int64_t s02 = nb02 / ts0;
|
||||
const int64_t s03 = nb03 / ts0;
|
||||
|
||||
norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);
|
||||
if (mul) {
|
||||
const ggml_tensor * weight = mul->src[0] == dst ? mul->src[1] : mul->src[0];
|
||||
const float * weight_d = (const float *) weight->data;
|
||||
if (add) {
|
||||
const ggml_tensor * bias = add->src[0] == mul ? add->src[1] : add->src[0];
|
||||
norm_f32_cuda<true, true>(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream, weight_d,
|
||||
(const float *) bias->data);
|
||||
} else {
|
||||
norm_f32_cuda<true, false>(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream, weight_d);
|
||||
}
|
||||
} else {
|
||||
norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream);
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
#include "common.cuh"
|
||||
|
||||
void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx,
|
||||
ggml_tensor * dst,
|
||||
ggml_tensor * mul = nullptr,
|
||||
ggml_tensor * add = nullptr);
|
||||
|
||||
void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#include "sleep.cuh"
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
// %globaltimer is a nanosecond wall clock, unlike clock64() it is unaffected by the SM clock and by frequency scaling
|
||||
static __device__ __forceinline__ uint64_t globaltimer_ns() {
|
||||
uint64_t t;
|
||||
asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
|
||||
return t;
|
||||
}
|
||||
|
||||
// a single thread is enough, the next kernel on the same stream cannot start before this one retires
|
||||
static __global__ void sleep_ns(const uint64_t ns) {
|
||||
const uint64_t t0 = globaltimer_ns();
|
||||
|
||||
while (globaltimer_ns() - t0 < ns) {}
|
||||
}
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
sleep_ns<<<1, 1, 0, ctx.stream()>>>(1000*(uint64_t) ggml_get_op_params_i32(dst, 0));
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
GGML_UNUSED(ctx);
|
||||
GGML_UNUSED(dst);
|
||||
GGML_ABORT("GGML_OP_SLEEP requires the %%globaltimer register, which is only available on CUDA");
|
||||
}
|
||||
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
@@ -0,0 +1,3 @@
|
||||
#include "common.cuh"
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
@@ -148,13 +148,14 @@ static void ssm_conv_f32_cuda(const float * src0, const float * src1, const floa
|
||||
};
|
||||
|
||||
switch (nc) {
|
||||
case 2: launch_kernel(std::integral_constant<int, 2 >{}); break;
|
||||
case 3: launch_kernel(std::integral_constant<int, 3 >{}); break;
|
||||
case 4: launch_kernel(std::integral_constant<int, 4 >{}); break;
|
||||
case 5: launch_kernel(std::integral_constant<int, 5 >{}); break;
|
||||
case 7: launch_kernel(std::integral_constant<int, 7 >{}); break;
|
||||
case 9: launch_kernel(std::integral_constant<int, 9 >{}); break;
|
||||
case 15: launch_kernel(std::integral_constant<int, 15>{}); break;
|
||||
default: GGML_ABORT("Only support kernel sizes 3, 4, 5, 7, 9, 15 right now.");
|
||||
default: GGML_ABORT("Only support kernel sizes 2, 3, 4, 5, 7, 9, 15 right now.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2551,6 +2551,26 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_
|
||||
char base[256];
|
||||
char name[256];
|
||||
|
||||
const bool has_tensor = ggml_metal_device_get_props(ggml_metal_library_get_device(lib))->has_tensor;
|
||||
|
||||
if (has_tensor) {
|
||||
// nr0 output positions per threadgroup, nr1 output channels, same tiles as kernel_mul_mm
|
||||
snprintf(base, 256, "kernel_conv_2d_mm_tensor_%s_%s", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type));
|
||||
snprintf(name, 256, "%s", base);
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
|
||||
}
|
||||
|
||||
res.nr0 = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y;
|
||||
res.nr1 = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X;
|
||||
res.nsg = N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y;
|
||||
res.smem = res.nr0 * N_MM_NK_TOTAL * sizeof(ggml_fp16_t);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
const int nr0 = op->src[0]->ne[3] <= 32 ? 32 : 64;
|
||||
const int nr1 = 2048/nr0;
|
||||
|
||||
@@ -2564,13 +2584,27 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_
|
||||
|
||||
res.nr0 = nr0;
|
||||
res.nr1 = nr1;
|
||||
res.nsg = 4;
|
||||
res.nsg = N_SG_CONV_2D;
|
||||
|
||||
res.smem = nr0*nr1*sizeof(float);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm_reduce(ggml_metal_library_t lib, const ggml_tensor * op) {
|
||||
assert(op->op == GGML_OP_CONV_2D);
|
||||
GGML_UNUSED(op);
|
||||
|
||||
const char * name = "kernel_conv_2d_mm_reduce";
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw(ggml_metal_library_t lib, const ggml_tensor * op, bool tiled) {
|
||||
assert(op->op == GGML_OP_CONV_2D_DW);
|
||||
|
||||
|
||||
@@ -170,6 +170,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake (ggml_metal_library_t lib, enum ggml_type type);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm_reduce (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tiled);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
|
||||
@@ -2019,7 +2019,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
||||
};
|
||||
}
|
||||
case GGML_OP_DIAG:
|
||||
return true;
|
||||
return op->src[0]->type == GGML_TYPE_F32;
|
||||
case GGML_OP_OPT_STEP_ADAMW:
|
||||
case GGML_OP_OPT_STEP_SGD:
|
||||
return has_simdgroup_reduction;
|
||||
@@ -2375,7 +2375,7 @@ void ggml_metal_buffer_memset_tensor(ggml_metal_buffer_t buf, struct ggml_tensor
|
||||
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
||||
|
||||
[encoder fillBuffer:bid_dst.metal
|
||||
range:NSMakeRange(bid_dst.offs, bid_dst.offs + size)
|
||||
range:NSMakeRange(bid_dst.offs, size)
|
||||
value:value];
|
||||
|
||||
[encoder endEncoding];
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
#define N_ARGMAX_NPART_MAX 256
|
||||
#define N_ARGMAX_CHUNK 4096
|
||||
|
||||
#define N_SG_CONV_2D 4
|
||||
|
||||
// kernel parameters for mat-vec threadgroups
|
||||
//
|
||||
// N_R0: number of src0 rows to process per simdgroup
|
||||
@@ -752,8 +754,14 @@ typedef struct {
|
||||
int32_t p1;
|
||||
int32_t d0;
|
||||
int32_t d1;
|
||||
int32_t nk; // reduction length per split
|
||||
} ggml_metal_kargs_conv_2d;
|
||||
|
||||
typedef struct {
|
||||
int32_t ne; // output elements
|
||||
int32_t nsplit;
|
||||
} ggml_metal_kargs_conv_2d_reduce;
|
||||
|
||||
typedef struct {
|
||||
uint64_t nb00; // kernel strides
|
||||
uint64_t nb01;
|
||||
|
||||
@@ -4959,6 +4959,33 @@ int ggml_metal_op_im2col(ggml_metal_op_t ctx, int idx) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// number of reduction splits of the conv_2d matmul kernels
|
||||
// A small output with a long reduction leaves the GPU idle, so the reduction is split until there are about 1M partial sums.
|
||||
static int ggml_metal_op_conv_2d_nsplit(const ggml_tensor * op, int32_t * nk) {
|
||||
const int64_t K = op->src[0]->ne[0]*op->src[0]->ne[1]*op->src[0]->ne[2];
|
||||
const int64_t nsteps = (K + 31)/32;
|
||||
const int64_t ne = ggml_nelements(op);
|
||||
|
||||
// at least 3 reduction steps of 32 per split, at most 32 splits
|
||||
const int64_t nsplit = std::max<int64_t>(1, std::min<int64_t>({ (1024*1024 + ne - 1)/ne, nsteps/3, 32 }));
|
||||
const int64_t steps = (nsteps + nsplit - 1)/nsplit;
|
||||
|
||||
if (nk) {
|
||||
*nk = (int32_t) (steps*32);
|
||||
}
|
||||
|
||||
return (int) ((nsteps + steps - 1)/steps);
|
||||
}
|
||||
|
||||
size_t ggml_metal_op_conv_2d_extra_tmp(const ggml_tensor * op) {
|
||||
assert(op->op == GGML_OP_CONV_2D);
|
||||
|
||||
const int nsplit = ggml_metal_op_conv_2d_nsplit(op, nullptr);
|
||||
|
||||
// partial sums of the reduction splits
|
||||
return nsplit > 1 ? nsplit*ggml_nelements(op)*sizeof(float) : 0;
|
||||
}
|
||||
|
||||
int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_tensor * op = ctx->node(idx);
|
||||
|
||||
@@ -5012,11 +5039,17 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
/*.p1 =*/ p1,
|
||||
/*.d0 =*/ d0,
|
||||
/*.d1 =*/ d1,
|
||||
/*.nk =*/ 0,
|
||||
};
|
||||
|
||||
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev);
|
||||
|
||||
if (props_dev->has_simdgroup_mm) {
|
||||
// the matmul kernels walk the patches with 32-bit byte offsets inside an image
|
||||
// larger images, possibly a strided view, take the generic kernel
|
||||
const uint64_t max_off = (ne12 - 1)*nb12 + (ne11 - 1)*nb11 + (ne10 - 1)*nb10 + sizeof(float);
|
||||
const bool fits_32 = max_off <= UINT32_MAX;
|
||||
|
||||
if (props_dev->has_simdgroup_mm && fits_32) {
|
||||
auto pipeline = ggml_metal_library_get_pipeline_conv_2d_mm(lib, op);
|
||||
|
||||
const int nr0 = pipeline.nr0;
|
||||
@@ -5026,15 +5059,48 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
const int64_t NP = ne3*ne1*ne0;
|
||||
const int64_t OC = ne03;
|
||||
|
||||
const int nsplit = ggml_metal_op_conv_2d_nsplit(op, &args.nk);
|
||||
|
||||
ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);
|
||||
ggml_metal_buffer_id bid_tmp = bid_dst;
|
||||
bid_tmp.offs += ggml_nbytes(op);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline);
|
||||
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
|
||||
ggml_metal_encoder_set_buffer (enc, nsplit > 1 ? bid_tmp : bid_dst, 3);
|
||||
|
||||
ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, 1, 32, nsg, 1);
|
||||
if (props_dev->has_tensor) {
|
||||
// per image: nr0 positions times nr1 channels per threadgroup
|
||||
const int64_t OHW = ne1*ne0;
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (OC + nr1 - 1)/nr1, (OHW + nr0 - 1)/nr0, ne3*nsplit, 32, nsg, 1);
|
||||
} else {
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, nsplit, 32, nsg, 1);
|
||||
}
|
||||
|
||||
if (nsplit > 1) {
|
||||
GGML_ASSERT(ggml_is_contiguous(op));
|
||||
|
||||
// sync the 2 kernels
|
||||
ggml_metal_op_concurrency_reset(ctx);
|
||||
|
||||
ggml_metal_kargs_conv_2d_reduce args0 = {
|
||||
/*.ne =*/ (int32_t) ggml_nelements(op),
|
||||
/*.nsplit =*/ nsplit,
|
||||
};
|
||||
|
||||
auto pipeline0 = ggml_metal_library_get_pipeline_conv_2d_mm_reduce(lib, op);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline0);
|
||||
ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_tmp, 1);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_dst, 2);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (args0.ne + 255)/256, 1, 1, 256, 1, 1);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -49,6 +49,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_rows(const struct ggml_tensor * op);
|
||||
|
||||
size_t ggml_metal_op_argmax_extra(const struct ggml_tensor * op);
|
||||
|
||||
size_t ggml_metal_op_conv_2d_extra_tmp(const struct ggml_tensor * op);
|
||||
|
||||
int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_acc (ggml_metal_op_t ctx, int idx);
|
||||
|
||||
@@ -311,6 +311,10 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
|
||||
{
|
||||
res = 2*sizeof(int32_t)*ggml_nelements(tensor->src[0]);
|
||||
} break;
|
||||
case GGML_OP_CONV_2D:
|
||||
{
|
||||
res += ggml_metal_op_conv_2d_extra_tmp(tensor);
|
||||
} break;
|
||||
case GGML_OP_ARGMAX:
|
||||
{
|
||||
res += ggml_metal_op_argmax_extra(tensor);
|
||||
|
||||
@@ -285,7 +285,7 @@ kernel void kernel_conv_2d_mm(
|
||||
ushort tiitg[[thread_index_in_threadgroup]],
|
||||
ushort tiisg[[thread_index_in_simdgroup]],
|
||||
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
|
||||
constexpr short NSG = 4;
|
||||
constexpr short NSG = N_SG_CONV_2D;
|
||||
constexpr short NTH = 32*NSG;
|
||||
constexpr short NK = 32;
|
||||
constexpr short NKB = NK/8;
|
||||
@@ -300,6 +300,10 @@ kernel void kernel_conv_2d_mm(
|
||||
const int K = args.IC*args.KH*args.KW;
|
||||
const int NP = args.N*args.OH*args.OW;
|
||||
|
||||
// this threadgroup reduces [k_begin, k_end), the splits are summed by kernel_conv_2d_mm_reduce
|
||||
const int k_begin = tgpig.z*args.nk;
|
||||
const int k_end = min(K, k_begin + args.nk);
|
||||
|
||||
const int r0 = tgpig.y*NR0;
|
||||
const int r1 = tgpig.x*NR1;
|
||||
|
||||
@@ -308,8 +312,14 @@ kernel void kernel_conv_2d_mm(
|
||||
const short lb_n = 8*sgitg + tiisg%8;
|
||||
const short lb_k = 8*(tiisg/8);
|
||||
|
||||
// the patches are walked with 32-bit byte offsets inside the image, the host checks that an image fits
|
||||
const uint nb10 = args.nb10;
|
||||
const uint nb11 = args.nb11;
|
||||
const uint nb12 = args.nb12;
|
||||
|
||||
int ix0[NPL];
|
||||
int iy0[NPL];
|
||||
uint off0[NPL];
|
||||
bool ip_ok[NPL];
|
||||
|
||||
device const char * srcb[NPL];
|
||||
@@ -324,6 +334,8 @@ kernel void kernel_conv_2d_mm(
|
||||
ix0[t] = iow*args.s0 - args.p0;
|
||||
iy0[t] = ioh*args.s1 - args.p1;
|
||||
|
||||
off0[t] = iy0[t]*nb11 + ix0[t]*nb10;
|
||||
|
||||
ip_ok[t] = ip < NP;
|
||||
|
||||
srcb[t] = src + in*args.nb13;
|
||||
@@ -337,7 +349,7 @@ kernel void kernel_conv_2d_mm(
|
||||
mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
|
||||
}
|
||||
|
||||
for (int k0 = 0; k0 < K; k0 += NK) {
|
||||
for (int k0 = k_begin; k0 < k_end; k0 += NK) {
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
{
|
||||
@@ -349,7 +361,7 @@ kernel void kernel_conv_2d_mm(
|
||||
const int oc = r0 + row;
|
||||
|
||||
half v = 0;
|
||||
if (kk < K && oc < args.OC) {
|
||||
if (kk < k_end && oc < args.OC) {
|
||||
v = *((device const TK *)(weights + oc*args.nb03) + kk);
|
||||
}
|
||||
|
||||
@@ -364,7 +376,11 @@ kernel void kernel_conv_2d_mm(
|
||||
|
||||
int kw = kk % args.KW;
|
||||
int kh = (kk / args.KW) % args.KH;
|
||||
int ic = kk / (args.KW*args.KH);
|
||||
int dx = kw*args.d0;
|
||||
int dy = kh*args.d1;
|
||||
|
||||
// byte offset of the reduction index kk, relative to the patch origin
|
||||
uint offk = (kk / (args.KW*args.KH))*nb12 + dy*nb11 + dx*nb10;
|
||||
|
||||
FOR_UNROLL (short i = 0; i < 8; i++) {
|
||||
const short k = lb_k + i;
|
||||
@@ -372,12 +388,12 @@ kernel void kernel_conv_2d_mm(
|
||||
FOR_UNROLL (short t = 0; t < NPL; t++) {
|
||||
const short n = 32*t + lb_n;
|
||||
|
||||
const int ix = ix0[t] + kw*args.d0;
|
||||
const int iy = iy0[t] + kh*args.d1;
|
||||
const int ix = ix0[t] + dx;
|
||||
const int iy = iy0[t] + dy;
|
||||
|
||||
half v = 0;
|
||||
if (ip_ok[t] && kk < K && ix >= 0 && ix < args.IW && iy >= 0 && iy < args.IH) {
|
||||
v = *(device const float *)(srcb[t] + ic*args.nb12 + iy*args.nb11 + ix*args.nb10);
|
||||
if (ip_ok[t] && kk < k_end && (uint) ix < (uint) args.IW && (uint) iy < (uint) args.IH) {
|
||||
v = *(device const float *)(srcb[t] + (off0[t] + offk));
|
||||
}
|
||||
|
||||
const short ib = NKB*(n/8) + k/8;
|
||||
@@ -386,11 +402,17 @@ kernel void kernel_conv_2d_mm(
|
||||
}
|
||||
|
||||
kk++;
|
||||
dx += args.d0;
|
||||
offk += args.d0*nb10;
|
||||
if (++kw == args.KW) {
|
||||
kw = 0;
|
||||
kw = 0;
|
||||
dx = 0;
|
||||
dy += args.d1;
|
||||
offk += args.d1*nb11 - args.KW*args.d0*nb10;
|
||||
if (++kh == args.KH) {
|
||||
kh = 0;
|
||||
ic++;
|
||||
kh = 0;
|
||||
dy = 0;
|
||||
offk += nb12 - args.KH*args.d1*nb11;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -398,7 +420,7 @@ kernel void kernel_conv_2d_mm(
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
const short nkb = min((K - k0 + 7)/8, (int) NKB);
|
||||
const short nkb = min((k_end - k0 + 7)/8, (int) NKB);
|
||||
|
||||
for (short ik = 0; ik < nkb; ik++) {
|
||||
FOR_UNROLL (short i = 0; i < 4; i++) {
|
||||
@@ -436,7 +458,8 @@ kernel void kernel_conv_2d_mm(
|
||||
const int oh = (p / args.OW) % args.OH;
|
||||
const int n = p / (args.OW*args.OH);
|
||||
|
||||
device char * dstp = dst + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
|
||||
// with several splits, dst is the partial sum buffer, one contiguous output per split
|
||||
device char * dstp = dst + (uint64_t) tgpig.z*args.N*args.nb3 + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
|
||||
|
||||
for (short j = 0; j < NR0*NR1/NTH; j++) {
|
||||
const short row = tiitg/NR1 + (NTH/NR1)*j;
|
||||
@@ -448,6 +471,158 @@ kernel void kernel_conv_2d_mm(
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef GGML_METAL_HAS_TENSOR
|
||||
// conv_2d as a matmul on the tensor cores, like kernel_mul_mm
|
||||
// The patches of NRA output positions are gathered into threadgroup memory, the kernel rows are read from device memory.
|
||||
// One threadgroup computes NRA positions of one image times NRB output channels, over the reduction range of its split.
|
||||
template <typename TK>
|
||||
kernel void kernel_conv_2d_mm_tensor(
|
||||
constant ggml_metal_kargs_conv_2d & args,
|
||||
device const char * weights,
|
||||
device const char * src,
|
||||
device char * dst,
|
||||
threadgroup char * shmem [[threadgroup(0)]],
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
uint3 tgpg[[threadgroups_per_grid]],
|
||||
ushort tiitg[[thread_index_in_threadgroup]]) {
|
||||
constexpr int NRB = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X; // output channels
|
||||
constexpr int NRA = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y; // output positions
|
||||
constexpr int NK = N_MM_NK_TOTAL;
|
||||
constexpr int NTH = N_SIMDWIDTH * N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y;
|
||||
|
||||
// each thread gathers NK/(NTH/NRA) consecutive reduction indices of one position
|
||||
constexpr int KPT = NK/(NTH/NRA);
|
||||
|
||||
const int K = args.IC*args.KH*args.KW;
|
||||
const int OHW = args.OH*args.OW;
|
||||
|
||||
const int nsplit = tgpg.z/args.N;
|
||||
const int img = tgpig.z/nsplit;
|
||||
const int split = tgpig.z%nsplit;
|
||||
|
||||
const int k_begin = split*args.nk;
|
||||
const int k_end = min(K, k_begin + args.nk);
|
||||
|
||||
const int ra = tgpig.y*NRA;
|
||||
const int rb = tgpig.x*NRB;
|
||||
|
||||
threadgroup half * sa = (threadgroup half *) shmem;
|
||||
|
||||
// the position this thread gathers
|
||||
const int row = tiitg%NRA;
|
||||
const int kc = (tiitg/NRA)*KPT;
|
||||
const int p = ra + row;
|
||||
const bool pok = p < OHW;
|
||||
const int ix0 = (p%args.OW)*args.s0 - args.p0;
|
||||
const int iy0 = (p/args.OW)*args.s1 - args.p1;
|
||||
|
||||
device const char * srcb = src + img*args.nb13;
|
||||
|
||||
// the patch is walked with 32-bit byte offsets inside the image, the host checks that an image fits
|
||||
const uint nb10 = args.nb10;
|
||||
const uint nb11 = args.nb11;
|
||||
const uint nb12 = args.nb12;
|
||||
|
||||
const uint off0 = iy0*nb11 + ix0*nb10;
|
||||
|
||||
device TK * ptrB = (device TK *) weights;
|
||||
const int strideB = args.nb03/sizeof(TK);
|
||||
|
||||
mpp::tensor_ops::matmul2d<
|
||||
mpp::tensor_ops::matmul2d_descriptor(
|
||||
NRB, NRA, static_cast<int>(dynamic_extent), false, true, true,
|
||||
mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
|
||||
execution_simdgroups<N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y>> mm;
|
||||
|
||||
auto tB0 = tensor(ptrB, dextents<int32_t, 2>(K, args.OC), array<int, 2>({1, strideB}));
|
||||
auto tA0 = tensor(sa, dextents<int32_t, 2>(NK, NRA));
|
||||
auto cT = mm.get_destination_cooperative_tensor<decltype(tB0), decltype(tA0), float>();
|
||||
|
||||
for (int k0 = k_begin; k0 < k_end; k0 += NK) {
|
||||
// the reduction indices past k_end are beyond the extent given to the matmul, they are not gathered
|
||||
if (k0 + kc < k_end) {
|
||||
// gather KPT patch values of this position, walking the reduction index incrementally
|
||||
int kk = k0 + kc;
|
||||
int kw = kk % args.KW;
|
||||
int kh = (kk / args.KW) % args.KH;
|
||||
int dx = kw*args.d0;
|
||||
int dy = kh*args.d1;
|
||||
|
||||
// byte offset of the reduction index kk, relative to the patch origin
|
||||
uint offk = (kk / (args.KW*args.KH))*nb12 + dy*nb11 + dx*nb10;
|
||||
|
||||
FOR_UNROLL (short i = 0; i < KPT; i++) {
|
||||
const int ix = ix0 + dx;
|
||||
const int iy = iy0 + dy;
|
||||
|
||||
half v = 0;
|
||||
if (pok && kk < k_end && (uint) ix < (uint) args.IW && (uint) iy < (uint) args.IH) {
|
||||
v = *(device const float *)(srcb + (off0 + offk));
|
||||
}
|
||||
sa[row*NK + kc + i] = v;
|
||||
|
||||
kk++;
|
||||
dx += args.d0;
|
||||
offk += args.d0*nb10;
|
||||
if (++kw == args.KW) {
|
||||
kw = 0;
|
||||
dx = 0;
|
||||
dy += args.d1;
|
||||
offk += args.d1*nb11 - args.KW*args.d0*nb10;
|
||||
if (++kh == args.KH) {
|
||||
kh = 0;
|
||||
dy = 0;
|
||||
offk += nb12 - args.KH*args.d1*nb11;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
// clamp the reduction extent so neither operand is read past the end of the split
|
||||
const int kExt = min(NK, k_end - k0);
|
||||
|
||||
auto tAv = tensor(sa, dextents<int32_t, 2>(kExt, NRA), array<int, 2>({1, NK}));
|
||||
auto tBv = tensor(ptrB + k0 + rb*strideB, dextents<int32_t, 2>(kExt, args.OC - rb), array<int, 2>({1, strideB}));
|
||||
|
||||
mm.run(tBv, tAv, cT);
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
}
|
||||
|
||||
// dst holds [OHW, OC] per image, with several splits it is the partial sum buffer, one contiguous output per split
|
||||
const int tileM = min(NRA, OHW - ra);
|
||||
const int tileN = min(NRB, args.OC - rb);
|
||||
|
||||
device float * dstTile = (device float *) (dst + (uint64_t) split*args.N*args.nb3 + img*args.nb3) + (uint64_t) rb*OHW + ra;
|
||||
|
||||
auto tD = tensor(dstTile, dextents<int32_t, 2>(tileM, tileN), array<int, 2>({1, OHW}));
|
||||
cT.store(tD);
|
||||
}
|
||||
|
||||
typedef decltype(kernel_conv_2d_mm_tensor<float>) kernel_conv_2d_mm_tensor_t;
|
||||
|
||||
template [[host_name("kernel_conv_2d_mm_tensor_f32_f32")]] kernel kernel_conv_2d_mm_tensor_t kernel_conv_2d_mm_tensor<float>;
|
||||
template [[host_name("kernel_conv_2d_mm_tensor_f16_f32")]] kernel kernel_conv_2d_mm_tensor_t kernel_conv_2d_mm_tensor<half>;
|
||||
#endif
|
||||
|
||||
kernel void kernel_conv_2d_mm_reduce(
|
||||
constant ggml_metal_kargs_conv_2d_reduce & args,
|
||||
device const float * tmp,
|
||||
device float * dst,
|
||||
uint tpig[[thread_position_in_grid]]) {
|
||||
if ((int) tpig >= args.ne) {
|
||||
return;
|
||||
}
|
||||
|
||||
float sum = 0.0f;
|
||||
for (int s = 0; s < args.nsplit; ++s) {
|
||||
sum += tmp[(uint64_t) s*args.ne + tpig];
|
||||
}
|
||||
dst[tpig] = sum;
|
||||
}
|
||||
|
||||
typedef decltype(kernel_conv_2d_mm<float, 64, 32>) kernel_conv_2d_mm_t;
|
||||
|
||||
template [[host_name("kernel_conv_2d_mm_f32_f32_64x32")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<float, 64, 32>;
|
||||
|
||||
@@ -12283,7 +12283,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
|
||||
case GGML_OP_TRANSPOSE:
|
||||
return true;
|
||||
case GGML_OP_DIAG:
|
||||
return true;
|
||||
return op->src[0]->type == GGML_TYPE_F32;
|
||||
case GGML_OP_ROPE: {
|
||||
const int mode = ((const int32_t *) op->op_params)[2];
|
||||
if (!ggml_rope_get_truncate(op)) {
|
||||
@@ -23277,8 +23277,31 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
|
||||
const int64_t kqv_m = pad_up(dv, 64);
|
||||
const int64_t tile_n = pad_up(n_q_chunk, 32);
|
||||
|
||||
// Pad the int8 row pitch by one cache line, as in the P8 KQV kernel.
|
||||
const int kv_pitch = (int) n_kv + 64;
|
||||
static const int64_t kv_pitch_round = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND");
|
||||
const int64_t v = (e && e[0]) ? strtoll(e, nullptr, 10) : 32768;
|
||||
return v > 0 && v <= INT_MAX && (v & (v - 1)) == 0 ? v : (int64_t) 0;
|
||||
}();
|
||||
static const int64_t kv_pitch_round_min = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND_MIN");
|
||||
return (e && e[0]) ? strtoll(e, nullptr, 10) : (int64_t) 16384;
|
||||
}();
|
||||
static const int64_t kv_pitch_round_max_dv = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND_MAX_DV");
|
||||
return (e && e[0]) ? strtoll(e, nullptr, 10) : (int64_t) 256;
|
||||
}();
|
||||
|
||||
// Round long P and V^T rows to reduce cache conflicts, then add one cache line.
|
||||
int64_t kv_pitch_value = n_kv + 64;
|
||||
if (kv_pitch_round > 0 && n_kv > kv_pitch_round_min && dv <= kv_pitch_round_max_dv) {
|
||||
const int64_t rounded_pitch = pad_up(n_kv, kv_pitch_round) + 64;
|
||||
if (rounded_pitch <= INT_MAX &&
|
||||
(size_t) rounded_pitch <= SIZE_MAX / n_q_chunk / n_head &&
|
||||
(size_t) rounded_pitch <= SIZE_MAX / dv / n_head_kv) {
|
||||
kv_pitch_value = rounded_pitch;
|
||||
}
|
||||
}
|
||||
const int kv_pitch = (int) kv_pitch_value;
|
||||
const size_t nblk = (size_t) kv_pitch / 32;
|
||||
|
||||
backend_ctx->prealloc_fa_pq.allocate(backend_ctx->context, (size_t) kv_pitch * n_q_chunk * n_head);
|
||||
|
||||
@@ -31,3 +31,20 @@ if (GGML_OPENVINO)
|
||||
message(FATAL_ERROR "OpenVINO: OpenVINO toolkit supports x86-64 and arm64 but not ${CMAKE_SYSTEM_PROCESSOR}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# Released runtimes retain the ordinary cache translators.
|
||||
include(CheckCXXSourceCompiles)
|
||||
unset(GGML_OPENVINO_HAS_CACHE_UPDATE CACHE)
|
||||
set(CMAKE_REQUIRED_LIBRARIES openvino::runtime)
|
||||
set(CMAKE_REQUIRED_INCLUDES "${CMAKE_CURRENT_SOURCE_DIR}/openvino/op")
|
||||
check_cxx_source_compiles("#include <recurrent_cache_update.hpp>
|
||||
int main() {
|
||||
auto conv = std::make_shared<ov::op::internal::RecurrentConvCacheUpdate>(ov::OutputVector{}, 3);
|
||||
auto gdn = std::make_shared<ov::op::internal::RecurrentGatedDeltaNetUpdate>(ov::OutputVector{});
|
||||
}"
|
||||
GGML_OPENVINO_HAS_CACHE_UPDATE)
|
||||
unset(CMAKE_REQUIRED_LIBRARIES)
|
||||
unset(CMAKE_REQUIRED_INCLUDES)
|
||||
if(GGML_OPENVINO_HAS_CACHE_UPDATE)
|
||||
target_compile_definitions(ggml-openvino PRIVATE GGML_OPENVINO_HAS_CACHE_UPDATE=1)
|
||||
endif()
|
||||
|
||||
@@ -1,3 +1,38 @@
|
||||
# Qwen MTP
|
||||
|
||||
The default Qwen3.5/Qwen3.6 GPU path is dynamic stateless execution with in-place fused recurrent ops and stable MTP sampling. Build against the matching OpenVINO runtime/GPU plugin with independent `RecurrentConvCacheUpdate` and `RecurrentGatedDeltaNetUpdate` ops. The native Update ops return computation results and update writable cache inputs in place, including rollback snapshots where required. No recurrent/sampling opt-in flag is needed on supported graphs. See the [development documentation index](dev_doc/README.md) for the implementation and the MTP sampling graph.
|
||||
|
||||
Use `--spec-type draft-mtp --spec-draft-n-max 2` with a GGUF that contains MTP weights. Parallel execution uses `-np` and `--kv-unified`. CPU snapshot execution, NPU execution, and forced static MTP are not supported.
|
||||
|
||||
Stateless recurrent execution uses the same reset, reorder, and partial cache-writeback paths at every slot capacity, including one slot without MTP.
|
||||
|
||||
The matched stateless GPU path uses `RecurrentConvCacheUpdate` and
|
||||
`RecurrentGatedDeltaNetUpdate`. Each cache is a writable Parameter backed by one
|
||||
stable caller-owned USM allocation. The ops return only convolution or attention;
|
||||
there are no recurrent cache Results. Required source rows are staged before
|
||||
writes, preserving resets, cyclic reorders and rollback. Cache readers must be
|
||||
exclusive, including Reshape aliases.
|
||||
`GGML_OPENVINO_DISABLE_FUSED_RECURRENCE=1` retains the ordinary cache decomposition.
|
||||
Unmatched snapshot GDN uses ordinary GDN token prefixes with functional writeback.
|
||||
Build both repos against the matching single-output op spec. Translation revision
|
||||
15 requires regenerating compiled-model blobs once without cache-only mode.
|
||||
|
||||
Cache-only MTP serving still requires the separate OpenVINO dynamic output-binding fix (PR #39354). Removing recurrent cache Results does not eliminate that independent trigger in imported MTP graphs.
|
||||
|
||||
Set `GGML_OPENVINO_ENABLE_KV_4D=1` to enable the upstream stateless in-place
|
||||
attention KV update pattern where eligible. It binds full-capacity cache tensors
|
||||
and lets the GPU fusion expose the live prefix to attention. The current upstream
|
||||
eligibility check supports one physical stream (including unified KV); explicit
|
||||
split multi-slot KV retains the ordinary full-cache path. This option does not
|
||||
change recurrent cache handling. Unified KV is recommended for multi-slot GPU
|
||||
execution because it enables this in-place attention update.
|
||||
|
||||
The recommended Qwen3.6-35B-A3B Q4_K_M setup uses GPU with `GGML_OPENVINO_REQUANT_KQUANT=q4_asym64_all` and `GGML_OPENVINO_NATIVE_SOFTPLUS=1`. Leave `GGML_OPENVINO_KV_SLICE` at its default (`1`). See [attention KV layout](dev_doc/OPENVINO-RECURRENT-CACHE.md#attention-kv-layout) for eligibility.
|
||||
|
||||
`q4_asym64_all` converts Q4_K/Q5_K/Q6_K/Q8_0/Q5_1 weights, including dense Q8_0 projections, to asymmetric INT4 with group size 64. `token_embd.weight` and `output.weight` retain symmetric channel-wise INT8. Q8_0/Q5_1 rows whose width is not divisible by 64 also use channel-wise INT8. This follows the primary/backup precision scheme of NNCF `INT4_ASYM`, `ratio=1.0`, `group_size=64`, `backup_mode=INT8_SYM` for these supported source types; it requantizes GGUF values rather than the original floating-point model and leaves floating-point weights unchanged. Earlier versions kept dense Q8_0 at INT8/group-32. Repopulate compiled-model caches without cache-only mode after this change; the cache fingerprint invalidates the old precision policy.
|
||||
|
||||
See the [implementation guide](dev_doc/OPENVINO-RECURRENT-CACHE.md) for buffer ownership and rollback.
|
||||
|
||||
# Compiled model cache
|
||||
|
||||
`GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR` exports compiled CPU/GPU graphs with their weights. It bypasses the plugin-level `GGML_OPENVINO_CACHE_DIR` and uses `OPTIMIZE_SPEED`, so weightless caching is disabled.
|
||||
@@ -7,7 +42,6 @@ One directory can hold blobs for different models and compilation settings. Run
|
||||
```sh
|
||||
GGML_OPENVINO_DEVICE=GPU \
|
||||
GGML_OPENVINO_NATIVE_SOFTPLUS=1 \
|
||||
GGML_OPENVINO_DISABLE_KV_SLICE=1 \
|
||||
GGML_OPENVINO_REQUANT_KQUANT=q4_asym64_all \
|
||||
GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/path/to/qwen-cache \
|
||||
./build/ReleaseOV/bin/llama-bench -m /path/to/model.gguf -r 1
|
||||
@@ -20,7 +54,6 @@ On later runs, supply the same compilation settings and enable `GGML_OPENVINO_CO
|
||||
```sh
|
||||
GGML_OPENVINO_DEVICE=GPU \
|
||||
GGML_OPENVINO_NATIVE_SOFTPLUS=1 \
|
||||
GGML_OPENVINO_DISABLE_KV_SLICE=1 \
|
||||
GGML_OPENVINO_REQUANT_KQUANT=q4_asym64_all \
|
||||
GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/path/to/qwen-cache \
|
||||
GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1 \
|
||||
@@ -29,10 +62,22 @@ GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1 \
|
||||
|
||||
Cache-only mode allocates backend address space without filling weight pages. On Windows, this also uses system commit capacity. The model-buffer size in the loader log is this virtual size. Weight uploads only record source identity; they do not read or requantize the weights. Graph conversion and compilation are skipped. Runtime buffers are still allocated and populated normally.
|
||||
|
||||
Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode.
|
||||
Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_EXACT_ZERO_POINT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK`, and `GGML_OPENVINO_DEBUG_NODE`, `GGML_OPENVINO_ENABLE_KV_4D`, and `GGML_OPENVINO_DISABLE_FUSED_RECURRENCE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode.
|
||||
|
||||
`GGML_OPENVINO_KV_SLICE` is a runtime binding option for dynamic graphs and can differ between cache population and cache-only runs. The same dynamic compiled graph accepts full-capacity and live-prefix attention K/V tensors. Slicing is enabled by default for contiguous ordinary K/V roots with one allocated stream, one active sequence at offset zero, and no sliding-window attention. Recurrent convolution/GDN caches and their rollback snapshots always retain their full stored row bank. Other layouts use full bindings. Repopulate caches made before this option was removed from the fingerprint once without cache-only mode.
|
||||
|
||||
A missing or incompatible graph fails with an error instead of compiling with absent weights. The fingerprint uses the dynamic graph's topology, ports, model parameters, weights, settings, and OpenVINO version; changing only dynamic token or KV sizes does not require a new entry. A different workload can still require another graph; populate it first without cache-only mode.
|
||||
|
||||
## MTP sampling and active slots
|
||||
|
||||
Qwen3.5/Qwen3.6 MTP uses a pure top-k backend sampler. Dynamic stateless execution retains its sampling branches for every configured slot and output row, including inactive branches. One packed `sampling_rows` input selects logits rows at runtime. Graph node names and candidate-output ports stay fixed, so changing the active subset of four slots reuses the same compiled draft model instead of creating up to 15 variants.
|
||||
|
||||
Active candidate IDs and scores are written to the normal GGML outputs. Inactive branches read row 0 (the padding row when there are no logits) and write to private scratch tensors (device USM on GPU), without changing sampler state or touching inactive GGML buffers. The top-k computations remain separate per branch; this trades some extra sampling work for a stable graph interface. Other sampler chains retain their existing branch-selection behavior.
|
||||
|
||||
This is one draft model per compatible graph configuration, not one compiled blob for the whole server. The trunk and other graph stages still have their own entries. Different sampler configurations, recurrent capacities, rollback modes or incompatible KV layouts can require additional models. Cache-only mode still requires initial preparation; changing just the active subset no longer requires exhaustive subset priming.
|
||||
|
||||
The generic recurrent-cache translation invalidates blobs made with the former single-slot shortcuts. Run once without `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY` to populate compatible entries before enabling cache-only mode again.
|
||||
|
||||
## Restrictions
|
||||
|
||||
- Cache-only mode requires Linux or Windows and mmap loading (`--load-mode mmap`, or the default when all selected devices support mmap). Do not use tensor validation or mlock when trying to avoid weight reads.
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# Stable multislot MTP graph
|
||||
|
||||
The captured UD draft graph contains 107 nodes regardless of which of four slots are active. It is the single MTP attention/MoE layer, not the recurrent trunk. Two speculative tokens require two consecutive draft calls, batched over active sequences.
|
||||
|
||||
```text
|
||||
MTP token IDs + input hidden state
|
||||
|
|
||||
0-84: MTP layer and normalization, N active sequences
|
||||
|
|
||||
85: logits [248320, N]
|
||||
86: PAD adds one safe row
|
||||
|
|
||||
+-- 87-91: slot 0 sampling
|
||||
+-- 92-96: slot 1 sampling
|
||||
+-- 97-101: slot 2 sampling
|
||||
+-- 102-106: slot 3 sampling
|
||||
```
|
||||
|
||||
Each branch contains VIEW, two Reshapes, TopK (ten candidate IDs), and GET_ROWS (their scores). Ten is the candidate count, not the speculative-token count.
|
||||
|
||||
| Active slots | Sampling COMPUTE flags, slots 0-3 | Raw/current nodes | Old filtered nodes | Packed sampling_rows |
|
||||
|---|---|---:|---:|---|
|
||||
| 3 | `[0,0,0,1]` | 107 | 92 | `[0,0,0,0]` |
|
||||
| 0,2 | `[1,0,1,0]` | 107 | 97 | `[0,0,1,0]` |
|
||||
| All four | `[1,1,1,1]` | 107 | 107 | `[0,1,2,3]` |
|
||||
|
||||
Old counts apply the old COMPUTE filter to the captures. Unselected nodes remain in the raw cgraph. Previously, removing them changed translated topology, slot-specific row Parameter names and output ports; even equal-sized subsets such as `{0,1}` and `{0,2}` differed. This could trigger up to 15 draft variants, with compilation/import on first use or a missing-cache error in cache-only mode.
|
||||
|
||||
Now all pure top-k branches remain. Their names include stable graph ordinals even when inactive. One int64 `sampling_rows` Parameter supplies row indices at runtime. For `{0,2}`, the two logits rows belong to slots 0 and 2:
|
||||
|
||||
```text
|
||||
sampling_rows = [0, 0, 1, 0]
|
||||
slot 0 -> row 0 -> normal candidate buffers
|
||||
slot 1 -> row 0 -> private scratch, ignored
|
||||
slot 2 -> row 1 -> normal candidate buffers
|
||||
slot 3 -> row 0 -> private scratch, ignored
|
||||
```
|
||||
|
||||
The vector is not an active mask. Host-side COMPUTE flags select normal versus scratch output destinations. PAD provides a valid row when there are no logits. Inactive outputs use device USM on GPU and do not affect sampler state or acceptance. Other sampler chains retain their selected-branch semantics.
|
||||
|
||||
Only small sampling work executes for inactive slots. The MTP layer and output projection still process N active rows. Top-k is separate per branch, not yet one batched TopK. The observed draft interface has 10 inputs and 12 outputs: eight candidate ID/score outputs, hidden state, full logits and two KV Results.
|
||||
|
||||
Dump shapes use GGML order `[width, rows, dim2, dim3]`; OV reverses it. `view_offs` is in bytes: one logits row occupies `248320 * 4 = 993280` bytes. Buffer labels describe GGML storage, not the device executing imported GPU operations.
|
||||
|
||||
The captures used cache-only mode, four slots, context 8192 and the default two-token MTP settings.
|
||||
@@ -0,0 +1,62 @@
|
||||
# Current Qwen GPU implementation
|
||||
|
||||
The default Qwen3.5/Qwen3.6 path is dynamic stateless execution with `RecurrentConvCacheUpdate`, `RecurrentGatedDeltaNetUpdate`, and stable pure-top-k MTP translation. It supports one or multiple slots and rollback. The implementation applies to `qwen35` and `qwen35moe`; the reference model is the UD Qwen3.6-35B-A3B MTP GGUF.
|
||||
|
||||
Build against the matching OpenVINO runtime/GPU plugin with single-output Update ops. Configure `GGML_OPENVINO_DEVICE=GPU`, native softplus, and the documented requantization policy. The cache translation revision is 15. Historical measurements retain their original source revisions.
|
||||
|
||||
The separate dynamic output-binding fix in OpenVINO PR #39354 remains required for cache-only MTP serving. Native cache-op export/import tests pass without it, but Qwen3.6 imported MTP graphs still trigger the missing-input-memory error without the fix.
|
||||
|
||||
## Buffer ownership and operation boundaries
|
||||
|
||||
| State | Storage and lifetime | Update path |
|
||||
|---|---|---|
|
||||
| Attention K/V | Full caller-owned USM allocation, sized by context and KV layout | Native stateless KV fusion when eligible; SDPA consumes the live prefix |
|
||||
| Convolution `cache_r` | Full caller-owned USM row bank, sized by slot/snapshot capacity | Fused reset, original-row gather/reorder, convolution and ordered history/snapshot writes |
|
||||
| GDN `cache_s` | Full caller-owned USM row bank, sized by slot/snapshot capacity | Fused reset, original-row gather/reorder, GDN and ordered final/snapshot writes |
|
||||
| Runtime controls | Reused USM host storage on GPU | Row maps, offsets and write controls remain runtime inputs |
|
||||
| Protected source rows | Private device scratch | Staged before writes to protect cycles, duplicate sources and overlaps |
|
||||
| Inactive sampling results | Private device-USM scratch | Discarded without touching inactive GGML output buffers |
|
||||
|
||||
Attention context capacity does not set recurrent-cache size. Each cache input Parameter binds a stable writable allocation; recurrent cache Results are omitted. There are no rotation banks or pointer swaps.
|
||||
|
||||
The GPU plugin uses two stages: first preserve required original rows in scratch while applying logical reset and precision rounding; then compute and perform ordered writes. This avoids overwriting a source before another destination has read it. Identity reorder sources need no scratch writes. In-place execution removes the spare full output bank, not every cache read or internal memory transaction.
|
||||
|
||||
`RecurrentConvCacheUpdate` has nine inputs, including depthwise weights, and returns convolution `[1,B,T,C]`. `RecurrentGatedDeltaNetUpdate` returns attention and writes the newest `min(T,snapshot_count)` states into its cache input, newest first. Neither op returns a cache tensor.
|
||||
|
||||
## Convolution cache stores
|
||||
|
||||
`build_conv_state_store` emits one CPY for the final history when rollback is disabled. With rollback enabled, it emits one SET_ROWS from overlapping `[history, window, channel, sequence]` views into strided snapshot planes. Window starts advance in time while destination planes descend, so plane zero holds the newest history. The fixed ARANGE/scale/cast chain supplies this descending slot order.
|
||||
|
||||
The backend recognizes this structure and refreshes source starts, cache heads, sequence counts, window counts and plane strides on every batch. Native fusion passes runtime `[source column, destination slot]` pairs to the existing convolution op; the pair-count dimension is dynamic. Functional translation gathers the same windows, scatters them into the full cache, and preserves unwritten slots and features. The native op spec is unchanged.
|
||||
|
||||
## Generic fallback and restrictions
|
||||
|
||||
Writable cache inputs must be GPU RemoteTensors with exclusive graph readers, optionally through Reshape aliases. The GPU plugin rejects shared cache readers and computed cache banks. Other inputs and the computation output must not overlap the cache. The frontend skips fusion for unmatched cache reads and retains functional cache writeback. Restore or clear state after a failed inference before retrying.
|
||||
|
||||
`GGML_OPENVINO_DISABLE_FUSED_RECURRENCE=1` retains ordinary cache decomposition. Naive graphs skip full-model recurrence fusion. Unmatched snapshot GDN uses ordinary GDN token prefixes to preserve state outputs. The fused path requires the matching native runtime; otherwise dynamic CPU and GPU execution uses ordinary GDN. Stateful and NPU MTP are outside this change.
|
||||
|
||||
## Attention KV layout
|
||||
|
||||
`GGML_OPENVINO_ENABLE_KV_4D=1` enables eligible native stateless in-place attention updates. The current eligibility covers one physical stream, including unified KV. Explicit split multi-slot KV retains the full-cache path and can be much slower even with only one active slot. Use `--kv-unified` for multi-slot OV GPU serving.
|
||||
|
||||
Leave `GGML_OPENVINO_KV_SLICE` at its default (`1`). Live-prefix slicing applies to eligible ordinary attention K/V roots, not recurrent caches. In-place KV fusion can bind the full allocation while exposing a live prefix internally. KV-slice on/off is a runtime binding choice and does not require different compiled blobs; incompatible graph layouts/capacities can still do so.
|
||||
|
||||
## Stable MTP sampling
|
||||
|
||||
The [graph walkthrough](MTP-SAMPLING-GRAPH.md) describes the fixed sampling interface. The backend keeps pure top-k branches for all configured slots, packs runtime row indices, and uses scratch for inactive outputs. It does not change the original cgraph or execute inactive stateful sampler chains. Only the small sampling branches stay at configured capacity; the expensive draft layer processes active rows.
|
||||
|
||||
One compatible draft graph is shared across active subsets. Trunk and genuinely different graph stages/configurations remain separate. Prepare a compatible revision-15 cache once, then use cache-only mode. Exhaustive active-subset priming is no longer required, though first-use model import and runtime preparation still have a cost.
|
||||
|
||||
## Validation
|
||||
|
||||
Native OV tests cover shape/attribute serialization, reset, cycles, duplicates, overlapping writes, mixed precision, changing shapes, writable cache input rebinding, exclusive Reshapes, shared readers, host/device USM and export/import. Convolution is checked against a host reference; GDN uses an independent cache mapping around ordinary GDN. See `docs/dev/qwen35-llamacpp-recurrent-cache.md` and `recurrent_cache_update.cpp` in the OV checkout for contracts and tests.
|
||||
|
||||
## Debugging and source map
|
||||
|
||||
- `ggml-decoder.cpp`: dynamic dimensions, control inputs, output names and `cgraph_ov.txt` dump.
|
||||
- `utils.cpp`: compute-branch filtering, compiled reuse and input/output binding.
|
||||
- `openvino/recurrent_cache_fusion.cpp`: structural pattern matching and native Update op translation.
|
||||
- `openvino/translate_session.cpp`: runtime sampling aliases and model construction.
|
||||
- OV `recurrent_cache_update` implementation: writable-input validation, source staging and GPU computation.
|
||||
|
||||
`GGML_OPENVINO_DUMP_CGRAPH=1` dumps the raw graph before filtering. It includes COMPUTE/OUTPUT flags and byte offsets and overwrites `cgraph_ov.txt` on each call. Enable dumps only for diagnosis; use the captured examples for a static view. GPU profiles and OpenCL copy traces add overhead, and lack of an explicit copy call does not prove zero traffic inside a kernel.
|
||||
@@ -0,0 +1,221 @@
|
||||
# OpenVINO backend environment variables for development
|
||||
|
||||
The variables for users, and the rules for values, are in [docs/backend/OPENVINO.md](../../../../docs/backend/OPENVINO.md#runtime-configuration). This file lists the other `GGML_OPENVINO_*` variables. They are for debugging, comparisons and experiments, and they can change or be removed.
|
||||
|
||||
## Debug output
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_DUMP_CGRAPH` | all | Boolean | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. |
|
||||
| `GGML_OPENVINO_DUMP_IR` | all | Boolean | `0` | Write the OpenVINO IR files, with timestamps. |
|
||||
| `GGML_OPENVINO_DEBUG_INPUT` | all | Boolean | `0` | Print the input tensors. |
|
||||
| `GGML_OPENVINO_DEBUG_OUTPUT` | all | Boolean | `0` | Print the output tensors. |
|
||||
| `GGML_OPENVINO_DEBUG_NODE` | all | String | `not set` | Add the named graph nodes as outputs of the compiled model. Separate several names with commas. |
|
||||
| `GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS` | all | Boolean | `0` | Print the tensor address map once. |
|
||||
| `GGML_OPENVINO_LOG_UNSUPPORTED_OPS` | all | Boolean | `0` | Log each op that the backend does not support, with the tensor details and the reason. |
|
||||
| `GGML_OPENVINO_DISABLE_CACHE` | all | Boolean | `0` | Turn the in-process cache of compiled models and decoders off. |
|
||||
| `GGML_OPENVINO_FULL_GRAPH_KEY` | all | Boolean | `0` | Add the external inputs of a graph to the key of the in-process cache. By default the key is the node count and the names of the first and last nodes. Use it when two different graphs could share that key. |
|
||||
| `GGML_OPENVINO_LOG_SWA_LAYERS` | all | Boolean | `0` | Log the attention extent of each layer and which layers use sliding-window attention, on an `ov-swa:` line at the WARN level. |
|
||||
| `GGML_OPENVINO_NPUW_DUMP_SUBS` | NPU | String | `not set` | Dump NPUW subgraphs: `YES` (all), `MIN` (unique function bodies), `last`, or indices such as `0,2`. |
|
||||
| `GGML_OPENVINO_NPUW_DUMP_SUBS_DIR` | NPU | String | `not set` | Directory for the NPUW subgraph dumps. |
|
||||
| `GGML_OPENVINO_NPUW_DUMP_SUBS_ON_FAIL` | NPU | String | `not set` | Passed to OpenVINO unchanged as `NPUW_DUMP_SUBS_ON_FAIL`. |
|
||||
| `GGML_OPENVINO_NPUW_DUMP_IO` | NPU | String | `not set` | Passed to OpenVINO unchanged as `NPUW_DUMP_IO`. |
|
||||
|
||||
```bash
|
||||
# Dump the GGML graph and the OpenVINO IR of a run
|
||||
GGML_OPENVINO_DUMP_CGRAPH=1 GGML_OPENVINO_DUMP_IR=1 \
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512
|
||||
|
||||
# Log unsupported ops. The lines have the WARN level, which the default log verbosity shows (-lv 2 or higher).
|
||||
GGML_OPENVINO_LOG_UNSUPPORTED_OPS=1 ./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512
|
||||
|
||||
# NPUW dumps. They need an OpenVINO developer build (NPU_PLUGIN_DEVELOPER_BUILD).
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_NPUW_DUMP_SUBS=YES
|
||||
export GGML_OPENVINO_NPUW_DUMP_SUBS_DIR=/tmp/npuw_dump # optional
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512
|
||||
```
|
||||
|
||||
## Switches for default optimizations
|
||||
|
||||
Each variable turns off an optimization that is on by default, or sets its strength. Use them to find the cause of a wrong result or of a speed change.
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_KV_SLICE` | all | Boolean | `1` | Bind the KV cache, and size the KQ masks, at the attended length instead of the full context. CPU and GPU: a view over the cache. NPU: the prefill graph is compiled at the attended length, rounded up to a power of two, and replaced when that size changes. With `0`, `llama-bench -d` fails on the NPU. |
|
||||
| `GGML_OPENVINO_NPU_BLOB_PREBUILD` | NPU | Boolean | `1` | Decode attends the smallest `1024 * 2^k` rows that cover the next position, with one decode graph for each window. All windows are built, or read from the cache, together with the first prefill and decode pair. `0`: one decode graph attends the whole context. Use the same value in the run that writes the cache and in the runs that read it. |
|
||||
| `GGML_OPENVINO_NPU_FAST_MASK` | NPU | Boolean | `1` | Write the prefill attention mask directly into the input tensor, without an intermediate copy. The result is identical. |
|
||||
| `GGML_OPENVINO_NPU_L0_HOST_TENSORS` | NPU | Boolean | `1` | Keep the KV cache and the compute buffers in Level Zero host memory, so the plugin reads them without an extra copy. Falls back to plain host memory if the allocation is not available. |
|
||||
| `GGML_OPENVINO_GDN_Q_SCALE` | NPU | Float | `64` | Factor of the GDN q pre-scaling, which keeps the GDN output above the FP16 flush-to-zero floor of the NPU. `0` or a negative value turns it off. Values below `64` are raised to `64`. |
|
||||
| `GGML_OPENVINO_MOE_OP` | GPU | Boolean | `1` | `0` keeps the unfused GatherMatmul path for mixture-of-experts layers. |
|
||||
| `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK` | CPU, GPU | Boolean | `0` | By default the model reads the sequence count and the attended KV length from the shape of the KQ mask, so the GPU plugin can select its fused attention kernel. `1` passes them as separate inputs. |
|
||||
| `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | CPU, GPU | Boolean | `0` | Stateful execution only. By default the sequence axis of the KV state is moved so that new tokens are appended in place. `1` keeps the original layout. |
|
||||
| `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS` | GPU | Boolean | `0` | By default an output tensor in a GPU buffer is bound directly to the infer request when its shape and type match. `1` turns this off. |
|
||||
| `GGML_OPENVINO_DISABLE_FUSED_RECURRENCE` | GPU | Boolean | `0` | Stateless execution only, with an OpenVINO build newer than 2026.4.1 (2026.4.1 does not build the fusion). By default the recurrent-state cache updates (Qwen3.5, LFM2) are fused into the OpenVINO model. `1` keeps the separate copies. |
|
||||
| `GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN` | GPU | Boolean | `0` | By default the operands of an element-wise op get the same rank, which works around a GPU plugin defect when the op is fused into RMS norm (gemma-4 in stateful execution gave wrong output). `1` turns the workaround off. |
|
||||
|
||||
```bash
|
||||
# Find out whether an optimization causes a wrong result: turn it off and compare the output
|
||||
GGML_OPENVINO_DEVICE=NPU GGML_OPENVINO_NPU_FAST_MASK=0 \
|
||||
./build/ReleaseOV/bin/llama-completion -m model.gguf -c 512 -n 32 --temp 0 -no-cnv -p "The capital of France is"
|
||||
```
|
||||
|
||||
## Experimental options
|
||||
|
||||
These options are off by default. They are not validated for general use: some help on one model or device and give a wrong or slower result on another.
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_NPU_DECODE_UNFOLD` | NPU | Boolean | `0` | Set `NPUW_UNFOLD_IREQS=YES` on the decode graph only. An explicit `GGML_OPENVINO_NPUW_UNFOLD_IREQS` or `GGML_OPENVINO_NPU_CONFIG` value takes precedence. |
|
||||
| `GGML_OPENVINO_NPU_PARALLEL_COMPILE` | NPU | Boolean | `0` | Compile the prefill and decode graphs in parallel. The load is shorter and the host memory peak during compilation is higher. |
|
||||
| `GGML_OPENVINO_NPU_KV_ON_DEMAND` | NPU | Boolean | `0` | Keep the ggml KV buffer non-resident and bind storage for each layer that grows with the rows in use, so a large `-c` costs memory only for the context that is reached. K is stored head-major and V transposed, as OpenVINO GenAI does. `llama-perplexity` fails with it. |
|
||||
| `GGML_OPENVINO_NPU_REQUANT_POLICY` | NPU | String | `group-128` | Layout of requantized weights: `group-128` (`Q4_0_128`) or `channel-wise` (`Q4_0_C`, lower accuracy). |
|
||||
| `GGML_OPENVINO_GDN_DECOMPOSE_CHUNK` | all | Integer | `16` | Tokens in one chunk when `GGML_OPENVINO_GDN_DECOMPOSE` is on. A power of 2 from 2 to 128; other values use `16`. |
|
||||
| `GGML_OPENVINO_GDN_NATIVE_STATE` | NPU | Boolean | `0` | Keep the `GATED_DELTA_NET` state in the layout of the OpenVINO op instead of the GGML layout. Every `GATED_DELTA_NET` must then run on the OpenVINO backend; the load aborts if one would run on the CPU backend. Saved states and `GGML_OPENVINO_DEBUG_NODE` dumps hold the OpenVINO layout. `test-backend-ops` fails with it. |
|
||||
| `GGML_OPENVINO_FORCE_STATIC` | CPU, GPU | Boolean | `0` | Run the static-shape path of the NPU on the CPU or the GPU, to separate translation bugs from NPU plugin or driver problems. Cannot be used with `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY`. |
|
||||
| `GGML_OPENVINO_ENABLE_FALLBACK` | all | Boolean | `0` | Detect graphs that the scheduler splits between OpenVINO and another backend, and translate them as partial graphs, without the compiled model cache. By default every graph is treated as a complete model graph. Cannot be used with `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY`. |
|
||||
| `GGML_OPENVINO_NATIVE_SOFTPLUS` | all | Boolean | `0` | Translate `SOFTPLUS` to the OpenVINO `SoftPlus` op instead of the default `relu(x) + log(1 + exp(-abs(x)))`. |
|
||||
| `GGML_OPENVINO_KV_SCATTER_ELEMENTS` | all | Boolean | `0` | Use `ScatterElementsUpdate` instead of `ScatterUpdate` for single-row KV writes in stateless execution. |
|
||||
| `GGML_OPENVINO_ENABLE_KV_4D` | GPU | Boolean | `0` | Bind the KV cache as `[1, ctx, n_heads_kv, head_size]`, so the GPU plugin can update and read it in place. Needs an OpenVINO build newer than 2026.4.1; do not use it with 2026.4.1. |
|
||||
| `GGML_OPENVINO_MEMORY_OPTIMIZE` | all | Boolean | `0` | Turn on `GGML_OPENVINO_REDUCE_COMPILE_MEM` and, on GPU, `GGML_OPENVINO_RELEASE_WEIGHTS`, unless they are set. |
|
||||
| `GGML_OPENVINO_REDUCE_COMPILE_MEM` | all | Boolean | value of `GGML_OPENVINO_MEMORY_OPTIMIZE` | Create fewer weight nodes during compilation, to lower the host memory in use. |
|
||||
| `GGML_OPENVINO_RELEASE_WEIGHTS` | GPU | Boolean | value of `GGML_OPENVINO_MEMORY_OPTIMIZE` | Release the host weight buffers when the compiled model holds its own copy. Needs stable graph shapes; do not use it for workloads that compile again. |
|
||||
| `GGML_OPENVINO_SPILL_DIR` | all | String | `not set` | Directory for a disk-backed weight buffer, so its pages can be reclaimed under memory pressure. It must be on real storage, not on tmpfs (`/tmp` on many systems). It is not part of the key of the compiled model cache and is ignored in cache-only mode. |
|
||||
| `GGML_OPENVINO_SELF_CONTAINED_BLOB` | NPU | Boolean | `0` | Requires `GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR`. Write the prefill and decode graphs as blobs with the weights inside; each blob takes about the size of the model on disk. Decode then attends the whole context. `GGML_OPENVINO_NPU_WEIGHTLESS_BANK` is the recommended cache layout for the NPU. |
|
||||
| `GGML_OPENVINO_NPU_SHARED_BANK` | NPU | Boolean | `0` | With `GGML_OPENVINO_SELF_CONTAINED_BLOB=1`: one weights bank for the prefill and decode blobs. With OpenVINO 2026.4.1 the blobs cannot be read back, so decode is compiled again. |
|
||||
| `GGML_OPENVINO_LAZY_HOST_THRESHOLD_MB` | NPU | Integer | `2048` | With `GGML_OPENVINO_SELF_CONTAINED_MMAP=1`: buffers of at least this size get demand-zero host memory instead of Level Zero host memory. |
|
||||
|
||||
```bash
|
||||
# NPU cache with the weights inside the blobs (experimental). Keep all other NPU settings the same in both runs.
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=/tmp/ov_cache
|
||||
export GGML_OPENVINO_SELF_CONTAINED_BLOB=1
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512 -lv 4 # first run: log shows "model cache WROTE"
|
||||
|
||||
export GGML_OPENVINO_SELF_CONTAINED_MMAP=1
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512 -lv 4 # later runs: log shows "model cache HIT"
|
||||
```
|
||||
|
||||
## Tips for Memory Optimization
|
||||
|
||||
With the default settings, the GPU and CPU plugins keep their own copy of the weights next to the backend's copy while a model compiles. On integrated GPUs both copies are in system RAM. The settings below lower that peak. They do not change the output.
|
||||
|
||||
```bash
|
||||
# GPU or CPU, large model: back the weight buffer with a file, so the kernel can drop its pages.
|
||||
# Use a directory on a local disk (not tmpfs) with free space of about the model size. Can lower prompt speed.
|
||||
export GGML_OPENVINO_SPILL_DIR=$HOME/ov_spill
|
||||
mkdir -p $GGML_OPENVINO_SPILL_DIR
|
||||
```
|
||||
|
||||
```bash
|
||||
# GPU or CPU, the same model started again and again: compile once, then load the compiled model only.
|
||||
export GGML_OPENVINO_DEVICE=GPU # or CPU
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=$HOME/ov_cache
|
||||
GGML_OPENVINO_SPILL_DIR=$HOME/ov_spill ./build/ReleaseOV/bin/llama-cli -m model.gguf # first run writes the cache
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf # later runs skip the weight upload
|
||||
```
|
||||
|
||||
```bash
|
||||
# GPU, one long-running model (llama-server): release the host copy of the weights after compilation.
|
||||
# One model per process; a graph that must compile again later stops the run.
|
||||
export GGML_OPENVINO_DEVICE=GPU
|
||||
export GGML_OPENVINO_MEMORY_OPTIMIZE=1
|
||||
```
|
||||
|
||||
```bash
|
||||
# CPU, MoE model: build fewer weight nodes during compilation.
|
||||
export GGML_OPENVINO_REDUCE_COMPILE_MEM=1
|
||||
```
|
||||
|
||||
```bash
|
||||
# NPU: weightless cache, then use the memory-mapped GGUF as the model buffer.
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR=$HOME/ov_cache
|
||||
export GGML_OPENVINO_NPU_WEIGHTLESS_BANK=1
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512 # first run writes the cache
|
||||
export GGML_OPENVINO_SELF_CONTAINED_MMAP=1
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512 # later runs
|
||||
```
|
||||
|
||||
- A cache hit without `GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1` shortens the load, but it uses about as much memory as a run without the cache.
|
||||
- MoE models are not written to the cache on CPU, so cache-only mode does not work for them on CPU.
|
||||
- `GGML_OPENVINO_SPILL_DIR` is ignored on Windows and in cache-only mode.
|
||||
- `-lm none` (no mmap) does not lower the memory in use, and it turns cache-only mode off.
|
||||
|
||||
## NPU plugin properties
|
||||
|
||||
The backend sets the properties below for the NPU. `GGML_OPENVINO_NPU_CONFIG` can change them and add others.
|
||||
|
||||
- **Backend default**: the value that the backend passes to OpenVINO when the user sets nothing.
|
||||
- **OpenVINO default**: the value that OpenVINO uses when nobody sets the property.
|
||||
- Values are those of OpenVINO, for example `YES` and `NO`.
|
||||
|
||||
| Property | Type | Backend default | OpenVINO default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `NPU_USE_NPUW` | Boolean | `YES` | `NO` | Use NPUW, the OpenVINO layer that splits a model into parts and compiles them for the NPU. |
|
||||
| `NPUW_DEVICES` | String | `NPU` | `NPU,CPU` | Devices that NPUW tries for each part, in order. With `NPU` only, no part runs on the CPU. |
|
||||
| `NPUW_FOLD` | Boolean | `YES` | `NO` | Compile the repeating blocks of the model once and reuse them. |
|
||||
| `NPUW_FUNCALL_FOR_ALL` | Boolean | `YES` | `NO` | Make every part of the model a function, not only the repeating blocks. |
|
||||
| `NPUW_FUNCALL_ASYNC` | Boolean | `YES` | `NO` | Run a function and the preparation of the next one as a pipeline. |
|
||||
| `NPUW_WEIGHTS_BANK` | String | `shared` | empty | Name of the weights bank. Graphs with the same name share one copy of the weights. |
|
||||
| `NPUW_DQ` | Boolean | `YES` | `NO` | Apply the dynamic quantization transformations in the plugin. |
|
||||
| `NPUW_DQ_FULL` | Boolean | `NO` | `YES` | Apply the full dynamic quantization pipeline in the plugin. |
|
||||
| `NPU_COMPILER_DYNAMIC_QUANTIZATION` | Boolean | `YES` | `NO` | Dynamic quantization in the NPU compiler. |
|
||||
| `NPU_COMPILER_TYPE` | String | `DRIVER` | `PREFER_PLUGIN` | The NPU compiler: `DRIVER` (in the NPU driver), `PLUGIN` (in OpenVINO) or `PREFER_PLUGIN` (`PLUGIN` when it is available, `DRIVER` if not). Models with `GATED_DELTA_NET` layers (Qwen3.5) compile with `PLUGIN` unless the user sets this property. |
|
||||
| `NPUW_CACHE_DIR` | String | value of `GGML_OPENVINO_CACHE_DIR` | empty | Directory for the compiled parts. |
|
||||
|
||||
Other properties that are useful with `GGML_OPENVINO_NPU_CONFIG`. The backend does not set them.
|
||||
|
||||
| Property | Type | Backend default | OpenVINO default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `COMPILATION_NUM_THREADS` | Integer | `not set` | number of CPU cores | Threads of the NPU compiler. A lower value lowers the host memory peak when a large model compiles. |
|
||||
| `NPUW_UNFOLD_IREQS` | Boolean | `not set` | `NO` | One infer request for each part, also for repeating blocks. |
|
||||
| `LOG_LEVEL` | String | `not set` | `LOG_ERROR` | Log level of the NPU plugin, for example `LOG_INFO`. |
|
||||
|
||||
```bash
|
||||
export GGML_OPENVINO_DEVICE=NPU
|
||||
export GGML_OPENVINO_NPU_CONFIG="NPU_COMPILER_TYPE=PLUGIN,COMPILATION_NUM_THREADS=1,LOG_LEVEL=LOG_INFO"
|
||||
```
|
||||
|
||||
```cpp
|
||||
// List the properties of the installed NPU plugin with their current values
|
||||
ov::Core core;
|
||||
for (const auto & name : core.get_property("NPU", ov::supported_properties)) {
|
||||
std::cout << name << " = " << core.get_property("NPU", name).as<std::string>() << "\n";
|
||||
}
|
||||
```
|
||||
|
||||
- All NPU properties: [OpenVINO NPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/npu-device.html).
|
||||
- `NPUW_*` options, with type and default: `src/plugins/intel_npu/src/al/include/intel_npu/config/npuw_option_defs.inc` in the OpenVINO source.
|
||||
- Graphs without a KV cache compile without NPUW: the backend removes `NPU_USE_NPUW` and the `NPUW_*` keys for them.
|
||||
|
||||
## GPU and CPU plugin properties
|
||||
|
||||
The backend sets no GPU or CPU property, except `CACHE_DIR` and `CACHE_MODE=OPTIMIZE_SIZE` when `GGML_OPENVINO_CACHE_DIR` is set.
|
||||
|
||||
```bash
|
||||
# CPU in FP16, to compare with the GPU and the NPU, which compute in FP16
|
||||
GGML_OPENVINO_DEVICE=CPU GGML_OPENVINO_CPU_CONFIG="INFERENCE_PRECISION_HINT=f16" \
|
||||
./build/ReleaseOV/bin/llama-cli -m model.gguf -c 512
|
||||
```
|
||||
|
||||
All properties: [OpenVINO GPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html), [OpenVINO CPU device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/cpu-device.html).
|
||||
|
||||
## Older variable names
|
||||
|
||||
Each of these variables sets one NPU property. They still work. `GGML_OPENVINO_NPU_CONFIG` sets the same properties and takes precedence when both are set.
|
||||
|
||||
| Variable | Device | Type | Default | Description |
|
||||
|---|---|---|---|---|
|
||||
| `GGML_OPENVINO_NPU_COMPILE_CONFIG` | NPU | String | `not set` | Sets `NPU_COMPILATION_MODE_PARAMS`. |
|
||||
| `GGML_OPENVINO_NPU_COMPILER_TYPE` | NPU | String | `not set` | Sets `NPU_COMPILER_TYPE`. |
|
||||
| `GGML_OPENVINO_NPUW_FUNCALL_FOR_ALL` | NPU | String | `not set` | Sets `NPUW_FUNCALL_FOR_ALL`. |
|
||||
| `GGML_OPENVINO_NPUW_UNFOLD_IREQS` | NPU | String | `not set` | Sets `NPUW_UNFOLD_IREQS`. |
|
||||
| `GGML_OPENVINO_COMPILATION_NUM_THREADS` | NPU | String | `not set` | Sets `COMPILATION_NUM_THREADS`. |
|
||||
|
||||
```bash
|
||||
# These two lines do the same. Use the first form.
|
||||
export GGML_OPENVINO_NPU_CONFIG="NPUW_UNFOLD_IREQS=YES"
|
||||
export GGML_OPENVINO_NPUW_UNFOLD_IREQS=YES
|
||||
```
|
||||
@@ -0,0 +1,8 @@
|
||||
# OpenVINO Qwen development documentation
|
||||
|
||||
The default supported Qwen3.5/Qwen3.6 GPU path is **dynamic stateless execution with in-place fused recurrent ops and stable MTP sampling**. Unified KV is the recommended multi-slot layout because it enables native in-place attention KV updates. No new inference switch is needed for the recurrent/sampling default.
|
||||
|
||||
- [Backend setup and cache usage](../README.md): runtime requirements, environment settings and restrictions.
|
||||
- [Implementation](OPENVINO-RECURRENT-CACHE.md): buffer ownership, fusion, rollback, fallback and source map.
|
||||
- [Multislot MTP graph](MTP-SAMPLING-GRAPH.md): why active-slot variants occurred and how stable translation works.
|
||||
- [Environment variables for development](OpenVINO-ENV-VARS.md): debug output, switches for default optimizations, experimental options, plugin properties and older variable names.
|
||||
@@ -0,0 +1,113 @@
|
||||
#include "npu_config.h"
|
||||
|
||||
#include "ggml-impl.h"
|
||||
#include "ggml-openvino/ggml-decoder.h"
|
||||
#include "ggml-openvino/ggml-openvino-extra.h"
|
||||
|
||||
#include <cstring>
|
||||
#include <openvino/runtime/properties.hpp>
|
||||
|
||||
// The environment does not set the NPU compiler type. Set by ggml_openvino_npu_build_compile_config().
|
||||
static bool npu_compiler_type_is_default = false;
|
||||
|
||||
ov::AnyMap ggml_openvino_npu_build_compile_config(const char * cache_dir) {
|
||||
ov::AnyMap compile_config = {
|
||||
{"NPU_COMPILER_DYNAMIC_QUANTIZATION", "YES" },
|
||||
{"NPU_USE_NPUW", "YES" },
|
||||
{"NPUW_DEVICES", "NPU" },
|
||||
{"NPUW_FOLD", "YES" },
|
||||
{"NPUW_WEIGHTS_BANK", "shared"},
|
||||
{"NPUW_FUNCALL_FOR_ALL", "YES" },
|
||||
{"NPUW_FUNCALL_ASYNC", "YES" },
|
||||
{"NPUW_DQ", "YES" },
|
||||
{"NPUW_DQ_FULL", "NO" },
|
||||
};
|
||||
if (cache_dir && strlen(cache_dir) > 0) {
|
||||
// Keep the default cache mode: CACHE_MODE=OPTIMIZE_SIZE makes the NPU plugin compile the NPUW submodels with
|
||||
// weights separation, which fails for them.
|
||||
compile_config["NPUW_CACHE_DIR"] = cache_dir;
|
||||
}
|
||||
// Copy an optional string env var into the compile config only when it is set and non-empty.
|
||||
auto set_compile_option_from_env = [&](const char * env_var, const char * config_key) {
|
||||
const char * value = ggml_openvino_getenv_str(env_var);
|
||||
if (value && strlen(value) > 0) {
|
||||
compile_config[config_key] = value;
|
||||
}
|
||||
};
|
||||
// Legacy alias for GGML_OPENVINO_NPU_CONFIG=NPU_COMPILATION_MODE_PARAMS=...
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPU_COMPILE_CONFIG", "NPU_COMPILATION_MODE_PARAMS");
|
||||
// PLUGIN | DRIVER | PREFER_PLUGIN. The in-plugin compiler and the driver compiler
|
||||
// can differ substantially in generated code quality for the same op. On the Intel
|
||||
// NPU the driver compiler emits markedly faster prefill kernels (~1940 vs ~1540 t/s
|
||||
// pp1024 on phi-4-mini), matching OpenVINO GenAI, so default to DRIVER below.
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPU_COMPILER_TYPE", "NPU_COMPILER_TYPE");
|
||||
// NPUW_FUNCALL_FOR_ALL=YES hangs the NPU (device lost via TDR) for context
|
||||
// lengths >= ~786 on the 2026.3 in-plugin compiler. Allow turning it off.
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPUW_FUNCALL_FOR_ALL", "NPUW_FUNCALL_FOR_ALL");
|
||||
// Unfolds function calls into separate infer requests, trading memory for the
|
||||
// per-call dispatch overhead that repeated funcalls otherwise pay.
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPUW_UNFOLD_IREQS", "NPUW_UNFOLD_IREQS");
|
||||
// The compiler runs one llvm worker per core by default; each carries its own
|
||||
// working set, so large graphs can exhaust host RAM. Capping the workers trades
|
||||
// compile time for peak memory.
|
||||
set_compile_option_from_env("GGML_OPENVINO_COMPILATION_NUM_THREADS", "COMPILATION_NUM_THREADS");
|
||||
// NPUW profiling dumps; need an OpenVINO build with NPU_PLUGIN_DEVELOPER_BUILD.
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPUW_DUMP_SUBS", "NPUW_DUMP_SUBS");
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPUW_DUMP_SUBS_DIR", "NPUW_DUMP_SUBS_DIR");
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPUW_DUMP_SUBS_ON_FAIL", "NPUW_DUMP_SUBS_ON_FAIL");
|
||||
set_compile_option_from_env("GGML_OPENVINO_NPUW_DUMP_IO", "NPUW_DUMP_IO");
|
||||
// Comma-separated KEY=VALUE pairs appended last, so they override anything above.
|
||||
// Escape hatch for bisecting plugin options without a rebuild.
|
||||
ggml_openvino_apply_config_overrides(compile_config, ggml_openvino_getenv_str("GGML_OPENVINO_NPU_CONFIG"), "NPU");
|
||||
npu_compiler_type_is_default = compile_config.find("NPU_COMPILER_TYPE") == compile_config.end();
|
||||
if (npu_compiler_type_is_default) {
|
||||
compile_config["NPU_COMPILER_TYPE"] = "DRIVER";
|
||||
}
|
||||
return compile_config;
|
||||
}
|
||||
|
||||
bool ggml_openvino_npu_gdn_uses_plugin_compiler() {
|
||||
return ggml_openvino_is_npu() && npu_compiler_type_is_default;
|
||||
}
|
||||
|
||||
ov::AnyMap ggml_openvino_npu_gdn_config(const ov::AnyMap & base) {
|
||||
ov::AnyMap gdn_config = base;
|
||||
if (ggml_openvino_npu_gdn_uses_plugin_compiler()) {
|
||||
gdn_config["NPU_COMPILER_TYPE"] = "PLUGIN";
|
||||
}
|
||||
return gdn_config;
|
||||
}
|
||||
|
||||
ov::AnyMap ggml_openvino_npu_without_npuw(const ov::AnyMap & config) {
|
||||
ov::AnyMap out;
|
||||
for (const auto & kv : config) {
|
||||
if (kv.first.rfind("NPUW", 0) == 0 || kv.first == "NPU_USE_NPUW") {
|
||||
continue;
|
||||
}
|
||||
out.insert(kv);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
ov::AnyMap ggml_openvino_npu_prefill_config(const ov::AnyMap & base, const GgmlOvDecoder & decoder) {
|
||||
ov::AnyMap prefill_config = base;
|
||||
if (ggml_openvino_npu_hfa_prefill_enabled()) {
|
||||
prefill_config["NPUW_ATTN_HISTORY_SIZE_INPUT"] = decoder.get_hfa_history_input_name();
|
||||
prefill_config["NPUW_ONLINE_PIPELINE"] = "REP";
|
||||
prefill_config["NPUW_ONLINE_ISOLATE"] = "ATTN";
|
||||
prefill_config["NPUW_ONLINE_KEEP_BLOCK_SIZE"] = "4";
|
||||
prefill_config["NPUW_UNFOLD_IREQS"] = "NO";
|
||||
prefill_config["NPUW_ATTN"] = "HFA";
|
||||
prefill_config["NPUW_ATTN_HFA_FUSED"] = "YES";
|
||||
}
|
||||
return prefill_config;
|
||||
}
|
||||
|
||||
ov::AnyMap ggml_openvino_npu_decode_config(const ov::AnyMap & base, const std::string & device) {
|
||||
ov::AnyMap decode_config = base;
|
||||
if (device == "NPU" && ggml_openvino_npu_decode_unfold_enabled() &&
|
||||
decode_config.find("NPUW_UNFOLD_IREQS") == decode_config.end()) {
|
||||
decode_config["NPUW_UNFOLD_IREQS"] = "YES";
|
||||
}
|
||||
return decode_config;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
#pragma once
|
||||
|
||||
// NPU (NPUW) compile configuration. Everything that decides which OpenVINO properties an NPU graph is
|
||||
// compiled with lives here, so "what does the NPU compile with" is answered in one place:
|
||||
// * the backend-wide base config (NPUW defaults, named env aliases, GGML_OPENVINO_NPU_CONFIG overrides)
|
||||
// * the per-graph variants of it (cacheless graphs, GATED_DELTA_NET graphs, static prefill, static decode)
|
||||
|
||||
#include <openvino/core/any.hpp>
|
||||
#include <string>
|
||||
|
||||
class GgmlOvDecoder;
|
||||
|
||||
// Base config for every NPU graph, built once at backend init. GGML_OPENVINO_NPU_CONFIG ("K=V,...") is
|
||||
// applied last, so it overrides the defaults and the named aliases.
|
||||
ov::AnyMap ggml_openvino_npu_build_compile_config(const char * cache_dir);
|
||||
|
||||
// Cacheless (encoder) graphs compile on the plain NPU plugin: drop every NPUW key.
|
||||
ov::AnyMap ggml_openvino_npu_without_npuw(const ov::AnyMap & config);
|
||||
|
||||
// Graphs with GATED_DELTA_NET (Qwen3.5). The driver compiler gives wrong output for them and the compiler bundled
|
||||
// with OpenVINO does not, so they compile with NPU_COMPILER_TYPE=PLUGIN unless the environment sets the compiler
|
||||
// type (GGML_OPENVINO_NPU_COMPILER_TYPE, or NPU_COMPILER_TYPE in GGML_OPENVINO_NPU_CONFIG).
|
||||
ov::AnyMap ggml_openvino_npu_gdn_config(const ov::AnyMap & base);
|
||||
|
||||
// True when ggml_openvino_npu_gdn_config() switches the compiler. Part of the model cache key.
|
||||
bool ggml_openvino_npu_gdn_uses_plugin_compiler();
|
||||
|
||||
// Static prefill graph. With GGML_OPENVINO_NPU_HFA_PREFILL, switches NPUW to the HFA attention pipeline.
|
||||
ov::AnyMap ggml_openvino_npu_prefill_config(const ov::AnyMap & base, const GgmlOvDecoder & decoder);
|
||||
|
||||
// Static decode graph. With GGML_OPENVINO_NPU_DECODE_UNFOLD, unfolds NPUW function calls into separate infer
|
||||
// requests, unless NPUW_UNFOLD_IREQS is already set.
|
||||
ov::AnyMap ggml_openvino_npu_decode_config(const ov::AnyMap & base, const std::string & device);
|
||||
@@ -0,0 +1,50 @@
|
||||
#include "npu_quant_policy.h"
|
||||
|
||||
#include "ggml-impl.h"
|
||||
#include "ggml.h"
|
||||
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
ExtraQuantType ggml_openvino_get_npu_requant_type() {
|
||||
const std::string policy = ggml_openvino_getenv_str("GGML_OPENVINO_NPU_REQUANT_POLICY", "group-128");
|
||||
if (policy == "group-128") {
|
||||
return ExtraQuantType::Q4_0_128;
|
||||
}
|
||||
if (policy == "channel-wise") {
|
||||
return ExtraQuantType::Q4_0_C;
|
||||
}
|
||||
GGML_ABORT("Unknown GGML_OPENVINO_NPU_REQUANT_POLICY: %s", policy.c_str());
|
||||
}
|
||||
|
||||
std::optional<ExtraQuantType> ggml_openvino_npu_token_embd_requant_type(const ggml_tensor * tensor) {
|
||||
// A Q6_K embedding table is stored as q8_0_c by default on the NPU. The int4 layouts only pay off for a tied
|
||||
// embedding, where the table is also the lm_head weight and GET_ROWS gathers the packed rows
|
||||
// before dequantizing them (see gather_compressed_rows).
|
||||
if (tensor->type != GGML_TYPE_Q6_K) {
|
||||
return std::nullopt;
|
||||
}
|
||||
const std::string quant = ggml_openvino_getenv_str("GGML_OPENVINO_NPU_TOKEN_EMBD_QUANT", "q8_0_c");
|
||||
ExtraQuantType type;
|
||||
if (quant == "f16") {
|
||||
type = ExtraQuantType::F16;
|
||||
} else if (quant == "q8_0_c") {
|
||||
type = ExtraQuantType::Q8_0_C;
|
||||
} else if (quant == "q4_0_128") {
|
||||
type = ExtraQuantType::Q4_0_128;
|
||||
} else if (quant == "q4_0_c") {
|
||||
type = ExtraQuantType::Q4_0_C;
|
||||
} else {
|
||||
GGML_ABORT("Unknown GGML_OPENVINO_NPU_TOKEN_EMBD_QUANT: %s", quant.c_str());
|
||||
}
|
||||
static std::once_flag logged;
|
||||
std::call_once(logged, [&] {
|
||||
GGML_LOG_INFO("ggml-openvino: NPU token_embd.weight requant type: %s\n", quant.c_str());
|
||||
});
|
||||
return type;
|
||||
}
|
||||
|
||||
std::optional<ExtraQuantType> ggml_openvino_npu_weight_requant_type(const ggml_tensor * tensor) {
|
||||
GGML_UNUSED(tensor);
|
||||
return ggml_openvino_get_npu_requant_type();
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
// NPU weight requantization rules. ggml_openvino_get_requant_type() (ggml-openvino-extra.cpp) delegates
|
||||
// the NPU-specific decisions here.
|
||||
|
||||
#include "ggml-openvino/ggml-openvino-extra.h"
|
||||
|
||||
#include <optional>
|
||||
|
||||
struct ggml_tensor;
|
||||
|
||||
// Layout selected by GGML_OPENVINO_NPU_REQUANT_POLICY (group-128 | channel-wise).
|
||||
ExtraQuantType ggml_openvino_get_npu_requant_type();
|
||||
|
||||
// token_embd.weight on NPU. nullopt: no NPU-specific rule, use the common default.
|
||||
std::optional<ExtraQuantType> ggml_openvino_npu_token_embd_requant_type(const ggml_tensor * tensor);
|
||||
|
||||
// Any other weight on NPU.
|
||||
std::optional<ExtraQuantType> ggml_openvino_npu_weight_requant_type(const ggml_tensor * tensor);
|
||||
File diff suppressed because it is too large
Load Diff
@@ -10,9 +10,11 @@
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <openvino/core/partial_shape.hpp>
|
||||
#include <openvino/runtime/tensor.hpp>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
struct ModelParams {
|
||||
@@ -33,11 +35,18 @@ struct ModelParams {
|
||||
bool has_rs_rollback = false;
|
||||
bool mixed_rope_params = false;
|
||||
bool is_cacheless_attn = false;
|
||||
// the graph writes a transposed V cache (`-fa off`: one value element per row, tokens along the row)
|
||||
bool v_trans = false;
|
||||
// the graph has FLASH_ATTN_EXT with logit soft-capping (gemma-2)
|
||||
bool softcap_attn = false;
|
||||
std::vector<int> swa_layers;
|
||||
// The sliding-window mask tensor, identified in compute_llm_params() by grouping attention
|
||||
// layers on the mask they consume. Only used to tell the two masks apart when naming OV
|
||||
// parameters -- both carry the same tensor name. Null when the graph has a single mask.
|
||||
const ggml_tensor * swa_mask = nullptr;
|
||||
// The attention masks that are graph inputs, by identity: gemma-4's KV-shared layers also read them through
|
||||
// GET_ROWS, which must resolve to the same OV parameter as the attention ops
|
||||
std::set<const ggml_tensor *> kq_masks;
|
||||
|
||||
std::vector<std::string> kv_names;
|
||||
size_t kv_buffer_ctx_id = 0;
|
||||
@@ -47,14 +56,15 @@ struct ModelParams {
|
||||
memcmp(rope_params, other.rope_params, sizeof(int32_t) * 16) == 0;
|
||||
}
|
||||
|
||||
// swa_layers: the masks and KV write indices of these layers are separate graph inputs
|
||||
bool can_reuse_dynamically(const ModelParams & other) const {
|
||||
return same_rope_params(other) && n_rs_slots == other.n_rs_slots &&
|
||||
has_rs_rollback == other.has_rs_rollback;
|
||||
return same_rope_params(other) && n_rs_slots == other.n_rs_slots && has_rs_rollback == other.has_rs_rollback &&
|
||||
swa_layers == other.swa_layers;
|
||||
}
|
||||
|
||||
bool can_reuse_statically(const ModelParams & other) const {
|
||||
return same_rope_params(other) && ctx == other.ctx && n_rs_slots == other.n_rs_slots &&
|
||||
has_rs_rollback == other.has_rs_rollback;
|
||||
has_rs_rollback == other.has_rs_rollback && swa_layers == other.swa_layers;
|
||||
}
|
||||
|
||||
bool kv_buffer_changed(const ModelParams & other) const { return kv_buffer_ctx_id != other.kv_buffer_ctx_id; }
|
||||
@@ -66,6 +76,9 @@ struct ComputeParams {
|
||||
int attention_size = -1;
|
||||
int attention_size_swa = -1;
|
||||
int attention_size_static = -1; // encoder/cross-attn KV fill level (whisper)
|
||||
// OV names of the KQ mask inputs that n_seq_active (mask->ne[3]) and attention_size(_swa) (mask->ne[0]) were read from
|
||||
std::string mask_name;
|
||||
std::string mask_swa_name;
|
||||
// Sliding window width, read back from the band of ggml's own SWA mask. ggml never passes
|
||||
// n_swa down to a backend, but fill_mask() bakes it into the mask contents, so the widest
|
||||
// unmasked row recovers it. Shorter than n_swa while the sequence is still short, which is
|
||||
@@ -106,17 +119,19 @@ struct ComputeParams {
|
||||
// [ 18432, 0, 1, 1] 0: GET_ROWS node_9
|
||||
// [ 18432, 0, 1, 1] 1: VIEW cache_r_l0 (view)
|
||||
|
||||
// Runtime writeback offsets are refreshed per batch. Convolution SET_ROWS also supplies overlapping window counts and snapshot plane strides.
|
||||
struct RsWriteback {
|
||||
int slot_begin = 0; // first cache slot written by the CPY
|
||||
int src_begin = -1; // first source column copied by a conv-state CPY
|
||||
int64_t slot_begin = 0; // first cache slot written
|
||||
int64_t src_begin = -1; // first source column of a conv-state window
|
||||
int64_t seq_count = -1;
|
||||
int64_t snapshot_stride = -1;
|
||||
int64_t window_count = -1;
|
||||
};
|
||||
|
||||
std::map<std::string, RsWriteback> rs_writebacks;
|
||||
// Destination slot offset of each state cache writeback CPY node, keyed by node name. It
|
||||
// changes with the batch (kv head, active sequence count) and, with rollback enabled
|
||||
// (cparams.n_rs_seq > 0), the conv state is written back once per snapshot slot. Passed to the
|
||||
// cached model as a runtime input. Dynamic models also receive the source-side offset; static
|
||||
// models use a fixed end-anchored offset in the translator.
|
||||
std::map<std::string, int64_t> sampling_rows;
|
||||
std::map<std::string, int64_t> gdn_attention_rows;
|
||||
std::map<std::string, std::vector<int64_t>> recurrent_controls;
|
||||
};
|
||||
|
||||
// defined below; declared here because GgmlOvDecoder uses it inline
|
||||
@@ -139,6 +154,8 @@ public:
|
||||
void * data_addr;
|
||||
};
|
||||
|
||||
const ggml_cgraph * get_cgraph() const { return m_cgraph; }
|
||||
|
||||
// Graph decoder
|
||||
GgmlOvDecoder(ggml_cgraph * cgraph,
|
||||
ModelParams & model_params,
|
||||
@@ -247,7 +264,13 @@ public:
|
||||
virtual void visit_subgraph(
|
||||
std::function<void(std::shared_ptr<GgmlDecoder>, int node_idx)> node_visitor) const override;
|
||||
|
||||
ggml_tensor * get_input_ggml_tensor(const std::string & name) const { return m_inputs.at(name); }
|
||||
ggml_tensor * get_input_ggml_tensor(const std::string & name) const {
|
||||
auto it = m_inputs.find(name);
|
||||
if (it == m_inputs.end()) {
|
||||
throw std::runtime_error("OpenVINO ggml input not found: " + name);
|
||||
}
|
||||
return it->second;
|
||||
}
|
||||
|
||||
virtual int get_op_case(int node_idx) const override { return m_node_info_list[node_idx].node_op_case; }
|
||||
|
||||
@@ -311,6 +334,8 @@ public:
|
||||
|
||||
virtual bool is_stateful() const override { return m_is_stateful; }
|
||||
|
||||
virtual bool is_prefill() const override { return m_is_prefill; }
|
||||
|
||||
int get_static_n_tokens() const { return m_is_prefill ? m_prefill_chunk_size : 1; }
|
||||
|
||||
virtual bool is_splited_model() const override { return m_model_is_splitted; }
|
||||
@@ -333,20 +358,90 @@ public:
|
||||
|
||||
const ggml_tensor * get_tensor_used_op(const ggml_tensor * tensor) const;
|
||||
|
||||
// True if the KV cache is a model input of shape [1, ctx, n_heads_kv, head_size] instead of
|
||||
// [1, 1, ctx, n_heads_kv * head_size]. Used for stateless GPU graphs when the cache holds one sequence, one token
|
||||
// per row, and is only written by SET_ROWS and read through views that split the rows into heads. The GPU
|
||||
// plugin can then fuse the cache write and read into StatelessKV: the new rows are written in place and attention
|
||||
// reads the cache without a copy.
|
||||
bool is_kvcache_4d(const ggml_tensor * cache) const;
|
||||
|
||||
// True if the KV cache is an input of this graph in the 4-D layout (see is_kvcache_4d)
|
||||
bool is_kvcache_4d_input(const ggml_tensor * cache) const { return m_kvcache_4d.count(cache) != 0; }
|
||||
|
||||
// True if each row of the KV cache holds one token. With `-fa off`, llama.cpp keeps the V cache transposed
|
||||
// and writes it one element per row, so its first rows do not hold the first tokens.
|
||||
bool is_kvcache_token_rows(const ggml_tensor * cache) const;
|
||||
|
||||
// Reuse the KV cache ov::Tensor while pointer, type and shape stay the same (a new USM tensor costs microseconds).
|
||||
template <typename Make>
|
||||
ov::Tensor cached_kv_tensor(const ggml_tensor * kv,
|
||||
const ov::element::Type & type,
|
||||
const ov::Shape & shape,
|
||||
Make && make) const {
|
||||
auto & entry = m_kv_tensor_cache[kv];
|
||||
if (!entry.tensor || entry.data != kv->data || entry.type != type || entry.shape != shape) {
|
||||
entry = {kv->data, type, shape, make(), next_tensor_id()};
|
||||
}
|
||||
m_last_kv_tensor_id = entry.id;
|
||||
return entry.tensor;
|
||||
}
|
||||
|
||||
// Id of the tensor returned by the last cached_kv_tensor() call, 0 if there was no call since the last read.
|
||||
uint64_t take_last_kv_tensor_id() const { return std::exchange(m_last_kv_tensor_id, 0); }
|
||||
|
||||
// Ids are never 0 and never reused: the same id means the same ov::Tensor.
|
||||
uint64_t next_tensor_id() const { return ++m_tensor_id; }
|
||||
|
||||
// What one infer request port holds between inferences (dynamic GPU path).
|
||||
struct ov_port_binding {
|
||||
ov::Tensor staging; // USM host tensor used in place of a small host tensor
|
||||
void * staging_ptr = nullptr;
|
||||
size_t staging_bytes = 0;
|
||||
uint64_t staging_id = 0;
|
||||
uint64_t bound_id = 0; // id of the tensor bound to the port, 0 if not known
|
||||
ov::Shape bound_shape;
|
||||
ov::element::Type bound_type;
|
||||
};
|
||||
|
||||
// Port state of `request`, reset when the decoder runs with another request.
|
||||
std::vector<ov_port_binding> & port_bindings(bool input, size_t n, const void * request) const {
|
||||
if (m_bindings_request != request) {
|
||||
m_input_bindings.clear();
|
||||
m_output_bindings.clear();
|
||||
m_bindings_request = request;
|
||||
}
|
||||
auto & v = input ? m_input_bindings : m_output_bindings;
|
||||
if (v.size() < n) {
|
||||
v.resize(n);
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
const ggml_tensor * get_tensor_from_name(const std::string & name) const;
|
||||
|
||||
std::string get_hfa_history_input_name() const;
|
||||
|
||||
void clear_model_weights() { m_model_weights.clear(); }
|
||||
|
||||
static std::pair<ModelParams, ComputeParams> compute_llm_params(ggml_cgraph * cgraph, bool is_static);
|
||||
|
||||
ModelParams get_model_params() const { return m_model_params; }
|
||||
const ModelParams & get_model_params() const { return m_model_params; }
|
||||
|
||||
ComputeParams get_compute_params() const { return m_compute_params; }
|
||||
const ComputeParams & get_compute_params() const { return m_compute_params; }
|
||||
|
||||
void set_model_params(const ModelParams & model_params) { m_model_params = model_params; }
|
||||
|
||||
void set_compute_params(const ComputeParams & compute_params) { m_compute_params = compute_params; }
|
||||
|
||||
// Per-request storage: scalar runtime controls are host-produced and shape inference
|
||||
// may read them. USM host allocations let the GPU consume them without tiny uploads.
|
||||
std::map<std::string, ov::Tensor> runtime_input_storage;
|
||||
std::map<std::string, ov::Tensor> runtime_input_tensors;
|
||||
std::map<std::string, ov::Tensor> runtime_output_scratch;
|
||||
|
||||
// OpenVINO device that runs the graph, e.g. GPU.1
|
||||
std::string device;
|
||||
|
||||
bool m_is_static = false;
|
||||
bool m_is_stateful = false;
|
||||
bool m_is_prefill = false;
|
||||
@@ -382,6 +477,15 @@ public:
|
||||
return tensor->op == GGML_OP_GET_ROWS && op->op == GGML_OP_RMS_NORM;
|
||||
}
|
||||
|
||||
static bool is_activation_input(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
if (tensor == nullptr || op == nullptr || tensor->type != GGML_TYPE_F32 || tensor->ne[2] != 1 ||
|
||||
tensor->ne[3] != 1 || (tensor->buffer != nullptr && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS)) {
|
||||
return false;
|
||||
}
|
||||
return (op->op == GGML_OP_RMS_NORM || op->op == GGML_OP_L2_NORM) && op->src[0] == tensor &&
|
||||
(tensor->op == GGML_OP_NONE || tensor->op == GGML_OP_CPY);
|
||||
}
|
||||
|
||||
static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]) ||
|
||||
(op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]);
|
||||
@@ -402,12 +506,78 @@ public:
|
||||
return op->op == GGML_OP_ROPE && tensor == op->src[2];
|
||||
}
|
||||
|
||||
inline static bool is_recurrent_cache(const ggml_tensor * tensor) {
|
||||
static bool is_recurrent_cache(const ggml_tensor * tensor) {
|
||||
return tensor != nullptr && (strncmp(tensor->name, "cache_r_l", strlen("cache_r_l")) == 0 ||
|
||||
strncmp(tensor->name, "cache_s_l", strlen("cache_s_l")) == 0 ||
|
||||
strncmp(tensor->name, "cache_ple_r_l", strlen("cache_ple_r_l")) == 0);
|
||||
}
|
||||
|
||||
static bool is_recurrent_cache_reset(const ggml_tensor * node) {
|
||||
const bool is_scale_or_fill = node->op == GGML_OP_SCALE || node->op == GGML_OP_FILL;
|
||||
return is_scale_or_fill && node->view_src != nullptr && is_recurrent_cache(node->view_src);
|
||||
}
|
||||
|
||||
static bool is_gdn_snapshot_writeback(const ggml_tensor * node) {
|
||||
if (!(node != nullptr && node->op == GGML_OP_CPY && node->src[0] != nullptr &&
|
||||
node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr &&
|
||||
node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET && node->src[0]->src[0]->op_params[0] > 1 &&
|
||||
node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->view_src != nullptr &&
|
||||
node->src[1]->view_src == node->view_src && node->src[0]->ne[0] == node->src[1]->ne[0] &&
|
||||
node->src[1]->nb[1] > 0 && node->src[1]->nb[1] == ggml_row_size(node->type, node->ne[0]) &&
|
||||
node->src[1]->nb[2] % node->src[1]->nb[1] == 0 &&
|
||||
node->src[1]->view_offs % node->src[1]->nb[1] == 0)) {
|
||||
return false;
|
||||
}
|
||||
const auto * source = node->src[0];
|
||||
const auto * destination = node->src[1];
|
||||
const auto * packed = source->src[0];
|
||||
if (node->type != GGML_TYPE_F32 || source->type != node->type || destination->type != node->type ||
|
||||
node->view_src->type != node->type || !ggml_is_contiguous(node->view_src) ||
|
||||
packed->nb[1] == 0 || source->view_offs % packed->nb[1] != 0 ||
|
||||
source->nb[0] != sizeof(float) || destination->nb[0] != sizeof(float) ||
|
||||
source->ne[3] != 1 || destination->ne[3] != 1) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < GGML_MAX_DIMS; ++i) {
|
||||
if (source->ne[i] <= 0 || source->ne[i] != destination->ne[i]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
const size_t plane_stride = destination->nb[2] / destination->nb[1];
|
||||
if (plane_stride == 0 || size_t(destination->ne[1]) > plane_stride -
|
||||
(destination->view_offs / destination->nb[1]) % plane_stride) {
|
||||
return false;
|
||||
}
|
||||
for (const auto * view : {source, destination}) {
|
||||
size_t remaining = ggml_nbytes(view == source ? packed : node->view_src);
|
||||
if (view->view_offs > remaining || remaining - view->view_offs < sizeof(float)) {
|
||||
return false;
|
||||
}
|
||||
remaining -= view->view_offs + sizeof(float);
|
||||
for (int i = 0; i < GGML_MAX_DIMS; ++i) {
|
||||
const size_t count = view->ne[i] - 1;
|
||||
if (view->nb[i] != 0 && count > remaining / view->nb[i]) {
|
||||
return false;
|
||||
}
|
||||
remaining -= count * view->nb[i];
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool is_sampling_view(const ggml_tensor * node) {
|
||||
return node != nullptr && node->op == GGML_OP_VIEW && node->src[0] != nullptr &&
|
||||
node->src[0]->op == GGML_OP_PAD && node->ne[0] == node->src[0]->ne[0] &&
|
||||
node->ne[1] == 1 && node->ne[2] == 1 && node->ne[3] == 1;
|
||||
}
|
||||
|
||||
static bool is_sampling_top_k(const ggml_tensor * node) {
|
||||
return node != nullptr && node->op == GGML_OP_TOP_K && node->src[0] != nullptr &&
|
||||
node->src[0]->op == GGML_OP_RESHAPE && is_sampling_view(node->src[0]->src[0]);
|
||||
}
|
||||
|
||||
bool is_direct_gdn_view(const ggml_tensor * node) const;
|
||||
|
||||
inline static bool is_cache(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
return is_recurrent_cache(tensor) || is_kvcache(tensor, op);
|
||||
}
|
||||
@@ -427,6 +597,25 @@ public:
|
||||
node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src;
|
||||
}
|
||||
|
||||
// build_conv_state_store writes overlapping [history, window, channel, seq] windows into newest-first rollback planes.
|
||||
// Match this layout separately from general SET_ROWS.
|
||||
static bool is_conv_state_snapshot_writeback(const ggml_tensor * node);
|
||||
|
||||
// SET_ROWS into a KV cache, directly or through the [1, n] reshape of the transposed V cache with `-fa off`.
|
||||
// Other SET_ROWS write rows of a compute tensor, e.g. the MoE expert-group mask
|
||||
inline static bool is_kvcache_write(const ggml_tensor * node) {
|
||||
const ggml_tensor * dst = node->src[2];
|
||||
return dst != nullptr && is_kvcache(dst->view_src != nullptr ? dst->view_src : dst, nullptr);
|
||||
}
|
||||
|
||||
// GET_ROWS batched over ggml dims 2 and 3, e.g. the MoE expert-group scores
|
||||
// ggml_get_rows(ggml_reshape_4d(groups, 1, n_exp_per_group, n_groups, n_tokens), top_groups)
|
||||
inline static bool is_get_rows_batched_4d(const ggml_tensor * node) {
|
||||
return node->op == GGML_OP_GET_ROWS && !ggml_is_quantized(node->src[0]->type) &&
|
||||
node->src[1]->ne[1] == node->src[0]->ne[2] && node->src[1]->ne[2] == node->src[0]->ne[3] &&
|
||||
node->src[1]->ne[3] == 1;
|
||||
}
|
||||
|
||||
static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
return op->op == GGML_OP_SET_ROWS && op->src[1] == tensor;
|
||||
}
|
||||
@@ -435,11 +624,27 @@ public:
|
||||
return m_model_params.swa_mask != nullptr && tensor == m_model_params.swa_mask;
|
||||
}
|
||||
|
||||
static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
bool is_kq_mask(const ggml_tensor * tensor) const { return m_model_params.kq_masks.count(tensor) != 0; }
|
||||
|
||||
// A graph-input index list of GET_ROWS: out_ids, or the inp->s_copy leaf when build_rs gathers
|
||||
// every recurrent state at once.
|
||||
static bool is_get_rows_input_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE &&
|
||||
op->src[1]->op == GGML_OP_NONE;
|
||||
}
|
||||
|
||||
// out_ids, which selects the output rows. The recurrent state gather reads a state cache instead.
|
||||
static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
if (!is_get_rows_input_idx(tensor, op)) {
|
||||
return false;
|
||||
}
|
||||
const ggml_tensor * data = op->src[0];
|
||||
while (data != nullptr && (data->op == GGML_OP_VIEW || data->op == GGML_OP_RESHAPE)) {
|
||||
data = data->src[0];
|
||||
}
|
||||
return !is_recurrent_cache(data);
|
||||
}
|
||||
|
||||
// the state permutation index input used in SSM/DeltaNet models (inp->s_copy in llama-graph.cpp)
|
||||
static bool is_inp_s_copy(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] &&
|
||||
@@ -476,6 +681,7 @@ public:
|
||||
private:
|
||||
void set_input_output();
|
||||
int compute_op_case(const ggml_tensor * node) const;
|
||||
bool is_gdn_out_prescaled(const ggml_tensor * gdn) const;
|
||||
bool node_is_used_as_src(const int node_idx);
|
||||
void compute_model_inputs();
|
||||
void compute_model_outputs();
|
||||
@@ -499,6 +705,22 @@ private:
|
||||
std::set<std::string> m_model_output_names;
|
||||
std::vector<NodeInfo> m_node_info_list;
|
||||
std::map<ggml_tensor *, int> m_node_dynamic_dims;
|
||||
std::set<const ggml_tensor *> m_kvcache_4d;
|
||||
|
||||
struct kv_tensor_entry {
|
||||
const void * data = nullptr;
|
||||
ov::element::Type type;
|
||||
ov::Shape shape;
|
||||
ov::Tensor tensor;
|
||||
uint64_t id = 0;
|
||||
};
|
||||
|
||||
mutable std::map<const ggml_tensor *, kv_tensor_entry> m_kv_tensor_cache;
|
||||
mutable uint64_t m_tensor_id = 0;
|
||||
mutable uint64_t m_last_kv_tensor_id = 0;
|
||||
mutable std::vector<ov_port_binding> m_input_bindings;
|
||||
mutable std::vector<ov_port_binding> m_output_bindings;
|
||||
mutable const void * m_bindings_request = nullptr;
|
||||
|
||||
ModelParams m_model_params;
|
||||
ComputeParams m_compute_params;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7,6 +7,7 @@
|
||||
#include <CL/cl.h>
|
||||
|
||||
#include <cstdlib>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <openvino/core/node.hpp>
|
||||
#include <openvino/runtime/remote_context.hpp>
|
||||
@@ -22,14 +23,14 @@ enum class ExtraQuantType { F16, Q4_0_C, Q8_1_C, Q4_0_128, Q4_0_64, Q8_0_C, Q8_0
|
||||
|
||||
ov::Core & ov_singleton_core();
|
||||
|
||||
// Get the remote context for the current device (returns empty optional for CPU)
|
||||
std::optional<ov::RemoteContext> ggml_openvino_get_remote_context();
|
||||
// Get the remote context for the device (returns empty optional for CPU)
|
||||
std::optional<ov::RemoteContext> ggml_openvino_get_remote_context(const std::string & device);
|
||||
|
||||
// Get the compile config for the current device
|
||||
const ov::AnyMap & ggml_openvino_get_compile_config();
|
||||
|
||||
// Get the OpenCL command queue for GPU operations (returns nullptr for CPU/NPU)
|
||||
cl_command_queue ggml_openvino_get_cl_queue();
|
||||
// Get the OpenCL command queue of the device for GPU operations (returns nullptr for CPU/NPU)
|
||||
cl_command_queue ggml_openvino_get_cl_queue(const std::string & device);
|
||||
|
||||
// Intel USM extension function type
|
||||
typedef cl_int(CL_API_CALL * clEnqueueMemFillINTEL_fn)(cl_command_queue queue,
|
||||
@@ -61,16 +62,21 @@ clEnqueueMemcpyINTEL_fn ggml_openvino_get_clEnqueueMemcpyINTEL();
|
||||
// =====================================================
|
||||
// Initialized once during backend init from GGML_OPENVINO_DEVICE env var
|
||||
|
||||
struct ggml_openvino_gpu_context {
|
||||
ov::RemoteContext remote_context;
|
||||
cl_command_queue cl_queue = nullptr;
|
||||
size_t max_alloc_size = SIZE_MAX;
|
||||
};
|
||||
|
||||
struct ggml_openvino_device_config {
|
||||
std::string device_name = "CPU";
|
||||
std::vector<std::string> available_devices;
|
||||
std::vector<std::string> selected_devices;
|
||||
bool is_npu = false;
|
||||
bool initialized = false;
|
||||
std::optional<ov::RemoteContext> remote_context;
|
||||
size_t max_alloc_size = SIZE_MAX;
|
||||
std::map<std::string, ggml_openvino_gpu_context> gpu_contexts;
|
||||
ov::AnyMap compile_config;
|
||||
std::unordered_map<std::string, std::string> environment_variables;
|
||||
cl_command_queue cl_queue = nullptr;
|
||||
clEnqueueMemFillINTEL_fn cl_mem_fill_fn = nullptr;
|
||||
clEnqueueMemcpyINTEL_fn cl_mem_cpy_fn = nullptr;
|
||||
|
||||
@@ -88,7 +94,10 @@ void ggml_openvino_init_device_config();
|
||||
const std::string & ggml_openvino_get_device_name();
|
||||
|
||||
// Get all available physical OpenVINO devices
|
||||
std::vector<std::string> ggml_openvino_get_available_devices();
|
||||
const std::vector<std::string> & ggml_openvino_get_available_devices();
|
||||
|
||||
// Check if the device is in GGML_OPENVINO_DEVICE
|
||||
bool ggml_openvino_is_selected_device(const std::string & device);
|
||||
|
||||
// Human-readable device name, e.g. "Intel(R) AI Boost (NPU 4000)"; the device id if unavailable
|
||||
std::string ggml_openvino_get_device_description(const std::string & device_name);
|
||||
@@ -96,19 +105,27 @@ std::string ggml_openvino_get_device_description(const std::string & device_name
|
||||
// Environment variable accessors. All GGML_OPENVINO_* env vars are read once
|
||||
// during backend init and cached on the device config; consumers must go
|
||||
// through these helpers (never call ::getenv directly) so behavior stays
|
||||
// consistent and centralized.
|
||||
// consistent and centralized. A variable must be in the table in
|
||||
// ggml-openvino-extra.cpp, which gives its type: backend init stops on a value
|
||||
// that does not fit the type (e.g. "1x" for a number, "on" for a toggle).
|
||||
//
|
||||
// Use ggml_openvino_getenv_str() for string / path values
|
||||
// (e.g. GGML_OPENVINO_DEVICE, GGML_OPENVINO_CACHE_DIR). The optional
|
||||
// default_value is returned when the var is unset or empty.
|
||||
//
|
||||
// Use ggml_openvino_getenv_int() for boolean toggles and integer settings.
|
||||
// It returns std::atoi(value) when set, otherwise default_value. For
|
||||
// boolean use, `if (ggml_openvino_getenv_int(name))` is true iff the value
|
||||
// is a non-zero integer (so "0" disables, "1" enables).
|
||||
// A toggle is cached as 1 (a positive value) or 0 (zero or a negative value),
|
||||
// so `if (ggml_openvino_getenv_int(name))` is true iff it is on.
|
||||
const char * ggml_openvino_getenv_str(const char * var, const char * default_value = nullptr);
|
||||
// Returns default_value when the variable is unset or empty.
|
||||
int ggml_openvino_getenv_int(const char * var, int default_value = 0);
|
||||
|
||||
// Apply plugin properties given as "KEY=VALUE,KEY=VALUE" (GGML_OPENVINO_NPU_CONFIG, GGML_OPENVINO_GPU_CONFIG,
|
||||
// GGML_OPENVINO_CPU_CONFIG) to a compile config. They override what the config already holds.
|
||||
void ggml_openvino_apply_config_overrides(ov::AnyMap & config, const char * spec, const char * device);
|
||||
// The plugin properties the environment sets for the selected device, as "GGML_OPENVINO_GPU_CONFIG=..."; empty if none.
|
||||
std::string ggml_openvino_config_overrides_in_use();
|
||||
|
||||
// Memory optimization toggles. GGML_OPENVINO_MEMORY_OPTIMIZE is an umbrella
|
||||
// switch; the fine-grained env vars still override it when explicitly set.
|
||||
bool ggml_openvino_reduce_compile_mem_enabled();
|
||||
@@ -117,11 +134,105 @@ bool ggml_openvino_release_weights_enabled();
|
||||
// Check if running on NPU
|
||||
bool ggml_openvino_is_npu();
|
||||
|
||||
// KV-cache slicing: bind the KV cache, and size the KQ masks, at the attended length instead of the full context.
|
||||
// One mechanism, two modes:
|
||||
// dynamic graphs (CPU/GPU): the KV input is a view over the cache; the graph is not recompiled.
|
||||
// static graphs (NPU, or any device with GGML_OPENVINO_FORCE_STATIC): the prefill graph is compiled at the aligned
|
||||
// attended length and recompiled when it changes; decode keeps the full context.
|
||||
// On by default for all devices; GGML_OPENVINO_KV_SLICE=0 turns it off.
|
||||
bool ggml_openvino_kv_slice_enabled();
|
||||
|
||||
// On by default: back NPU buffers with Level Zero host memory; GGML_OPENVINO_NPU_L0_HOST_TENSORS=0 turns it off.
|
||||
bool ggml_openvino_npu_l0_host_tensors_enabled();
|
||||
|
||||
// On by default: build the static prefill mask directly in the input tensor; GGML_OPENVINO_NPU_FAST_MASK=0 turns it off.
|
||||
bool ggml_openvino_npu_fast_mask_enabled();
|
||||
|
||||
// NPU-only opt-in performance toggles (all default off). Centralized here so each
|
||||
// env-var name lives in one place; see docs/backend/OPENVINO.md for semantics.
|
||||
bool ggml_openvino_npu_hfa_prefill_enabled();
|
||||
bool ggml_openvino_npu_decode_unfold_enabled();
|
||||
bool ggml_openvino_npu_parallel_compile_enabled();
|
||||
|
||||
// GGML_OPENVINO_NPU_BLOB_PREBUILD (default 1): static NPU graphs attend a KV window of 1024*2^k rows (clamped to ctx)
|
||||
// instead of the whole ctx, with one decode model per window, all built/imported with the prefill/decode pair;
|
||||
// with head-major KV the prefill models of every window are also exported and preloaded.
|
||||
// Returns the window base (1024), 0 = off (GGML_OPENVINO_NPU_BLOB_PREBUILD=0).
|
||||
int ggml_openvino_npu_decode_bucket();
|
||||
|
||||
// Rows a static prefill attends with GGML_OPENVINO_KV_SLICE: the attended length rounded up to a power of two,
|
||||
// clamped to ctx.
|
||||
int ggml_openvino_static_kv_window(int rows, int ctx);
|
||||
|
||||
// On-demand NPU KV cache (GGML_OPENVINO_NPU_KV_ON_DEMAND): the ggml KV buffer stays non-resident and the
|
||||
// NPU binds per-tensor L0 storage that grows (base*2^k rows, base = decode bucket or 1024) with the rows in use.
|
||||
bool ggml_openvino_npu_kv_on_demand_enabled();
|
||||
// On-demand KV with BLOB_PREBUILD and KV_SLICE: K storage is [heads][rows][head_size] and V storage is
|
||||
// [heads][head_size][rows] (head-major with heads * head_size channels); every static graph binds the whole storage
|
||||
// as [1, heads, rows, head_size] / [1, heads, head_size, rows], and attention uses V^T.
|
||||
bool ggml_openvino_npu_kv_head_major_enabled();
|
||||
|
||||
// NPU blob cache modes that keep the raw GGUF weights and skip requantization on a warm hit:
|
||||
// GGML_OPENVINO_SELF_CONTAINED_BLOB (weights embedded in each blob) or
|
||||
// GGML_OPENVINO_NPU_WEIGHTLESS_BANK (weightless blobs plus one shared weights file).
|
||||
bool ggml_openvino_npu_raw_weight_cache_enabled();
|
||||
bool ggml_openvino_npu_weightless_bank_enabled();
|
||||
void ggml_openvino_kv_storage_register(void * data, size_t row_bytes, size_t rows);
|
||||
void ggml_openvino_kv_storage_unregister(const void * base, size_t size);
|
||||
// Storage address backing ggml address `ptr` with at least `bytes` valid (grown if needed);
|
||||
// nullptr if `ptr` is not inside an on-demand KV tensor.
|
||||
void * ggml_openvino_kv_storage(const void * ptr, size_t bytes);
|
||||
// Grows the storage of the on-demand KV tensor at `data` to at least `rows` rows (head-major when heads > 0);
|
||||
// true if existing storage moved.
|
||||
bool ggml_openvino_kv_storage_reserve(const void * data, size_t rows, size_t heads = 0);
|
||||
// Rows currently backed by storage for the on-demand KV tensor at `data` (0 if none).
|
||||
size_t ggml_openvino_kv_storage_capacity(const void * data);
|
||||
enum ggml_openvino_kv_copy_op { GGML_OPENVINO_KV_READ, GGML_OPENVINO_KV_WRITE, GGML_OPENVINO_KV_FILL };
|
||||
// Host access by ggml address in the ggml row layout (FILL uses *host as the byte value); false if not on-demand KV.
|
||||
bool ggml_openvino_kv_storage_copy(const void * ptr, void * host, size_t size, ggml_openvino_kv_copy_op op);
|
||||
// Writes rows [dst_row, dst_row + n) of head-major storage of the KV tensor at `data` from `src`, laid out as
|
||||
// [heads][src_stride] entries of (row_bytes / heads) bytes, starting at entry src_col of every head.
|
||||
// False if `data` is not head-major on-demand KV.
|
||||
bool ggml_openvino_kv_storage_write_rows(const void * data, const void * src, size_t src_stride, size_t src_col,
|
||||
size_t dst_row, size_t n);
|
||||
void ggml_openvino_kv_storage_clear(const void * base, size_t size, uint8_t value);
|
||||
|
||||
// Check if running on a GPU (GPU, GPU.0, GPU.1, ...)
|
||||
bool ggml_openvino_is_gpu();
|
||||
|
||||
// Read n_seq_active and attention_size(_swa) from the KQ mask shape (off with GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK=1)
|
||||
bool ggml_openvino_shape_from_mask_enabled();
|
||||
|
||||
// GDN q pre-scale for eligible graphs on the NPU static path. On by default;
|
||||
// GGML_OPENVINO_GDN_Q_SCALE=0 (or a negative or non-numeric value) turns it off.
|
||||
bool ggml_openvino_gdn_prescale_enabled();
|
||||
|
||||
// GDN q pre-scale factor (NPU static path, keeps the tiny GDN output above the
|
||||
// f16 subnormal range). Defaults to 64.0f and is clamped to a minimum of
|
||||
// 64.0f. Only consulted when ggml_openvino_gdn_prescale_enabled() is true.
|
||||
float ggml_openvino_gdn_q_scale();
|
||||
|
||||
// Keep the GDN state in the OV op layout [B, H_v, key_dim, value_dim], skipping two transposes per layer.
|
||||
// Only the OpenVINO backend may then read the state. NPU only, enabled when GGML_OPENVINO_GDN_NATIVE_STATE is a positive integer.
|
||||
bool ggml_openvino_gdn_native_state_enabled();
|
||||
|
||||
// True when the device is an NPU with DEVICE_ARCHITECTURE 4000 (Lunar Lake).
|
||||
bool ggml_openvino_npu_is_4000();
|
||||
|
||||
// Translate GATED_DELTA_NET into a MatMul decomposition instead of the GatedDeltaNet op: chunked for multi-token
|
||||
// graphs (prefill), one step for single-token graphs (decode).
|
||||
// The NPU4000 GDN prefill kernel drives the DPU from SHAVE, which the LNL firmware does not support. On Linux the
|
||||
// decode kernel loses the device too (ZE_RESULT_ERROR_DEVICE_LOST).
|
||||
// Default: on for NPU4000 only. GGML_OPENVINO_GDN_DECOMPOSE=0 turns it off, a positive integer forces it on.
|
||||
// Snapshot (MTP) graphs still use the op.
|
||||
bool ggml_openvino_gdn_decompose_enabled();
|
||||
|
||||
// Tokens per chunk of the decomposed GATED_DELTA_NET: GGML_OPENVINO_GDN_DECOMPOSE_CHUNK,
|
||||
// a power of 2 in [2, 128], default 16.
|
||||
int ggml_openvino_gdn_decompose_chunk();
|
||||
|
||||
// Largest single memory object the device can allocate, SIZE_MAX when there is no known limit
|
||||
size_t ggml_openvino_max_alloc_size();
|
||||
size_t ggml_openvino_max_alloc_size(const std::string & device);
|
||||
|
||||
// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS, GPU only).
|
||||
// register: record a host weight buffer (idempotent per data pointer).
|
||||
@@ -134,6 +245,11 @@ bool ggml_openvino_weight_buffers_released();
|
||||
// Get requantization type for a tensor type (returns nullopt if no requant needed)
|
||||
std::optional<ExtraQuantType> ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false);
|
||||
|
||||
// True if an asymmetric weight keeps an exact f16 zero point instead of a rounded integer one.
|
||||
// Always true for MoE expert weights (llama.cpp's *_exps / *_chexps tensors). Other weights need
|
||||
// GGML_OPENVINO_EXACT_ZERO_POINT.
|
||||
bool ggml_openvino_use_exact_zero_point(const ggml_tensor * tensor);
|
||||
|
||||
// =====================================================
|
||||
// OpenVINO Tensor Extra Types
|
||||
// =====================================================
|
||||
@@ -180,10 +296,12 @@ struct ggml_openvino_quantized_weight_extra : public ggml_openvino_extra_base {
|
||||
// Extra data for KV cache / compute tensors - stores ov::Tensor for infer_request
|
||||
struct ggml_openvino_tensor_extra : public ggml_openvino_extra_base {
|
||||
std::shared_ptr<ov::Tensor> tensor; // For direct use with infer_request
|
||||
const void * data; // tensor->data that `tensor` wraps
|
||||
|
||||
explicit ggml_openvino_tensor_extra(std::shared_ptr<ov::Tensor> t) :
|
||||
ggml_openvino_tensor_extra(std::shared_ptr<ov::Tensor> t, const void * data) :
|
||||
ggml_openvino_extra_base(Type::TENSOR),
|
||||
tensor(std::move(t)) {}
|
||||
tensor(std::move(t)),
|
||||
data(data) {}
|
||||
};
|
||||
|
||||
// =====================================================
|
||||
@@ -212,7 +330,9 @@ struct ggml_openvino_extracted_layout {
|
||||
// Calculate the buffer layout for extracted quantized data
|
||||
ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_tensor * tensor, bool use_bias = false);
|
||||
|
||||
ggml_openvino_tensor_extra * ggml_openvino_create_tensor_extra(const ggml_tensor * tensor, bool is_remote);
|
||||
ggml_openvino_tensor_extra * ggml_openvino_create_tensor_extra(const ggml_tensor * tensor,
|
||||
bool is_remote,
|
||||
const std::string & device);
|
||||
|
||||
// Check if a tensor's buffer uses remote (device) memory (e.g. GPU USM)
|
||||
bool ggml_openvino_buffer_is_remote(const ggml_tensor * tensor);
|
||||
|
||||
@@ -21,6 +21,9 @@
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <openvino/core/type/element_type.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/convert.hpp>
|
||||
#include <openvino/op/parameter.hpp>
|
||||
#include <openvino/openvino.hpp>
|
||||
#include <openvino/runtime/allocator.hpp>
|
||||
#include <openvino/runtime/intel_gpu/ocl/ocl.hpp>
|
||||
@@ -31,6 +34,7 @@
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
|
||||
#ifdef _WIN32
|
||||
@@ -39,6 +43,7 @@
|
||||
# define NOMINMAX
|
||||
# endif
|
||||
# include <windows.h>
|
||||
# include <psapi.h>
|
||||
#else
|
||||
# include <sys/mman.h>
|
||||
# include <unistd.h>
|
||||
@@ -73,10 +78,17 @@ struct ggml_backend_openvino_buffer_context {
|
||||
size_t size;
|
||||
bool is_remote;
|
||||
|
||||
// File-backed spill or cache-only virtual memory.
|
||||
// OS mapping backing the buffer: anonymous memory, or the file-backed spill.
|
||||
void * spill_mapping = nullptr;
|
||||
size_t spill_size = 0;
|
||||
|
||||
// Set for a model buffer that directly wraps the loader's GGUF mmap.
|
||||
bool external_memory = false;
|
||||
|
||||
// KV buffer whose tensors are backed by on-demand L0 storage (GGML_OPENVINO_NPU_KV_ON_DEMAND); its own
|
||||
// memory is never touched, so it stays demand-zero and non-resident.
|
||||
bool kv_on_demand = false;
|
||||
|
||||
// Wrapping of the buffer
|
||||
std::shared_ptr<ov::Tensor> ov_buffer;
|
||||
|
||||
@@ -86,9 +98,23 @@ struct ggml_backend_openvino_buffer_context {
|
||||
std::vector<ggml_openvino_source_mapping> source_mappings;
|
||||
|
||||
// Used for re-allocation on device for kvcache
|
||||
void * data_prev;
|
||||
void * data_prev = nullptr;
|
||||
|
||||
ggml_backend_openvino_buffer_context(int device, size_t size, bool is_remote = false) :
|
||||
ggml_backend_openvino_buffer_context(int device, void * external_data, size_t size) :
|
||||
device(device),
|
||||
name(std::string(GGML_OPENVINO_NAME) + std::to_string(device)),
|
||||
id([]() {
|
||||
static std::atomic<size_t> next_id{1};
|
||||
return next_id.fetch_add(1);
|
||||
}()),
|
||||
data(external_data),
|
||||
size(size),
|
||||
is_remote(false),
|
||||
external_memory(true) {
|
||||
ov_buffer = std::make_shared<ov::Tensor>(ov::element::u8, ov::Shape{size}, data);
|
||||
}
|
||||
|
||||
ggml_backend_openvino_buffer_context(int device, size_t size, bool is_remote = false, bool lazy = false) :
|
||||
device(device),
|
||||
name(std::string(GGML_OPENVINO_NAME) + std::to_string(device)),
|
||||
id([]() {
|
||||
@@ -102,36 +128,54 @@ struct ggml_backend_openvino_buffer_context {
|
||||
return;
|
||||
}
|
||||
|
||||
const auto & device_name = ggml_openvino_get_device_name();
|
||||
const auto & device_name = ggml_openvino_get_available_devices()[device];
|
||||
|
||||
if (is_remote) {
|
||||
GGML_ASSERT(ggml_openvino_is_gpu());
|
||||
auto remote_context = ggml_openvino_get_remote_context();
|
||||
auto remote_context = ggml_openvino_get_remote_context(device_name);
|
||||
auto gpu_context = remote_context->as<ov::intel_gpu::ocl::ClContext>();
|
||||
ov::intel_gpu::ocl::USMTensor usm_tensor =
|
||||
gpu_context.create_usm_device_tensor(ov::element::u8, ov::Shape{size});
|
||||
data = usm_tensor.get();
|
||||
ov_buffer = std::make_shared<ov::intel_gpu::ocl::USMTensor>(std::move(usm_tensor));
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
if (ggml_openvino_model_cache_only()) {
|
||||
data = spill_mapping = VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
|
||||
if (data == nullptr) {
|
||||
return;
|
||||
// In strict warm-mmap mode the model weights are supplied via buffer_from_host_ptr,
|
||||
// so every alloc_buffer here is graph scratch / KV. The large prefill compute buffer
|
||||
// is never touched on the host by the on-device fused OV model, yet an L0 host tensor
|
||||
// commits/zeroes it eagerly (fully resident). For such large buffers, reserve
|
||||
// demand-zero pages instead (no L0, no memset) so they stay non-resident. KV and small
|
||||
// I/O buffers keep the L0 host tensor for zero-copy import (decode throughput).
|
||||
const bool strict_mmap = device_name == "NPU" &&
|
||||
ggml_openvino_npu_raw_weight_cache_enabled() &&
|
||||
ggml_openvino_getenv_int("GGML_OPENVINO_SELF_CONTAINED_MMAP") &&
|
||||
ggml_openvino_getenv_str("GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR");
|
||||
// 2 GiB keeps a large-context KV cache (~1 GiB at ctx 8K) on L0 while the bigger prefill
|
||||
// compute buffer still goes lazy.
|
||||
const size_t lazy_host_threshold =
|
||||
(size_t) ggml_openvino_getenv_int("GGML_OPENVINO_LAZY_HOST_THRESHOLD_MB", 2048) * 1024 * 1024;
|
||||
const bool lazy_host_alloc = lazy || (strict_mmap && size >= lazy_host_threshold);
|
||||
// Prefer importable Level Zero host memory on NPU: the plugin imports it without an
|
||||
// extra host-to-device copy each infer.
|
||||
if (!lazy_host_alloc && ggml_openvino_is_npu() && ggml_openvino_npu_l0_host_tensors_enabled() &&
|
||||
!ggml_openvino_model_cache_only()) {
|
||||
try {
|
||||
auto npu_context = ov_singleton_core().get_default_context("NPU");
|
||||
auto host_tensor = npu_context.create_host_tensor(ov::element::u8, ov::Shape{size});
|
||||
data = host_tensor.data();
|
||||
memset(data, 0, size);
|
||||
ov_buffer = std::make_shared<ov::Tensor>(std::move(host_tensor));
|
||||
} catch (const std::exception & e) {
|
||||
GGML_LOG_WARN("%s: L0 host tensor allocation of %zu bytes failed (%s); using anonymous memory\n",
|
||||
__func__, size, e.what());
|
||||
data = nullptr;
|
||||
}
|
||||
spill_size = size;
|
||||
ov_buffer = std::make_shared<ov::Tensor>(ov::element::u8, ov::Shape{size}, data);
|
||||
}
|
||||
if (data != nullptr) {
|
||||
// backed by the Level Zero host tensor above
|
||||
} else
|
||||
#else
|
||||
if (ggml_openvino_model_cache_only()) {
|
||||
void * m = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
if (m == MAP_FAILED) {
|
||||
return;
|
||||
}
|
||||
data = spill_mapping = m;
|
||||
spill_size = size;
|
||||
ov_buffer = std::make_shared<ov::Tensor>(ov::element::u8, ov::Shape{size}, data);
|
||||
} else if (const char * spill_dir = ggml_openvino_getenv_str("GGML_OPENVINO_SPILL_DIR")) {
|
||||
#ifndef _WIN32
|
||||
if (const char * spill_dir =
|
||||
ggml_openvino_model_cache_only() ? nullptr : ggml_openvino_getenv_str("GGML_OPENVINO_SPILL_DIR")) {
|
||||
// Disk-backed weight buffer: back the repacked weights with a temp file via MAP_SHARED
|
||||
// instead of anonymous memory. Anonymous pages can only be evicted to swap, so the
|
||||
// repacked buffer stays pinned alongside the mmap'd source and both are resident at once
|
||||
@@ -171,13 +215,27 @@ struct ggml_backend_openvino_buffer_context {
|
||||
#endif
|
||||
{
|
||||
#ifdef _WIN32
|
||||
if (ggml_openvino_getenv_str("GGML_OPENVINO_SPILL_DIR")) {
|
||||
if (!ggml_openvino_model_cache_only() && ggml_openvino_getenv_str("GGML_OPENVINO_SPILL_DIR")) {
|
||||
GGML_LOG_WARN("%s: GGML_OPENVINO_SPILL_DIR is not supported on Windows, ignoring\n", __func__);
|
||||
}
|
||||
#endif
|
||||
data = ggml_aligned_malloc(size);
|
||||
GGML_ASSERT(data);
|
||||
memset(data, 0, size);
|
||||
// Anonymous OS memory reads as zero without being touched, so a buffer becomes resident only as it
|
||||
// is written: a weight buffer tensor by tensor while the model loads, the host-untouched prefill
|
||||
// compute buffer never.
|
||||
#ifdef _WIN32
|
||||
data = VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
|
||||
if (data == nullptr) {
|
||||
return;
|
||||
}
|
||||
#else
|
||||
void * m = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
if (m == MAP_FAILED) {
|
||||
return;
|
||||
}
|
||||
data = m;
|
||||
#endif
|
||||
spill_mapping = data;
|
||||
spill_size = size;
|
||||
ov_buffer = std::make_shared<ov::Tensor>(ov::element::u8, ov::Shape{size}, data);
|
||||
}
|
||||
}
|
||||
@@ -202,17 +260,15 @@ struct ggml_backend_openvino_buffer_context {
|
||||
delete pair.second;
|
||||
}
|
||||
tensor_extras.clear();
|
||||
if (kv_on_demand) {
|
||||
ggml_openvino_kv_storage_unregister(data, size);
|
||||
}
|
||||
if (spill_mapping != nullptr) {
|
||||
#ifdef _WIN32
|
||||
if (spill_mapping != nullptr) {
|
||||
VirtualFree(spill_mapping, 0, MEM_RELEASE);
|
||||
} else
|
||||
#else
|
||||
if (spill_mapping != nullptr) {
|
||||
munmap(spill_mapping, spill_size);
|
||||
} else
|
||||
#endif
|
||||
if (!is_remote && data != nullptr) {
|
||||
ggml_aligned_free(data, size);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -228,7 +284,7 @@ struct ggml_backend_openvino_buffer_type_context {
|
||||
// Host weight-buffer release (GGML_OPENVINO_RELEASE_WEIGHTS)
|
||||
// =====================================================
|
||||
// The OpenVINO weight Constants are zero-copy views into the host buffers
|
||||
// allocated here (ggml_aligned_malloc, anonymous memory). On GPU the plugin
|
||||
// allocated here (anonymous memory). On GPU the plugin
|
||||
// holds its own device copy after compile_model, so the host pages are dead
|
||||
// weight for inference and can be dropped to reclaim RSS (~weights size).
|
||||
//
|
||||
@@ -316,6 +372,12 @@ static bool is_stateful_enabled() {
|
||||
return ggml_openvino_getenv_int("GGML_OPENVINO_STATEFUL_EXECUTION") != 0;
|
||||
}
|
||||
|
||||
static bool is_self_contained_mmap_enabled() {
|
||||
return ggml_openvino_is_npu() && ggml_openvino_npu_raw_weight_cache_enabled() &&
|
||||
ggml_openvino_getenv_int("GGML_OPENVINO_SELF_CONTAINED_MMAP") &&
|
||||
ggml_openvino_getenv_str("GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR");
|
||||
}
|
||||
|
||||
static enum ggml_status ggml_backend_openvino_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) {
|
||||
// GGML_LOG_DEBUG("%s: buffer usage=%d, tensor name=%s\n", __func__, buffer->usage, tensor->name);
|
||||
ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context;
|
||||
@@ -333,6 +395,26 @@ static enum ggml_status ggml_backend_openvino_buffer_init_tensor(ggml_backend_bu
|
||||
tensor->data = (char *) ctx->data + ((char *) tensor->data - (char *) data_prev);
|
||||
}
|
||||
|
||||
// On-demand NPU KV: swap the eagerly committed buffer for demand-zero memory at its first KV tensor
|
||||
// and register every KV tensor; the NPU binds grown L0 storage instead.
|
||||
const bool is_kv_tensor = strncmp(tensor->name, "cache_k_l", 9) == 0 || strncmp(tensor->name, "cache_v_l", 9) == 0;
|
||||
if (is_kv_tensor && tensor->view_src == nullptr && !ctx->is_remote && ggml_openvino_npu_kv_on_demand_enabled() &&
|
||||
tensor->ne[2] == 1 && tensor->ne[3] == 1 && tensor->nb[1] == ggml_row_size(tensor->type, tensor->ne[0])) {
|
||||
if (!ctx->kv_on_demand && ctx->tensor_extras.empty() && tensor->data == ctx->data) {
|
||||
auto device = ctx->device;
|
||||
auto size = ctx->size;
|
||||
auto * data_prev = ctx->data;
|
||||
delete ctx;
|
||||
ctx = new ggml_backend_openvino_buffer_context(device, size, false, true);
|
||||
ctx->kv_on_demand = true;
|
||||
buffer->context = ctx;
|
||||
tensor->data = (char *) ctx->data + ((char *) tensor->data - (char *) data_prev);
|
||||
}
|
||||
if (ctx->kv_on_demand) {
|
||||
ggml_openvino_kv_storage_register(tensor->data, tensor->nb[1], tensor->ne[1]);
|
||||
}
|
||||
}
|
||||
|
||||
// Views share the extra from view_src
|
||||
if (tensor->view_src != nullptr) {
|
||||
GGML_ASSERT(tensor->view_src->buffer->buft == buffer->buft);
|
||||
@@ -347,7 +429,8 @@ static enum ggml_status ggml_backend_openvino_buffer_init_tensor(ggml_backend_bu
|
||||
if (!ggml_are_same_shape(tensor, tensor->view_src) && ggml_is_contiguous(tensor) &&
|
||||
!ggml_is_quantized(tensor->type) && tensor->data != nullptr && ggml_nbytes(tensor) > 0) {
|
||||
if (ggml_openvino_tensor_extra * extra =
|
||||
ggml_openvino_create_tensor_extra(tensor, ctx->is_remote)) {
|
||||
ggml_openvino_create_tensor_extra(tensor, ctx->is_remote,
|
||||
ggml_openvino_get_available_devices()[ctx->device])) {
|
||||
auto it = ctx->tensor_extras.find(tensor);
|
||||
if (it != ctx->tensor_extras.end()) {
|
||||
delete it->second;
|
||||
@@ -364,8 +447,9 @@ static enum ggml_status ggml_backend_openvino_buffer_init_tensor(ggml_backend_bu
|
||||
|
||||
ctx = (ggml_backend_openvino_buffer_context *) buffer->context;
|
||||
|
||||
if (tensor->data != nullptr && !ggml_is_quantized(tensor->type)) {
|
||||
ggml_openvino_tensor_extra * extra = ggml_openvino_create_tensor_extra(tensor, ctx->is_remote);
|
||||
if (tensor->data != nullptr && !ggml_is_quantized(tensor->type) && ggml_nbytes(tensor) > 0) {
|
||||
ggml_openvino_tensor_extra * extra = ggml_openvino_create_tensor_extra(
|
||||
tensor, ctx->is_remote, ggml_openvino_get_available_devices()[ctx->device]);
|
||||
if (extra != nullptr) {
|
||||
auto it = ctx->tensor_extras.find(tensor);
|
||||
if (it != ctx->tensor_extras.end()) {
|
||||
@@ -390,12 +474,12 @@ static void ggml_backend_openvino_buffer_memset_tensor(ggml_backend_buffer_t buf
|
||||
|
||||
if (ctx->is_remote) {
|
||||
// For remote (device) buffers, use OpenCL USM memfill
|
||||
cl_command_queue queue = ggml_openvino_get_cl_queue();
|
||||
cl_command_queue queue = ggml_openvino_get_cl_queue(ggml_openvino_get_available_devices()[ctx->device]);
|
||||
auto mem_fill_fn = ggml_openvino_get_clEnqueueMemFillINTEL();
|
||||
if (mem_fill_fn != nullptr) {
|
||||
uint8_t pattern = value;
|
||||
cl_int err = mem_fill_fn(queue, (char *) tensor->data + offset, &pattern, sizeof(pattern), size, 0, nullptr,
|
||||
nullptr);
|
||||
cl_int err = mem_fill_fn(queue, (char *) tensor->data + offset, &pattern,
|
||||
sizeof(pattern), size, 0, nullptr, nullptr);
|
||||
if (err != CL_SUCCESS) {
|
||||
GGML_LOG_ERROR("%s: clEnqueueMemFillINTEL failed with error %d\n", __func__, err);
|
||||
}
|
||||
@@ -404,10 +488,93 @@ static void ggml_backend_openvino_buffer_memset_tensor(ggml_backend_buffer_t buf
|
||||
GGML_LOG_ERROR("%s: clEnqueueMemFillINTEL not available for GPU buffer\n", __func__);
|
||||
}
|
||||
} else {
|
||||
memset((char *) tensor->data + offset, value, size);
|
||||
if (!ctx->kv_on_demand ||
|
||||
!ggml_openvino_kv_storage_copy((char *) tensor->data + offset, &value, size, GGML_OPENVINO_KV_FILL)) {
|
||||
memset((char *) tensor->data + offset, value, size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Inputs are set on every inference: keep the tensor extra when it still wraps the same memory, shape and type.
|
||||
static bool ggml_openvino_extra_unchanged(const ggml_backend_openvino_buffer_context * ctx,
|
||||
const ggml_tensor * tensor) {
|
||||
auto it = ctx->tensor_extras.find(const_cast<ggml_tensor *>(tensor));
|
||||
if (it == ctx->tensor_extras.end() || it->second == nullptr || tensor->extra != it->second ||
|
||||
it->second->type != ggml_openvino_extra_base::Type::TENSOR) {
|
||||
return false;
|
||||
}
|
||||
const auto * extra = static_cast<const ggml_openvino_tensor_extra *>(it->second);
|
||||
const auto & t = extra->tensor;
|
||||
if (extra->data != tensor->data || t == nullptr || t->get_element_type() != GgmlOvDecoder::get_ov_type(tensor)) {
|
||||
return false;
|
||||
}
|
||||
const auto & shape = t->get_shape();
|
||||
if (shape.size() != GGML_MAX_DIMS) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < GGML_MAX_DIMS; ++i) {
|
||||
if (shape[GGML_MAX_DIMS - 1 - i] != static_cast<size_t>(tensor->ne[i])) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#ifdef _WIN32
|
||||
// QueryWorkingSetEx reports Locked only for pages in the working set; a locked page always is.
|
||||
static bool ggml_openvino_range_has_locked_pages(uintptr_t first, uintptr_t last, uintptr_t page) {
|
||||
std::vector<PSAPI_WORKING_SET_EX_INFORMATION> pages(std::min<size_t>((last - first) / page, 4096));
|
||||
for (uintptr_t addr = first; addr < last;) {
|
||||
const size_t n = std::min<size_t>(pages.size(), (last - addr) / page);
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
pages[i].VirtualAddress = reinterpret_cast<void *>(addr + i * page);
|
||||
}
|
||||
if (!QueryWorkingSetEx(GetCurrentProcess(), pages.data(), DWORD(n * sizeof(pages[0])))) {
|
||||
return true; // unknown, so keep the pages
|
||||
}
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
if (pages[i].VirtualAttributes.Valid && pages[i].VirtualAttributes.Locked) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
addr += n * page;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
// A weight's GGUF bytes have been converted into its OpenVINO buffer. Pages of a read-only file mapping are clean and
|
||||
// come back from the file if they are read again, so release them instead of letting them compete with the buffer for
|
||||
// RAM while the model loads. Only whole pages are released, and locked pages (--load-mode mlock) are kept.
|
||||
static void ggml_openvino_release_source_pages(ggml_backend_openvino_buffer_context * ctx, const void * data, size_t size) {
|
||||
const ggml_openvino_source_mapping * mapping = ggml_openvino_find_source_mapping(data, size, ctx->source_mappings);
|
||||
if (mapping == nullptr || !mapping->read_only) {
|
||||
return;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
SYSTEM_INFO info;
|
||||
GetSystemInfo(&info);
|
||||
const uintptr_t page = info.dwPageSize;
|
||||
#else
|
||||
const uintptr_t page = (uintptr_t) sysconf(_SC_PAGESIZE);
|
||||
#endif
|
||||
const uintptr_t first = ((uintptr_t) data + page - 1) & ~(page - 1);
|
||||
const uintptr_t last = ((uintptr_t) data + size) & ~(page - 1);
|
||||
if (last <= first) {
|
||||
return;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
// VirtualUnlock removes pages that are not locked from the working set, which moves clean file pages to the standby
|
||||
// list, and then reports ERROR_NOT_LOCKED. On locked pages it would unlock them instead, so skip such a range.
|
||||
if (!ggml_openvino_range_has_locked_pages(first, last, page)) {
|
||||
VirtualUnlock(reinterpret_cast<void *>(first), last - first);
|
||||
}
|
||||
#elif defined(__linux__)
|
||||
// MADV_DONTNEED fails on locked (mlock) memory and leaves it resident
|
||||
madvise(reinterpret_cast<void *>(first), last - first, MADV_DONTNEED);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer,
|
||||
ggml_tensor * tensor,
|
||||
const void * data,
|
||||
@@ -438,6 +605,13 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer
|
||||
bool is_supported_weight_shape = is_2d || (tensor->ne[3] == 1 && ggml_is_quantized(tensor->type));
|
||||
|
||||
if (is_weight_buffer && is_full_tensor_set && is_supported_weight_shape) {
|
||||
if (ggml_openvino_is_npu() && ggml_openvino_npu_raw_weight_cache_enabled() &&
|
||||
ggml_openvino_getenv_str("GGML_OPENVINO_COMPILED_MODEL_CACHE_DIR")) {
|
||||
// Keep raw bytes for cache fingerprint validation and compile fallback; the imported
|
||||
// self-contained blob supplies the runtime weights, so eager requant is unnecessary.
|
||||
memcpy(tensor->data, data, size);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
auto result = process_weight_tensor(tensor, data, tensor->data);
|
||||
result.weight_node->set_friendly_name(tensor->name);
|
||||
@@ -489,6 +663,9 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer
|
||||
ggml_openvino_register_weight_buffer(ctx->data, ctx->size);
|
||||
}
|
||||
|
||||
if (!ctx->external_memory) {
|
||||
ggml_openvino_release_source_pages(ctx, data, size);
|
||||
}
|
||||
} catch (const std::exception & e) {
|
||||
GGML_LOG_ERROR("%s: failed to process weight tensor for %s: %s\n", __func__, tensor->name, e.what());
|
||||
memcpy((char *) tensor->data + offset, data, size);
|
||||
@@ -496,22 +673,28 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer
|
||||
} else {
|
||||
// Non-weight tensor (KV cache, activations, etc.) - copy data. test-backend-ops also goes here
|
||||
if (ctx->is_remote) {
|
||||
cl_command_queue queue = ggml_openvino_get_cl_queue();
|
||||
cl_command_queue queue = ggml_openvino_get_cl_queue(ggml_openvino_get_available_devices()[ctx->device]);
|
||||
auto mem_cpy_fn = ggml_openvino_get_clEnqueueMemcpyINTEL();
|
||||
if (mem_cpy_fn != nullptr) {
|
||||
cl_int err =
|
||||
mem_cpy_fn(queue, CL_TRUE, (char *) tensor->data + offset, data, size, 0, nullptr, nullptr);
|
||||
cl_int err = mem_cpy_fn(queue, CL_TRUE, (char *) tensor->data + offset, data, size,
|
||||
0, nullptr, nullptr);
|
||||
if (err != CL_SUCCESS) {
|
||||
GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL failed with error %d\n", __func__, err);
|
||||
}
|
||||
} else {
|
||||
GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL not available for GPU buffer\n", __func__);
|
||||
}
|
||||
} else {
|
||||
} else if (!ctx->kv_on_demand || !ggml_openvino_kv_storage_copy((char *) tensor->data + offset,
|
||||
const_cast<void *>(data), size,
|
||||
GGML_OPENVINO_KV_WRITE)) {
|
||||
memcpy((char *) tensor->data + offset, data, size);
|
||||
}
|
||||
|
||||
ggml_openvino_tensor_extra * extra = ggml_openvino_create_tensor_extra(tensor, ctx->is_remote);
|
||||
if (ggml_openvino_extra_unchanged(ctx, tensor)) {
|
||||
return;
|
||||
}
|
||||
ggml_openvino_tensor_extra * extra = ggml_openvino_create_tensor_extra(
|
||||
tensor, ctx->is_remote, ggml_openvino_get_available_devices()[ctx->device]);
|
||||
if (extra == nullptr) {
|
||||
// GGML_LOG_ERROR("%s: failed to create tensor extra for %s\n", __func__, tensor->name);
|
||||
return;
|
||||
@@ -526,6 +709,64 @@ static void ggml_backend_openvino_buffer_set_tensor(ggml_backend_buffer_t buffer
|
||||
}
|
||||
}
|
||||
|
||||
// Evaluates a weight node (Constant -> Convert -> [Subtract] -> Multiply -> [Reshape]) on the host.
|
||||
static ov::Tensor ggml_openvino_eval_weight_node(const ov::Output<ov::Node> & output) {
|
||||
const auto node = output.get_node_shared_ptr();
|
||||
if (const auto constant = ov::as_type_ptr<ov::op::v0::Constant>(node)) {
|
||||
return constant->get_tensor_view();
|
||||
}
|
||||
ov::TensorVector inputs;
|
||||
for (const auto & input : node->input_values()) {
|
||||
inputs.push_back(ggml_openvino_eval_weight_node(input));
|
||||
}
|
||||
ov::TensorVector outputs;
|
||||
for (size_t i = 0; i < node->get_output_size(); i++) {
|
||||
outputs.emplace_back(node->get_output_element_type(i), node->get_output_shape(i));
|
||||
}
|
||||
if (!node->evaluate(outputs, inputs)) {
|
||||
OPENVINO_THROW("cannot evaluate ", node->get_type_name());
|
||||
}
|
||||
return outputs[output.get_index()];
|
||||
}
|
||||
|
||||
// set_tensor converts a quantized weight in place (extracted or requantized layout), so tensor->data
|
||||
// no longer holds the ggml bytes. Rebuild them from the weight node: dequantize, then quantize back
|
||||
// to the tensor type. The result is close to the original data, but not bit-exact.
|
||||
static bool ggml_openvino_weight_to_ggml(const ggml_tensor * tensor, std::vector<uint8_t> & out) {
|
||||
const auto * extra = static_cast<const ggml_openvino_extra_base *>(tensor->extra);
|
||||
std::shared_ptr<ov::Node> weight_node;
|
||||
if (extra->type == ggml_openvino_extra_base::Type::QUANTIZED_WEIGHT) {
|
||||
weight_node = static_cast<const ggml_openvino_quantized_weight_extra *>(extra)->weight_node;
|
||||
} else if (extra->type == ggml_openvino_extra_base::Type::WEIGHT) {
|
||||
weight_node = static_cast<const ggml_openvino_weight_extra *>(extra)->weight_node;
|
||||
}
|
||||
if (weight_node == nullptr || ggml_quantize_requires_imatrix(tensor->type)) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
ov::Tensor values = ggml_openvino_eval_weight_node(weight_node->output(0));
|
||||
if (values.get_size() != (size_t) ggml_nelements(tensor)) {
|
||||
return false;
|
||||
}
|
||||
if (values.get_element_type() != ov::element::f32) {
|
||||
auto param = std::make_shared<ov::op::v0::Parameter>(values.get_element_type(), values.get_shape());
|
||||
auto convert = std::make_shared<ov::op::v0::Convert>(param, ov::element::f32);
|
||||
ov::TensorVector f32{ov::Tensor(ov::element::f32, values.get_shape())};
|
||||
if (!convert->evaluate(f32, {values})) {
|
||||
return false;
|
||||
}
|
||||
values = f32[0];
|
||||
}
|
||||
out.resize(ggml_nbytes(tensor));
|
||||
ggml_quantize_chunk(tensor->type, values.data<float>(), out.data(), 0, ggml_nrows(tensor), tensor->ne[0],
|
||||
nullptr);
|
||||
return true;
|
||||
} catch (const std::exception & e) {
|
||||
GGML_LOG_WARN("%s: cannot rebuild %s: %s\n", __func__, tensor->name, e.what());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static void ggml_backend_openvino_buffer_get_tensor(ggml_backend_buffer_t buffer,
|
||||
const ggml_tensor * tensor,
|
||||
void * data,
|
||||
@@ -539,20 +780,30 @@ static void ggml_backend_openvino_buffer_get_tensor(ggml_backend_buffer_t buffer
|
||||
GGML_ABORT("ggml-openvino: cannot read unloaded weights in cache-only mode");
|
||||
}
|
||||
|
||||
if (ggml_is_quantized(tensor->type) && tensor->view_src == nullptr && tensor->extra != nullptr &&
|
||||
static_cast<const ggml_openvino_extra_base *>(tensor->extra)->type != ggml_openvino_extra_base::Type::TENSOR) {
|
||||
std::vector<uint8_t> rebuilt;
|
||||
if (ggml_openvino_weight_to_ggml(tensor, rebuilt)) {
|
||||
memcpy(data, rebuilt.data() + offset, size);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (ctx->is_remote) {
|
||||
// For remote (device) buffers, use OpenCL USM memcpy (device-to-host)
|
||||
cl_command_queue queue = ggml_openvino_get_cl_queue();
|
||||
cl_command_queue queue = ggml_openvino_get_cl_queue(ggml_openvino_get_available_devices()[ctx->device]);
|
||||
auto mem_cpy_fn = ggml_openvino_get_clEnqueueMemcpyINTEL();
|
||||
if (mem_cpy_fn != nullptr) {
|
||||
cl_int err =
|
||||
mem_cpy_fn(queue, CL_TRUE, data, (const char *) tensor->data + offset, size, 0, nullptr, nullptr);
|
||||
cl_int err = mem_cpy_fn(queue, CL_TRUE, data, (const char *) tensor->data + offset,
|
||||
size, 0, nullptr, nullptr);
|
||||
if (err != CL_SUCCESS) {
|
||||
GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL failed with error %d\n", __func__, err);
|
||||
}
|
||||
} else {
|
||||
GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL not available for GPU buffer\n", __func__);
|
||||
}
|
||||
} else {
|
||||
} else if (!ctx->kv_on_demand ||
|
||||
!ggml_openvino_kv_storage_copy((const char *) tensor->data + offset, data, size, GGML_OPENVINO_KV_READ)) {
|
||||
memcpy(data, (const char *) tensor->data + offset, size);
|
||||
}
|
||||
}
|
||||
@@ -566,7 +817,7 @@ static bool ggml_backend_openvino_buffer_cpy_tensor(ggml_backend_buffer_t buffer
|
||||
|
||||
if (ctx->is_remote) {
|
||||
// For remote (device) buffers, use OpenCL USM memcpy
|
||||
cl_command_queue queue = ggml_openvino_get_cl_queue();
|
||||
cl_command_queue queue = ggml_openvino_get_cl_queue(ggml_openvino_get_available_devices()[ctx->device]);
|
||||
auto mem_cpy_fn = ggml_openvino_get_clEnqueueMemcpyINTEL();
|
||||
if (mem_cpy_fn == nullptr) {
|
||||
GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL not available for GPU buffer\n", __func__);
|
||||
@@ -574,7 +825,8 @@ static bool ggml_backend_openvino_buffer_cpy_tensor(ggml_backend_buffer_t buffer
|
||||
}
|
||||
// Can copy from host to device
|
||||
if (ggml_backend_buffer_is_host(src->buffer)) {
|
||||
cl_int err = mem_cpy_fn(queue, CL_TRUE, dst->data, src->data, ggml_nbytes(src), 0, nullptr, nullptr);
|
||||
cl_int err = mem_cpy_fn(queue, CL_TRUE, dst->data,
|
||||
src->data, ggml_nbytes(src), 0, nullptr, nullptr);
|
||||
if (err != CL_SUCCESS) {
|
||||
GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL (host-to-device) failed with error %d\n", __func__, err);
|
||||
return false;
|
||||
@@ -585,8 +837,10 @@ static bool ggml_backend_openvino_buffer_cpy_tensor(ggml_backend_buffer_t buffer
|
||||
if (ggml_backend_buffer_is_openvino(src->buffer)) {
|
||||
ggml_backend_openvino_buffer_context * src_ctx =
|
||||
(ggml_backend_openvino_buffer_context *) src->buffer->context;
|
||||
if (src_ctx->is_remote) {
|
||||
cl_int err = mem_cpy_fn(queue, CL_TRUE, dst->data, src->data, ggml_nbytes(src), 0, nullptr, nullptr);
|
||||
if (src_ctx->is_remote && src_ctx->device == ctx->device) {
|
||||
cl_int err = mem_cpy_fn(queue, CL_TRUE, dst->data,
|
||||
src->data, ggml_nbytes(src),
|
||||
0, nullptr, nullptr);
|
||||
if (err != CL_SUCCESS) {
|
||||
GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL (device-to-device) failed with error %d\n", __func__, err);
|
||||
return false;
|
||||
@@ -599,7 +853,17 @@ static bool ggml_backend_openvino_buffer_cpy_tensor(ggml_backend_buffer_t buffer
|
||||
|
||||
// Host buffer - can copy from any host buffer
|
||||
if (ggml_backend_buffer_is_host(src->buffer)) {
|
||||
memcpy(dst->data, src->data, ggml_nbytes(src));
|
||||
const size_t n = ggml_nbytes(src);
|
||||
std::vector<uint8_t> staged;
|
||||
const void * from = src->data;
|
||||
if (ggml_openvino_kv_storage_copy(src->data, nullptr, 0, GGML_OPENVINO_KV_READ)) {
|
||||
staged.resize(n);
|
||||
ggml_openvino_kv_storage_copy(src->data, staged.data(), n, GGML_OPENVINO_KV_READ);
|
||||
from = staged.data();
|
||||
}
|
||||
if (!ggml_openvino_kv_storage_copy(dst->data, const_cast<void *>(from), n, GGML_OPENVINO_KV_WRITE)) {
|
||||
memcpy(dst->data, from, n);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
@@ -609,7 +873,7 @@ static void ggml_backend_openvino_buffer_clear(ggml_backend_buffer_t buffer, uin
|
||||
ggml_backend_openvino_buffer_context * ctx = (ggml_backend_openvino_buffer_context *) buffer->context;
|
||||
GGML_ASSERT(ctx->data != nullptr);
|
||||
if (ctx->is_remote) {
|
||||
cl_command_queue queue = ggml_openvino_get_cl_queue();
|
||||
cl_command_queue queue = ggml_openvino_get_cl_queue(ggml_openvino_get_available_devices()[ctx->device]);
|
||||
auto mem_fill_fn = ggml_openvino_get_clEnqueueMemFillINTEL();
|
||||
if (mem_fill_fn != nullptr) {
|
||||
uint8_t pattern = value;
|
||||
@@ -621,6 +885,8 @@ static void ggml_backend_openvino_buffer_clear(ggml_backend_buffer_t buffer, uin
|
||||
} else {
|
||||
GGML_LOG_WARN("%s: clEnqueueMemFillINTEL not available for GPU buffer clear\n", __func__);
|
||||
}
|
||||
} else if (ctx->kv_on_demand) {
|
||||
ggml_openvino_kv_storage_clear(ctx->data, ctx->size, value);
|
||||
} else {
|
||||
memset(ctx->data, value, ctx->size);
|
||||
}
|
||||
@@ -671,15 +937,19 @@ static size_t ggml_backend_openvino_buffer_type_get_alignment(ggml_backend_buffe
|
||||
}
|
||||
|
||||
static size_t ggml_backend_openvino_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
|
||||
GGML_UNUSED(buft);
|
||||
// A GPU caps a single memory object, so let ggml split a large buffer into parts that fit
|
||||
return ggml_openvino_max_alloc_size();
|
||||
ggml_backend_openvino_buffer_type_context * buft_ctx = (ggml_backend_openvino_buffer_type_context *) buft->context;
|
||||
return ggml_openvino_max_alloc_size(ggml_openvino_get_available_devices()[buft_ctx->device]);
|
||||
}
|
||||
|
||||
static size_t ggml_backend_openvino_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft,
|
||||
const ggml_tensor * tensor) {
|
||||
GGML_UNUSED(buft);
|
||||
|
||||
if (is_self_contained_mmap_enabled()) {
|
||||
return ggml_nbytes(tensor);
|
||||
}
|
||||
|
||||
// For quantized weight tensors, we need extra space for extracted data.
|
||||
if (!ggml_openvino_model_cache_only() && ggml_is_quantized(tensor->type) && tensor->ne[3] == 1) {
|
||||
ggml_openvino_extracted_layout layout = ggml_openvino_get_extracted_layout(tensor);
|
||||
@@ -836,6 +1106,11 @@ bool ggml_backend_buft_is_openvino_host(ggml_backend_buffer_type_t buft) {
|
||||
return buft->iface.get_name == ggml_backend_openvino_host_buffer_type_get_name;
|
||||
}
|
||||
|
||||
// Fingerprints of weights outside OpenVINO buffers (e.g. a CPU-buffer token_embd), keyed by address, size and name. A
|
||||
// later model can reuse an address, so the map is cleared with the shared compiled-model cache.
|
||||
static std::mutex g_fallback_fingerprints_mutex;
|
||||
static std::map<std::tuple<const void *, size_t, std::string>, uint64_t> g_fallback_fingerprints;
|
||||
|
||||
uint64_t ggml_backend_openvino_weight_fingerprint(const ggml_tensor * tensor) {
|
||||
if (ggml_backend_buffer_is_openvino(tensor->buffer)) {
|
||||
auto * ctx = static_cast<ggml_backend_openvino_buffer_context *>(tensor->buffer->context);
|
||||
@@ -843,10 +1118,24 @@ uint64_t ggml_backend_openvino_weight_fingerprint(const ggml_tensor * tensor) {
|
||||
if (it != ctx->weight_fingerprints.end()) {
|
||||
return it->second;
|
||||
}
|
||||
if (ctx->external_memory) {
|
||||
// GGML_OPENVINO_SELF_CONTAINED_MMAP wraps the GGUF mmap, so the weights are never uploaded
|
||||
// through set_tensor. Identify them from the same mapping set_tensor saw on the cold run.
|
||||
const uint64_t fp = ggml_openvino_source_fingerprint(tensor->data, ggml_nbytes(tensor), ctx->source_mappings);
|
||||
ctx->weight_fingerprints[tensor->data] = fp;
|
||||
return fp;
|
||||
}
|
||||
GGML_ABORT("ggml-openvino: missing source identity for weight %s", tensor->name);
|
||||
}
|
||||
const auto key =
|
||||
std::make_tuple(static_cast<const void *>(tensor->data), ggml_nbytes(tensor), std::string(tensor->name));
|
||||
std::lock_guard<std::mutex> lock(g_fallback_fingerprints_mutex);
|
||||
auto it = g_fallback_fingerprints.find(key);
|
||||
if (it != g_fallback_fingerprints.end()) {
|
||||
return it->second;
|
||||
}
|
||||
std::vector<ggml_openvino_source_mapping> mappings;
|
||||
return ggml_openvino_source_fingerprint(tensor->data, ggml_nbytes(tensor), mappings);
|
||||
return g_fallback_fingerprints[key] = ggml_openvino_source_fingerprint(tensor->data, ggml_nbytes(tensor), mappings);
|
||||
}
|
||||
|
||||
static void ggml_backend_openvino_free(ggml_backend_t backend) {
|
||||
@@ -864,6 +1153,9 @@ static void ggml_backend_openvino_free(ggml_backend_t backend) {
|
||||
// reuses it instead of recompiling against zeroed weights.
|
||||
if (!ggml_openvino_weight_buffers_released()) {
|
||||
cache->graphs.clear();
|
||||
ov_clear_imported_blobs();
|
||||
std::lock_guard<std::mutex> fingerprints_lock(g_fallback_fingerprints_mutex);
|
||||
g_fallback_fingerprints.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -911,11 +1203,11 @@ static ggml_guid_t ggml_backend_openvino_guid(void) {
|
||||
return &guid;
|
||||
}
|
||||
|
||||
static std::shared_ptr<ov_runtime_context> get_ov_runtime_context_ptr() {
|
||||
static std::shared_ptr<ov_runtime_context> get_ov_runtime_context_ptr(int device) {
|
||||
// Share compiled models, but give every backend its own requests and KV state.
|
||||
static auto cache = std::make_shared<ov_compiled_model_cache>();
|
||||
auto r_ctx = std::make_shared<ov_runtime_context>();
|
||||
r_ctx->device = ggml_openvino_get_device_name();
|
||||
r_ctx->device = ggml_openvino_get_available_devices()[device];
|
||||
r_ctx->stateful = is_stateful_enabled() && !ggml_openvino_is_npu();
|
||||
r_ctx->compiled_cache = cache;
|
||||
std::lock_guard<std::mutex> cache_lock(cache->mutex);
|
||||
@@ -936,7 +1228,7 @@ GGML_BACKEND_API ggml_backend_t ggml_backend_openvino_init(int device) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ctx->runtime_context = get_ov_runtime_context_ptr();
|
||||
ctx->runtime_context = get_ov_runtime_context_ptr(device);
|
||||
if (ctx->runtime_context == nullptr) {
|
||||
GGML_LOG_ERROR("%s: failed to allocate runtime context\n", __func__);
|
||||
delete ctx;
|
||||
@@ -1124,9 +1416,9 @@ static void ggml_backend_openvino_device_get_memory(ggml_backend_dev_t dev, size
|
||||
|
||||
static enum ggml_backend_dev_type ggml_backend_openvino_device_get_type(ggml_backend_dev_t dev) {
|
||||
ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context;
|
||||
// Only the device selected by GGML_OPENVINO_DEVICE is offered for offload. The others are
|
||||
// Only the devices selected by GGML_OPENVINO_DEVICE are offered for offload. The others are
|
||||
// registered for discovery (--list-devices) only; llama.cpp skips IGPU devices when a GPU exists.
|
||||
return ctx->ov_name == ggml_openvino_get_device_name() ? GGML_BACKEND_DEVICE_TYPE_GPU : GGML_BACKEND_DEVICE_TYPE_IGPU;
|
||||
return ggml_openvino_is_selected_device(ctx->ov_name) ? GGML_BACKEND_DEVICE_TYPE_GPU : GGML_BACKEND_DEVICE_TYPE_IGPU;
|
||||
}
|
||||
|
||||
static void ggml_backend_openvino_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {
|
||||
@@ -1138,7 +1430,7 @@ static void ggml_backend_openvino_device_get_props(ggml_backend_dev_t dev, ggml_
|
||||
props->caps = {
|
||||
/* .async = */ false,
|
||||
/* .host_buffer = */ false,
|
||||
/* .buffer_from_host_ptr = */ false,
|
||||
/* .buffer_from_host_ptr = */ is_self_contained_mmap_enabled(),
|
||||
/* .events = */ false,
|
||||
/* .mmap_support = */ true,
|
||||
};
|
||||
@@ -1147,7 +1439,7 @@ static void ggml_backend_openvino_device_get_props(ggml_backend_dev_t dev, ggml_
|
||||
static ggml_backend_t ggml_backend_openvino_device_init(ggml_backend_dev_t dev, const char * params) {
|
||||
GGML_UNUSED(params);
|
||||
ggml_backend_openvino_device_context * ctx = (ggml_backend_openvino_device_context *) dev->context;
|
||||
if (ctx->ov_name != ggml_openvino_get_device_name()) {
|
||||
if (!ggml_openvino_is_selected_device(ctx->ov_name)) {
|
||||
// Not an error: test-backend-ops initializes every device
|
||||
GGML_LOG_WARN("%s: %s (OpenVINO %s) is not the selected device, no ops will run on it; "
|
||||
"set GGML_OPENVINO_DEVICE=%s to use it\n",
|
||||
@@ -1166,6 +1458,17 @@ static ggml_backend_buffer_type_t ggml_backend_openvino_device_get_host_buffer_t
|
||||
return ggml_backend_openvino_host_buffer_type(ctx->device);
|
||||
}
|
||||
|
||||
static ggml_backend_buffer_t ggml_backend_openvino_device_buffer_from_host_ptr(ggml_backend_dev_t dev,
|
||||
void * ptr,
|
||||
size_t size,
|
||||
size_t max_tensor_size) {
|
||||
auto * dev_ctx = static_cast<ggml_backend_openvino_device_context *>(dev->context);
|
||||
auto * ctx = new ggml_backend_openvino_buffer_context(dev_ctx->device, ptr, size);
|
||||
GGML_UNUSED(max_tensor_size);
|
||||
return ggml_backend_buffer_init(ggml_backend_openvino_buffer_type(dev_ctx->device),
|
||||
ggml_backend_openvino_buffer_interface, ctx, size);
|
||||
}
|
||||
|
||||
static bool has_view_op_input(const ggml_tensor * op) {
|
||||
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
||||
if (op->src[i] == nullptr) {
|
||||
@@ -1181,6 +1484,11 @@ static bool has_view_op_input(const ggml_tensor * op) {
|
||||
// OV slices whole elements per axis, so each stride must be a multiple of the next smaller one
|
||||
// (e.g. a batch stride of m*nb[1] + pad bytes cannot be expressed and would be read wrongly).
|
||||
static bool has_strides_on_element_grid(const ggml_tensor * t) {
|
||||
// nothing is read from an empty tensor (gemma-4's KV-shared layers in a batch without output rows), and its strides
|
||||
// past the empty dim are 0
|
||||
if (ggml_nelements(t) == 0) {
|
||||
return true;
|
||||
}
|
||||
std::vector<size_t> strides;
|
||||
for (int i = 0; i < GGML_MAX_DIMS; i++) {
|
||||
if (t->ne[i] > 1) {
|
||||
@@ -1213,16 +1521,31 @@ static bool is_supported_flash_attn_pattern(const ggml_tensor * op) {
|
||||
// PERMUTE -> VIEW -> base (view_src==nullptr) (llama KV-cache path)
|
||||
// PERMUTE -> RESHAPE -> base (view_src==nullptr) (whisper Q)
|
||||
// VIEW -> base (view_src==nullptr) (whisper K/V from kv_pad)
|
||||
// PERMUTE -> VIEW -> RESHAPE -> base (Q of a layer without RoPE, e.g. SmolLM3)
|
||||
for (int i = 0; i < 3; i++) {
|
||||
const ggml_tensor * src = op->src[i];
|
||||
if (src->op == GGML_OP_PERMUTE) {
|
||||
if (src->src[0] == nullptr) {
|
||||
return false;
|
||||
}
|
||||
// MLA: Q = PERMUTE -> CONCAT(q_nope_absorbed, q_pe); V = PERMUTE -> VIEW -> VIEW -> K cache
|
||||
// (V is the first kv_lora_rank values of each K cache row)
|
||||
if (i == 0 && src->src[0]->op == GGML_OP_CONCAT) {
|
||||
continue;
|
||||
}
|
||||
if (i == 2 && src->src[0]->op == GGML_OP_VIEW && src->src[0]->src[0] != nullptr &&
|
||||
src->src[0]->src[0]->op == GGML_OP_VIEW && src->src[0]->src[0]->src[0] != nullptr &&
|
||||
src->src[0]->src[0]->src[0]->view_src == nullptr) {
|
||||
continue;
|
||||
}
|
||||
if (src->src[0]->op != GGML_OP_VIEW && src->src[0]->op != GGML_OP_RESHAPE) {
|
||||
return false;
|
||||
}
|
||||
if (src->src[0]->src[0] == nullptr || src->src[0]->src[0]->view_src != nullptr) {
|
||||
const ggml_tensor * base = src->src[0]->src[0];
|
||||
if (base != nullptr && src->src[0]->op == GGML_OP_VIEW && base->op == GGML_OP_RESHAPE) {
|
||||
base = base->src[0];
|
||||
}
|
||||
if (base == nullptr || base->view_src != nullptr) {
|
||||
return false;
|
||||
}
|
||||
} else if (src->op == GGML_OP_VIEW) {
|
||||
@@ -1300,7 +1623,8 @@ static bool cpy_output_view_is_supported(const ggml_tensor * op) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return ggml_nbytes(op) == 0 || ggml_is_contiguous(op) || GgmlOvDecoder::is_conv_state_writeback(op);
|
||||
return ggml_nbytes(op) == 0 || ggml_is_contiguous(op) || GgmlOvDecoder::is_conv_state_writeback(op) ||
|
||||
GgmlOvDecoder::is_gdn_snapshot_writeback(op);
|
||||
}
|
||||
|
||||
static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) {
|
||||
@@ -1368,6 +1692,12 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
}
|
||||
|
||||
switch (op->op) {
|
||||
case GGML_OP_TOP_K: {
|
||||
if (op->src[0]->type != GGML_TYPE_F32 || op->type != GGML_TYPE_I32 || ggml_openvino_is_npu()) {
|
||||
return {false, "TOP_K requires F32 input and I32 output on CPU or GPU"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_CONCAT: {
|
||||
if (op->type == GGML_TYPE_I64) {
|
||||
return {false, "CONCAT with I64 type is not supported"};
|
||||
@@ -1397,12 +1727,13 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
}
|
||||
case GGML_OP_GET_ROWS:
|
||||
case GGML_OP_SET_ROWS: {
|
||||
if (op->ne[3] != 1) {
|
||||
return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"};
|
||||
if (GgmlOvDecoder::is_conv_state_snapshot_writeback(op) &&
|
||||
(ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC"))) {
|
||||
return {false, "convolution snapshots require dynamic execution"};
|
||||
}
|
||||
if (op->op == GGML_OP_GET_ROWS && ggml_openvino_is_gpu() &&
|
||||
op->src[0]->type == GGML_TYPE_BF16) {
|
||||
return {false, "GET_ROWS with BF16 src0 is not supported on GPU"};
|
||||
if (op->ne[3] != 1 && !GgmlOvDecoder::is_get_rows_batched_4d(op) &&
|
||||
!GgmlOvDecoder::is_conv_state_snapshot_writeback(op)) {
|
||||
return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"};
|
||||
}
|
||||
if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K ||
|
||||
op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_1)) {
|
||||
@@ -1473,12 +1804,26 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
// The GPU plugin can fuse broadcast DIV into the preceding FFN GEMM path
|
||||
// and produce infs for per-channel scale vectors. Keep those DIVs on CPU
|
||||
// until the fused GPU kernel is reliable. (falied case llama-arch-test mpt)
|
||||
// Only a weight divisor is such a scale; at one token an activation divisor has the same shape
|
||||
// (qwen2moe's shared-expert sigmoid, silu(x) / x).
|
||||
if (ggml_openvino_is_gpu() && op->src[1]->ne[0] == op->ne[0] &&
|
||||
op->src[1]->ne[1] == 1 && op->src[1]->ne[2] == 1 && op->src[1]->ne[3] == 1) {
|
||||
op->src[1]->ne[1] == 1 && op->src[1]->ne[2] == 1 && op->src[1]->ne[3] == 1 &&
|
||||
op->src[1]->buffer != nullptr && op->src[1]->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
|
||||
return {false, "DIV per-channel scale broadcast is not supported on GPU"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_POOL_1D: {
|
||||
const ggml_op_pool pool_mode = static_cast<ggml_op_pool>(op->op_params[0]);
|
||||
if (pool_mode != GGML_OP_POOL_MAX && pool_mode != GGML_OP_POOL_AVG) {
|
||||
return {false, "POOL_1D mode " + std::to_string(pool_mode) + " is not supported"};
|
||||
}
|
||||
// a window fully in the padding gives -FLT_MAX (max) or 0 (avg) in ggml, OV differs
|
||||
if (op->op_params[3] >= op->op_params[1]) {
|
||||
return {false, "POOL_1D with padding >= kernel size is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_POOL_2D: {
|
||||
if (ggml_openvino_is_gpu()) {
|
||||
const int32_t * params = op->op_params;
|
||||
@@ -1531,8 +1876,8 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
return {false, "FLASH_ATTN_EXT with K/V row lists is not supported"};
|
||||
}
|
||||
|
||||
if (op->src[4] != nullptr) {
|
||||
return {false, "FLASH_ATTN_EXT with sinks is not supported"};
|
||||
if (op->src[4] != nullptr && op->src[3] == nullptr) {
|
||||
return {false, "FLASH_ATTN_EXT with sinks but no mask is not supported"};
|
||||
}
|
||||
if (!is_supported_flash_attn_pattern(op)) {
|
||||
return {false, "FLASH_ATTN_EXT unsupported attention pattern"};
|
||||
@@ -1540,8 +1885,14 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
if (max_bias > 0) {
|
||||
return {false, "FLASH_ATTN_EXT with max_bias > 0 (max_bias=" + std::to_string(max_bias) + ") is not supported"};
|
||||
}
|
||||
if (logit_softcap != 0) {
|
||||
return {false, "FLASH_ATTN_EXT with logit_softcap != 0 (logit_softcap=" + std::to_string(logit_softcap) + ") is not supported"};
|
||||
// soft-capping (gemma-2) is applied on the manual attention path of translate_flash_attn_ext, which needs one
|
||||
// head size for Q, K and V
|
||||
if (logit_softcap != 0 && (op->src[1]->ne[0] != op->src[0]->ne[0] || op->src[2]->ne[0] != op->src[0]->ne[0])) {
|
||||
return {false, "FLASH_ATTN_EXT with logit_softcap != 0 needs one head size for Q, K and V"};
|
||||
}
|
||||
// sinks are applied only by SDPA, which has no soft-capping
|
||||
if (logit_softcap != 0 && op->src[4] != nullptr) {
|
||||
return {false, "FLASH_ATTN_EXT with sinks and logit_softcap != 0 is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1608,6 +1959,14 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
op->src[0]->buffer == nullptr) {
|
||||
return {false, "MUL_MAT with unbound Q4_1/Q4_K src[0] on GPU is not supported"};
|
||||
}
|
||||
// The GPU plugin fails to compile a MatMul with batched quantized weights (one matrix per batch, e.g. MLA's
|
||||
// attn_k_b) when shapes are static: wrong results or "ProgramBuilder build failed". Dynamic shapes work. Op
|
||||
// tests (unbound weights, see above) and the static pipeline compile with static shapes. Remove once the GPU
|
||||
// plugin is fixed.
|
||||
if (ggml_openvino_is_gpu() && ggml_is_quantized(op->src[0]->type) && op->src[0]->ne[2] > 1 &&
|
||||
(op->src[0]->buffer == nullptr || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC"))) {
|
||||
return {false, "MUL_MAT with batched quantized src[0] and static shapes on GPU is not supported"};
|
||||
}
|
||||
if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) {
|
||||
return {false, "MUL_MAT with incompatible broadcast on ne[3]: src0->ne[3]=" + std::to_string(op->src[0]->ne[3]) +
|
||||
", src1->ne[3]=" + std::to_string(op->src[1]->ne[3])};
|
||||
@@ -1624,9 +1983,6 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
return {false, "MUL_MAT_ID with single-expert or empty ne[2] <= 1 (ne[2]=" +
|
||||
std::to_string(op->src[0]->ne[2]) + ") is not supported"};
|
||||
}
|
||||
if (ggml_openvino_is_gpu() && op->src[0] != nullptr && !ggml_is_quantized(op->src[0]->type)) {
|
||||
return {false, "MUL_MAT_ID with non-quantized weights on GPU is not supported"};
|
||||
}
|
||||
// The GPU plugin's GatherMatmul returns wrong values for the layouts test-backend-ops
|
||||
// produces: it builds a rank-4 input layout ([n_used, n_tokens, k, 1]) instead of rank 3
|
||||
// and the kernel misreads it, silently returning garbage (NMSE ~86) rather than asserting.
|
||||
@@ -1669,16 +2025,25 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
// // CVS-186471
|
||||
// return true;
|
||||
// }
|
||||
const char * reason = nullptr;
|
||||
if (op->src[2]->op == GGML_OP_PERMUTE) {
|
||||
return {false, "GATED_DELTA_NET with PERMUTE src2 is not supported"};
|
||||
reason = "GATED_DELTA_NET with PERMUTE src2 is not supported";
|
||||
}
|
||||
// kda (per-key-dimension gating) not supported by fused GatedDeltaNet op
|
||||
if (op->src[3]->ne[0] != 1) {
|
||||
return {false, "GATED_DELTA_NET with kda (per-key-dimension gating) is not supported"};
|
||||
else if (op->src[3]->ne[0] != 1) {
|
||||
// kda (per-key-dimension gating) not supported by fused GatedDeltaNet op
|
||||
reason = "GATED_DELTA_NET with kda (per-key-dimension gating) is not supported";
|
||||
}
|
||||
// K > 1 (multiple state snapshots) not supported by fused op
|
||||
if (((const int32_t *) op->op_params)[0] > 1) {
|
||||
return {false, "GATED_DELTA_NET with K > 1 (multiple state snapshots) is not supported"};
|
||||
else if (op->op_params[0] > 1) {
|
||||
if (ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC")) {
|
||||
reason = "GATED_DELTA_NET snapshots require dynamic execution";
|
||||
}
|
||||
}
|
||||
if (reason != nullptr) {
|
||||
if (ggml_openvino_gdn_native_state_enabled()) {
|
||||
// the CPU fallback reads the state in the ggml layout and would silently corrupt it
|
||||
GGML_ABORT("ggml-openvino: GGML_OPENVINO_GDN_NATIVE_STATE is set, but %s", reason);
|
||||
}
|
||||
return {false, reason};
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1784,7 +2149,7 @@ static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(gg
|
||||
GGML_ASSERT(dev->reg != nullptr);
|
||||
|
||||
ggml_backend_openvino_device_context * dev_ctx = (ggml_backend_openvino_device_context *) dev->context;
|
||||
if (dev_ctx->ov_name != ggml_openvino_get_device_name()) {
|
||||
if (!ggml_openvino_is_selected_device(dev_ctx->ov_name)) {
|
||||
// Data placed on a non-selected device (e.g. with -dev) can never run here; stop with a hint
|
||||
// instead of the generic scheduler abort. Unallocated tensors (test-backend-ops) pass through.
|
||||
for (int i = -1; i < GGML_MAX_SRC; i++) {
|
||||
@@ -1801,9 +2166,22 @@ static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(gg
|
||||
return {false, "device is not the selected OpenVINO device"};
|
||||
}
|
||||
|
||||
// The write-back of GATED_DELTA_NET rollback snapshots stores into the recurrent cache in this backend's buffer,
|
||||
// so it always runs here. When the GDN that produces the snapshots falls back to ggml, the write-back reads the
|
||||
// packed GDN output wrongly and silently corrupts the cache, so stop with the reason instead.
|
||||
if (GgmlOvDecoder::is_gdn_snapshot_writeback(op)) {
|
||||
const auto gdn_support = ggml_backend_openvino_device_supports_op_impl(dev, op->src[0]->src[0]);
|
||||
if (!gdn_support.is_supported) {
|
||||
GGML_ABORT(
|
||||
"ggml-openvino: %s. Recurrent rollback (n_rs_seq > 0, used by MTP, EAGLE3, DFlash and DSpark "
|
||||
"speculative decoding) is not supported for this model on this device",
|
||||
gdn_support.reason.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
static std::unordered_set<ggml_type> supported_types{
|
||||
GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64, GGML_TYPE_I32, GGML_TYPE_Q4_0,
|
||||
GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K,
|
||||
GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K,
|
||||
GGML_TYPE_MXFP4};
|
||||
|
||||
// derive supported op sets from the op_table map, keys in
|
||||
@@ -1850,9 +2228,12 @@ static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(gg
|
||||
if (!supported) {
|
||||
return {false, "unary op " + std::string(ggml_unary_op_name(ggml_get_unary_op(op))) + " has no op translator"};
|
||||
}
|
||||
if (op->type == GGML_TYPE_F32 && (ggml_get_unary_op(op) == GGML_UNARY_OP_EXP ||
|
||||
ggml_get_unary_op(op) == GGML_UNARY_OP_EXPM1)) {
|
||||
return {false, "UNARY_EXP / UNARY_EXPM1 with F32 type is not supported"};
|
||||
// OV GPU matches ggml, CPU fails some tests
|
||||
if (op->type == GGML_TYPE_F32 && ggml_get_unary_op(op) == GGML_UNARY_OP_EXP && !ggml_openvino_is_gpu()) {
|
||||
return {false, "UNARY_EXP with F32 type is not supported on " + ggml_openvino_get_device_name()};
|
||||
}
|
||||
if (op->type == GGML_TYPE_F32 && ggml_get_unary_op(op) == GGML_UNARY_OP_EXPM1) {
|
||||
return {false, "UNARY_EXPM1 with F32 type is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -1893,12 +2274,17 @@ static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(gg
|
||||
if (supported_types.find(src->type) == supported_types.end()) {
|
||||
return {false, "src[" + std::to_string(i) + "] type " + std::string(ggml_type_name(src->type)) + " is not supported"};
|
||||
}
|
||||
if (!has_strides_on_element_grid(src)) {
|
||||
if (!has_strides_on_element_grid(src) && !(i == 1 && GgmlOvDecoder::is_gdn_snapshot_writeback(op)) &&
|
||||
!(i == 2 && GgmlOvDecoder::is_conv_state_snapshot_writeback(op))) {
|
||||
return {false, "src[" + std::to_string(i) + "] strides are not multiples of each other"};
|
||||
}
|
||||
const bool is_supported_3d_moe_expert =
|
||||
op->op == GGML_OP_MUL_MAT_ID && i == 0 && (src->type == GGML_TYPE_MXFP4 || src->ne[3] == 1);
|
||||
if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_moe_expert) {
|
||||
// one weight matrix per activation batch, e.g. MLA's per-head attn_k_b / attn_v_b (a batched MatMul)
|
||||
const bool is_supported_3d_batched_weight = op->op == GGML_OP_MUL_MAT && i == 0 && src->ne[3] == 1 &&
|
||||
op->src[1]->ne[2] == src->ne[2] && op->src[1]->ne[3] == 1;
|
||||
if (ggml_is_quantized(src->type) && src->ne[2] != 1 && !is_supported_3d_moe_expert &&
|
||||
!is_supported_3d_batched_weight) {
|
||||
return {false, "3D quantized tensor for src[" + std::to_string(i) + "] is not supported"};
|
||||
}
|
||||
}
|
||||
@@ -1923,8 +2309,13 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
|
||||
}
|
||||
|
||||
static bool ggml_backend_openvino_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
|
||||
return ggml_backend_buft_is_openvino(buft) || ggml_backend_buft_is_host(buft);
|
||||
GGML_UNUSED(dev);
|
||||
if (ggml_backend_buft_is_openvino(buft)) {
|
||||
ggml_backend_openvino_device_context * dev_ctx = (ggml_backend_openvino_device_context *) dev->context;
|
||||
ggml_backend_openvino_buffer_type_context * buft_ctx =
|
||||
(ggml_backend_openvino_buffer_type_context *) buft->context;
|
||||
return buft_ctx->device == dev_ctx->device;
|
||||
}
|
||||
return ggml_backend_buft_is_host(buft);
|
||||
}
|
||||
|
||||
static const struct ggml_backend_device_i ggml_backend_openvino_device_interface = {
|
||||
@@ -1936,7 +2327,7 @@ static const struct ggml_backend_device_i ggml_backend_openvino_device_interface
|
||||
/* .init_backend = */ ggml_backend_openvino_device_init,
|
||||
/* .get_buffer_type = */ ggml_backend_openvino_device_get_buffer_type,
|
||||
/* .get_host_buffer_type = */ ggml_backend_openvino_device_get_host_buffer_type,
|
||||
/* .buffer_from_host_ptr = */ NULL,
|
||||
/* .buffer_from_host_ptr = */ ggml_backend_openvino_device_buffer_from_host_ptr,
|
||||
/* .supports_op = */ ggml_backend_openvino_device_supports_op,
|
||||
/* .supports_buft = */ ggml_backend_openvino_device_supports_buft,
|
||||
/* .offload_op = */ NULL,
|
||||
@@ -2001,7 +2392,7 @@ GGML_BACKEND_API ggml_backend_reg_t ggml_backend_openvino_reg(void) {
|
||||
dev_ctx->ov_name = openvino_devices[i];
|
||||
// The device is chosen with GGML_OPENVINO_DEVICE, not -dev, so show the value to set
|
||||
dev_ctx->description = "GGML_OPENVINO_DEVICE=" + dev_ctx->ov_name +
|
||||
(dev_ctx->ov_name == ggml_openvino_get_device_name() ? " (selected)" : "") +
|
||||
(ggml_openvino_is_selected_device(dev_ctx->ov_name) ? " (selected)" : "") +
|
||||
" - " + ggml_openvino_get_device_description(dev_ctx->ov_name);
|
||||
dev_ctx->total_memory = 0;
|
||||
if (ov_device_has_prefix(dev_ctx->ov_name, "GPU")) {
|
||||
|
||||
@@ -171,6 +171,38 @@ void extract_q4_1_data(const ggml_tensor * tensor,
|
||||
}
|
||||
}
|
||||
|
||||
// Extracts (weight, scales) from Q5_0 tensors.
|
||||
// Data layout is: |16 bit scale|32 bit qh (5th bits)|32 x 4bit low nibbles|.
|
||||
// Reconstructed quant q in [0,31]: q = (low nibble) | (qh_bit << 4). Dequant: (q - 16) * d.
|
||||
// Weights are stored as signed i8 (q - 16), symmetric like Q8_0, so there is no zero point.
|
||||
void extract_q5_0_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr) {
|
||||
const uint64_t bytes_per_block = 22; // 2 scale + 4 qh + 16 (32x0.5) weights
|
||||
const int qk = 32;
|
||||
|
||||
OPENVINO_ASSERT(weights_arr.get_element_type() == ov::element::i8, "Q5_0 weights are extracted as i8");
|
||||
|
||||
auto * data = static_cast<uint8_t *>(tensor->data);
|
||||
auto * weights = static_cast<int8_t *>(weights_arr.data());
|
||||
auto * scales = scales_arr.data<ov::element_type_traits<ov::element::f16>::value_type>();
|
||||
|
||||
ov::parallel_for(scales_arr.get_size(), [&](size_t i) {
|
||||
const uint8_t * block = data + i * bytes_per_block;
|
||||
uint16_t d;
|
||||
memcpy(&d, block, sizeof(d));
|
||||
uint32_t qh;
|
||||
memcpy(&qh, block + 2, sizeof(qh));
|
||||
const uint8_t * qs = block + 6;
|
||||
int8_t * dst = weights + i * qk;
|
||||
scales[i] = ov::float16::from_bits(d);
|
||||
for (int j = 0; j < qk / 2; ++j) {
|
||||
const uint8_t bit_lo = (qh >> j) & 1;
|
||||
const uint8_t bit_hi = (qh >> (j + qk / 2)) & 1;
|
||||
dst[j] = static_cast<int8_t>(((qs[j] & 0x0F) | (bit_lo << 4)) - 16); // first 16 weights
|
||||
dst[j + qk / 2] = static_cast<int8_t>(((qs[j] >> 4) | (bit_hi << 4)) - 16); // last 16 weights
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Extracts (weight, scales, zp) from Q5_1 tensors.
|
||||
// Data layout is: |16 bit scale|16 bit min|32 bit qh (5th bits)|32 x 4bit low nibbles|.
|
||||
// Reconstructed quant q in [0,31]: q = (low nibble) | (qh_bit << 4). Dequant: w*d + m.
|
||||
@@ -771,7 +803,8 @@ void quantize_q4_0(const float * x,
|
||||
ov::Tensor & scales_arr,
|
||||
ov::Tensor & zp_arr,
|
||||
int64_t k,
|
||||
int64_t qk) {
|
||||
int64_t qk,
|
||||
int64_t block_offset = 0) {
|
||||
assert(k % qk == 0);
|
||||
const int nb = k / qk;
|
||||
|
||||
@@ -782,6 +815,9 @@ void quantize_q4_0(const float * x,
|
||||
if (!is_symmetric) {
|
||||
auto * zp = static_cast<uint8_t *>(zp_arr.data());
|
||||
for (int i = 0; i < nb; i++) {
|
||||
// Destination block index. Chunks are emitted in increasing order, so the
|
||||
// even-block assignment always precedes the odd-block OR of the same zp byte.
|
||||
const int64_t gi = block_offset + i;
|
||||
float amax = 0.0f;
|
||||
float max = 0.0f;
|
||||
for (int j = 0; j < qk; j++) {
|
||||
@@ -793,33 +829,34 @@ void quantize_q4_0(const float * x,
|
||||
}
|
||||
const float d = max / -8;
|
||||
if (d == 0) {
|
||||
scales[i] = ov::float16(1.0f);
|
||||
if (i % 2 == 0) {
|
||||
zp[i / 2] = 8;
|
||||
scales[gi] = ov::float16(1.0f);
|
||||
if (gi % 2 == 0) {
|
||||
zp[gi / 2] = 8;
|
||||
} else {
|
||||
zp[i / 2] |= (8 << 4);
|
||||
zp[gi / 2] |= (8 << 4);
|
||||
}
|
||||
memset(weights + i * qk / 2, 8 | (8 << 4), qk / 2);
|
||||
memset(weights + gi * qk / 2, 8 | (8 << 4), qk / 2);
|
||||
continue;
|
||||
}
|
||||
const float id = 1.0f / d;
|
||||
scales[i] = ov::float16(d);
|
||||
if (i % 2 == 0) {
|
||||
zp[i / 2] = 8;
|
||||
scales[gi] = ov::float16(d);
|
||||
if (gi % 2 == 0) {
|
||||
zp[gi / 2] = 8;
|
||||
} else {
|
||||
zp[i / 2] |= (8 << 4);
|
||||
zp[gi / 2] |= (8 << 4);
|
||||
}
|
||||
for (int j = 0; j < qk / 2; ++j) {
|
||||
const float x0 = x[i * qk + 2 * j] * id;
|
||||
const float x1 = x[i * qk + 2 * j + 1] * id;
|
||||
const uint8_t xi0 = MIN(15, (int8_t) (x0 + 8.5f));
|
||||
const uint8_t xi1 = MIN(15, (int8_t) (x1 + 8.5f));
|
||||
weights[i * qk / 2 + j] = xi0 | (xi1 << 4);
|
||||
weights[gi * qk / 2 + j] = xi0 | (xi1 << 4);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Symmetric: produce signed i4 values in [-8, 7]
|
||||
for (int i = 0; i < nb; i++) {
|
||||
const int64_t gi = block_offset + i;
|
||||
float amax = 0.0f;
|
||||
float max = 0.0f;
|
||||
for (int j = 0; j < qk; j++) {
|
||||
@@ -831,20 +868,20 @@ void quantize_q4_0(const float * x,
|
||||
}
|
||||
const float d = max / -8;
|
||||
if (d == 0) {
|
||||
scales[i] = ov::float16(1.0f);
|
||||
scales[gi] = ov::float16(1.0f);
|
||||
// i4 value 0 packed: 0x00
|
||||
memset(weights + i * qk / 2, 0, qk / 2);
|
||||
memset(weights + gi * qk / 2, 0, qk / 2);
|
||||
continue;
|
||||
}
|
||||
const float id = 1.0f / d;
|
||||
scales[i] = ov::float16(d);
|
||||
scales[gi] = ov::float16(d);
|
||||
for (int j = 0; j < qk / 2; ++j) {
|
||||
const float x0 = x[i * qk + 2 * j] * id;
|
||||
const float x1 = x[i * qk + 2 * j + 1] * id;
|
||||
// Signed i4: range [-8, 7]. Quantize as round(x*id), then pack as 4-bit two's complement.
|
||||
int8_t si0 = (int8_t) std::max(-8, std::min(7, (int) roundf(x0)));
|
||||
int8_t si1 = (int8_t) std::max(-8, std::min(7, (int) roundf(x1)));
|
||||
weights[i * qk / 2 + j] = (si0 & 0x0F) | ((si1 & 0x0F) << 4);
|
||||
weights[gi * qk / 2 + j] = (si0 & 0x0F) | ((si1 & 0x0F) << 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1019,7 +1056,7 @@ std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
|
||||
ov::Tensor & scales,
|
||||
ov::Tensor & zp,
|
||||
// Use an exact f16 zero point (vs. a rounded integer one); always
|
||||
// used for for_gather_matmul (3D MoE expert) weights regardless of
|
||||
// used for MoE expert weights regardless of
|
||||
// this flag, and also settable explicitly for test-backend-ops.
|
||||
bool use_bias = false) {
|
||||
// Create a temporary tensor for extraction functions that read from tensor->data
|
||||
@@ -1044,6 +1081,7 @@ std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
|
||||
weights_per_block = 32;
|
||||
break;
|
||||
case GGML_TYPE_Q8_0:
|
||||
case GGML_TYPE_Q5_0:
|
||||
case GGML_TYPE_Q5_1:
|
||||
case GGML_TYPE_Q5_K:
|
||||
is_u4 = false;
|
||||
@@ -1058,12 +1096,11 @@ std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
|
||||
std::string(ggml_type_name(tensor->type)));
|
||||
}
|
||||
|
||||
// 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point extraction
|
||||
// (see make_int8_weights/make_int4_weights) rather than the rounded integer zero point --
|
||||
// round(min/scale) error is what corrupts Q4_K/Q5_1 experts, and the f16-zp form still fuses
|
||||
// into GatherMatmulCompressed since it stays a Subtract, not an Add.
|
||||
// An exact f16 zero point avoids the round(min/scale) error of the integer one, which corrupts
|
||||
// Q4_K/Q5_1 MoE experts and costs accuracy on dense weights. It stays a Subtract, so the
|
||||
// weights still fuse into GatherMatmulCompressed / compressed FullyConnected.
|
||||
const bool for_gather_matmul = tensor->ne[2] > 1;
|
||||
use_bias = use_bias || for_gather_matmul;
|
||||
use_bias = use_bias || ggml_openvino_use_exact_zero_point(tensor);
|
||||
|
||||
// Extract quantized data
|
||||
switch (tensor->type) {
|
||||
@@ -1076,6 +1113,9 @@ std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
|
||||
case GGML_TYPE_Q4_K:
|
||||
extract_q4_k_data(&temp_tensor, weights, scales, zp, use_bias);
|
||||
break;
|
||||
case GGML_TYPE_Q5_0:
|
||||
extract_q5_0_data(&temp_tensor, weights, scales);
|
||||
break;
|
||||
case GGML_TYPE_Q5_1:
|
||||
extract_q5_1_data(&temp_tensor, weights, scales, zp, use_bias);
|
||||
break;
|
||||
@@ -1125,19 +1165,18 @@ std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor,
|
||||
bool is_u4 = (requant_type == ExtraQuantType::Q4_0_C || requant_type == ExtraQuantType::Q4_0_128 ||
|
||||
requant_type == ExtraQuantType::Q4_0_64 || requant_type == ExtraQuantType::Q4_1_64);
|
||||
|
||||
// Streaming dequant (opt-in via GGML_OPENVINO_REDUCE_COMPILE_MEM or
|
||||
// GGML_OPENVINO_MEMORY_OPTIMIZE): instead of
|
||||
// materializing the full n_elements F32 array (e.g. ~1 GB for token_embd), dequantize
|
||||
// a chunk of complete rows into a small scratch and quantize/convert it straight into
|
||||
// the output buffers, capping the transient F32 footprint at CHUNK_ROWS*ne0 floats.
|
||||
// Streaming dequant: instead of materializing the full n_elements F32 array (e.g. ~1 GB
|
||||
// for token_embd), dequantize a chunk of complete rows into a small scratch and
|
||||
// quantize/convert it straight into the output buffers, capping the transient F32
|
||||
// footprint at CHUNK_ROWS*ne0 floats, with identical output.
|
||||
//
|
||||
// Only valid (and only used) for the Q8_0_C / Q8_1_C / F16 targets whose block size
|
||||
// divides a row (channel-wise _C uses block_size == ne0) so no target block straddles
|
||||
// a row boundary, and Q8/F16 have no cross-block packing. The u4 (Q4_0) path packs two
|
||||
// weights per byte with running zp ORs that assume a single whole-array call, so it is
|
||||
// never streamed. When the flag is off, behavior is identical to the original
|
||||
// full-materialization path.
|
||||
const bool stream_requant = ggml_openvino_reduce_compile_mem_enabled() && !is_u4 &&
|
||||
// Only valid (and only used) when the target block size divides a row (channel-wise _C
|
||||
// uses block_size == ne0) so no target block straddles a row boundary. The u4 (Q4_0)
|
||||
// path packs two weights per byte and ORs zp nibbles in pairs, but both are indexed by
|
||||
// an absolute block index and chunks are emitted in increasing order, so whole-row
|
||||
// chunks stay byte-aligned and the even-block write always precedes the odd-block OR.
|
||||
// Q4_1_64 (asymmetric u4) has no chunked quantizer, so it always takes the full path.
|
||||
const bool stream_requant = requant_type != ExtraQuantType::Q4_1_64 &&
|
||||
!(block_size > 0 && ne0 % block_size != 0);
|
||||
|
||||
if (!stream_requant) {
|
||||
@@ -1178,7 +1217,9 @@ std::shared_ptr<ov::Node> requantize_to_buffers(const ggml_tensor * tensor,
|
||||
->from_float_ref(scratch.data(), f16_base + (r0 * ne0) * sizeof(uint16_t), elems);
|
||||
} else {
|
||||
const int64_t block_offset = (r0 * ne0) / block_size;
|
||||
if (requant_type == ExtraQuantType::Q8_1_C) {
|
||||
if (is_u4) {
|
||||
quantize_q4_0(scratch.data(), weights, scales, zp, elems, block_size, block_offset);
|
||||
} else if (requant_type == ExtraQuantType::Q8_1_C) {
|
||||
quantize_q8_1(scratch.data(), weights, scales, zp, elems, block_size, block_offset);
|
||||
} else {
|
||||
quantize_q8_0(scratch.data(), weights, scales, zp, elems, block_size, block_offset);
|
||||
@@ -1258,13 +1299,9 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo
|
||||
OPENVINO_THROW("Unsupported quantized type: ", ggml_type_name(tensor->type));
|
||||
}
|
||||
|
||||
// 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point path (see
|
||||
// extract_quantized_weights) -- must be kept in sync with the "use_bias || for_gather_matmul"
|
||||
// check in ggml_openvino_get_extracted_layout, which sizes/offsets the zp slot accordingly.
|
||||
// Requantized tensors (layout.is_requant) are handled by requantize_to_buffers instead, whose
|
||||
// zp sizing/type is unaffected by for_gather_matmul, so they are excluded here.
|
||||
const bool for_gather_matmul = tensor->ne[2] > 1;
|
||||
const bool zp_is_f16 = !layout.is_requant && (use_bias || for_gather_matmul);
|
||||
// Keep in sync with ggml_openvino_get_extracted_layout, which sizes the zp slot. Requantized
|
||||
// tensors are handled by requantize_to_buffers and keep an integer zero point.
|
||||
const bool zp_is_f16 = !layout.is_requant && (use_bias || ggml_openvino_use_exact_zero_point(tensor));
|
||||
|
||||
const bool is_3d_mxfp4_moe = tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1);
|
||||
if (is_3d_mxfp4_moe) {
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
#include "model-cache.h"
|
||||
|
||||
#include "device/npu_config.h"
|
||||
#include "ggml-backend-impl.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml-impl.h"
|
||||
#include "ggml-openvino-extra.h"
|
||||
#include "openvino/rt_info/weightless_caching_attributes.hpp"
|
||||
|
||||
#include <cerrno>
|
||||
#include <algorithm>
|
||||
@@ -13,10 +15,14 @@
|
||||
#include <iomanip>
|
||||
#include <limits>
|
||||
#include <sstream>
|
||||
#include <openvino/core/model.hpp>
|
||||
#include <openvino/core/version.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <string>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_WIN32)
|
||||
@@ -83,25 +89,14 @@ uint64_t weight_fingerprint(const ggml_tensor * t) {
|
||||
// fingerprinted once, deterministically.
|
||||
template <typename F>
|
||||
void for_each_weight(const ggml_cgraph * cgraph, F && fn) {
|
||||
std::vector<const ggml_tensor *> seen;
|
||||
std::unordered_set<const ggml_tensor *> seen;
|
||||
for (int i = 0; i < cgraph->n_nodes; ++i) {
|
||||
const ggml_tensor * node = cgraph->nodes[i];
|
||||
for (int s = 0; s < GGML_MAX_SRC; ++s) {
|
||||
const ggml_tensor * src = node->src[s];
|
||||
if (!is_weight_src(src)) {
|
||||
if (!is_weight_src(src) || !seen.insert(src).second) {
|
||||
continue;
|
||||
}
|
||||
bool dup = false;
|
||||
for (const auto * p : seen) {
|
||||
if (p == src) {
|
||||
dup = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (dup) {
|
||||
continue;
|
||||
}
|
||||
seen.push_back(src);
|
||||
fn(src);
|
||||
}
|
||||
}
|
||||
@@ -150,6 +145,12 @@ bool make_dirs(const std::string & path) {
|
||||
// Create each intermediate component (skip a leading "/" root).
|
||||
if (sep && acc.size() > 1) {
|
||||
std::string component = acc.substr(0, acc.size() - 1);
|
||||
#if defined(_WIN32)
|
||||
// A drive root such as "D:" cannot be mkdir'ed (EACCES, not EEXIST).
|
||||
if (component.size() == 2 && component[1] == ':') {
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
if (!make_dir(component)) {
|
||||
return false;
|
||||
}
|
||||
@@ -165,16 +166,26 @@ bool ggml_openvino_model_cache_only() {
|
||||
}
|
||||
|
||||
static const char * cache_settings[] = {
|
||||
"GGML_OPENVINO_GPU_CONFIG",
|
||||
"GGML_OPENVINO_CPU_CONFIG",
|
||||
"GGML_OPENVINO_REQUANT_KQUANT",
|
||||
"GGML_OPENVINO_NATIVE_SOFTPLUS",
|
||||
"GGML_OPENVINO_DISABLE_KV_SLICE",
|
||||
// GGML_OPENVINO_KV_SLICE only changes dynamic runtime bindings. Dynamic
|
||||
// attention graphs accept either shape and share the same compiled blob.
|
||||
"GGML_OPENVINO_GDN_NATIVE_STATE",
|
||||
"GGML_OPENVINO_GDN_DECOMPOSE",
|
||||
"GGML_OPENVINO_GDN_DECOMPOSE_CHUNK",
|
||||
"GGML_OPENVINO_MANUAL_GQA_ATTN",
|
||||
"GGML_OPENVINO_EXACT_ZERO_POINT",
|
||||
"GGML_OPENVINO_STATEFUL_EXECUTION",
|
||||
"GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT",
|
||||
"GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS",
|
||||
"GGML_OPENVINO_REDUCE_COMPILE_MEM",
|
||||
"GGML_OPENVINO_MEMORY_OPTIMIZE",
|
||||
"GGML_OPENVINO_PROFILING",
|
||||
"GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK",
|
||||
"GGML_OPENVINO_KV_SCATTER_ELEMENTS",
|
||||
"GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN",
|
||||
};
|
||||
|
||||
void ggml_openvino_model_cache_init() {
|
||||
@@ -203,17 +214,15 @@ void ggml_openvino_model_cache_init() {
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t ggml_openvino_source_fingerprint(const void * data, size_t size, std::vector<ggml_openvino_source_mapping> & mappings) {
|
||||
const ggml_openvino_source_mapping * ggml_openvino_find_source_mapping(const void * data, size_t size,
|
||||
std::vector<ggml_openvino_source_mapping> & mappings) {
|
||||
const uintptr_t address = reinterpret_cast<uintptr_t>(data);
|
||||
auto contains = [&](const ggml_openvino_source_mapping & m) {
|
||||
return address >= m.begin && address < m.end && size <= m.end - address;
|
||||
};
|
||||
auto fingerprint = [&](const ggml_openvino_source_mapping & m) {
|
||||
return fnv1a_u64(m.identity, m.offset + address - m.begin);
|
||||
};
|
||||
for (const auto & m : mappings) {
|
||||
if (contains(m)) {
|
||||
return fingerprint(m);
|
||||
return &m;
|
||||
}
|
||||
}
|
||||
#ifdef __linux__
|
||||
@@ -228,7 +237,7 @@ uint64_t ggml_openvino_source_fingerprint(const void * data, size_t size, std::v
|
||||
&offset, &dev_major, &dev_minor, &inode, &path_start) != 7 || inode == 0) {
|
||||
continue;
|
||||
}
|
||||
ggml_openvino_source_mapping m{uintptr_t(begin), uintptr_t(end), offset, FNV_OFFSET};
|
||||
ggml_openvino_source_mapping m{uintptr_t(begin), uintptr_t(end), offset, FNV_OFFSET, permissions[1] != 'w'};
|
||||
if (!contains(m)) {
|
||||
continue;
|
||||
}
|
||||
@@ -246,7 +255,7 @@ uint64_t ggml_openvino_source_fingerprint(const void * data, size_t size, std::v
|
||||
m.identity = fnv1a_u64(m.identity, st.st_ctim.tv_sec);
|
||||
m.identity = fnv1a_u64(m.identity, st.st_ctim.tv_nsec);
|
||||
mappings.push_back(m);
|
||||
return fingerprint(m);
|
||||
return &mappings.back();
|
||||
}
|
||||
#elif defined(_WIN32)
|
||||
MEMORY_BASIC_INFORMATION memory;
|
||||
@@ -276,8 +285,10 @@ uint64_t ggml_openvino_source_fingerprint(const void * data, size_t size, std::v
|
||||
const uint64_t file_size = (uint64_t(info.nFileSizeHigh) << 32) | info.nFileSizeLow;
|
||||
const uintptr_t begin = reinterpret_cast<uintptr_t>(memory.AllocationBase);
|
||||
if (file_size <= std::numeric_limits<uintptr_t>::max() - begin) {
|
||||
// A view created read-only can never be made writable.
|
||||
const bool read_only = (memory.AllocationProtect & (PAGE_READONLY | PAGE_EXECUTE_READ)) != 0;
|
||||
ggml_openvino_source_mapping m{begin, begin + static_cast<uintptr_t>(file_size), 0,
|
||||
fnv1a(FNV_OFFSET, "win32", 5)};
|
||||
fnv1a(FNV_OFFSET, "win32", 5), read_only};
|
||||
if (contains(m)) {
|
||||
m.identity = fnv1a_u64(m.identity, info.dwVolumeSerialNumber);
|
||||
m.identity = fnv1a_u64(m.identity, (uint64_t(info.nFileIndexHigh) << 32) | info.nFileIndexLow);
|
||||
@@ -286,7 +297,7 @@ uint64_t ggml_openvino_source_fingerprint(const void * data, size_t size, std::v
|
||||
info.ftLastWriteTime.dwLowDateTime);
|
||||
m.identity = fnv1a_u64(m.identity, static_cast<uint64_t>(basic.ChangeTime.QuadPart));
|
||||
mappings.push_back(m);
|
||||
return fingerprint(m);
|
||||
return &mappings.back();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -294,6 +305,13 @@ uint64_t ggml_openvino_source_fingerprint(const void * data, size_t size, std::v
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
uint64_t ggml_openvino_source_fingerprint(const void * data, size_t size, std::vector<ggml_openvino_source_mapping> & mappings) {
|
||||
if (const auto * m = ggml_openvino_find_source_mapping(data, size, mappings)) {
|
||||
return fnv1a_u64(m->identity, m->offset + reinterpret_cast<uintptr_t>(data) - m->begin);
|
||||
}
|
||||
if (ggml_openvino_model_cache_only()) {
|
||||
GGML_ABORT("ggml-openvino: could not identify mapped GGUF weight; use --load-mode mmap");
|
||||
}
|
||||
@@ -344,12 +362,19 @@ uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph,
|
||||
uint64_t extra_cfg,
|
||||
const std::string & graph_signature) {
|
||||
uint64_t h = FNV_OFFSET;
|
||||
h = fnv1a_u64(h, 2);
|
||||
// Version 3 includes dense Q8_0/Q5_1 requantization in q4_asym64_all.
|
||||
// Version 4 uses GatedDeltaNet output 1 for snapshots in snapshot mode; older
|
||||
// graphs use the incompatible three-output contract.
|
||||
// Version 5 requantizes K-quant MoE experts to Q4_1_64 on GPU by default.
|
||||
// Version 6 clears recurrent-cache slots with Select, so a NaN in a cleared slot does not survive.
|
||||
// Version 7 reads ROPE frequency factors named other than rope_freqs.weight as a Constant, not a Parameter.
|
||||
h = fnv1a_u64(h, 7);
|
||||
h = fnv1a(h, graph_signature.data(), graph_signature.size());
|
||||
for (const char * name : cache_settings) {
|
||||
const char * value = ggml_openvino_getenv_str(name, "");
|
||||
h = fnv1a(h, value, strlen(value) + 1);
|
||||
}
|
||||
h = fnv1a_u64(h, ggml_openvino_gdn_decompose_enabled() ? ggml_openvino_gdn_decompose_chunk() : 0);
|
||||
if (const char * debug_nodes = ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) {
|
||||
h = fnv1a(h, "GGML_OPENVINO_DEBUG_NODE", sizeof("GGML_OPENVINO_DEBUG_NODE"));
|
||||
h = fnv1a(h, debug_nodes, strlen(debug_nodes) + 1);
|
||||
@@ -361,10 +386,19 @@ uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph,
|
||||
// Topology: node count + each node's op and name (cheap, and distinguishes
|
||||
// graphs that share weights but differ structurally).
|
||||
h = fnv1a_u64(h, static_cast<uint64_t>(cgraph->n_nodes));
|
||||
bool has_gdn = false;
|
||||
for (int i = 0; i < cgraph->n_nodes; ++i) {
|
||||
const ggml_tensor * node = cgraph->nodes[i];
|
||||
h = fnv1a_u64(h, static_cast<uint64_t>(node->op));
|
||||
h = fnv1a(h, node->name, strlen(node->name));
|
||||
has_gdn = has_gdn || node->op == GGML_OP_GATED_DELTA_NET;
|
||||
}
|
||||
if (has_gdn && ggml_openvino_is_npu()) {
|
||||
// The GDN q pre-scale factor is a constant in the graph (0: no pre-scale).
|
||||
const float q_scale = ggml_openvino_gdn_prescale_enabled() ? ggml_openvino_gdn_q_scale() : 0.0f;
|
||||
h = fnv1a(h, &q_scale, sizeof(q_scale));
|
||||
// GDN graphs switch to the PLUGIN compiler unless the environment sets the compiler type.
|
||||
h = fnv1a_u64(h, ggml_openvino_npu_gdn_uses_plugin_compiler() ? 1u : 0u);
|
||||
}
|
||||
|
||||
// Weights: the model identity.
|
||||
@@ -482,3 +516,196 @@ bool ggml_openvino_model_cache_verify_manifest(const std::string & path,
|
||||
f >> std::ws;
|
||||
return f.eof();
|
||||
}
|
||||
|
||||
std::string ggml_openvino_model_cache_weights_path(const std::string & dir, uint64_t fp_a, uint64_t fp_b) {
|
||||
return dir + "/" + hex64(fp_a) + "-" + hex64(fp_b) + ".weights.bin";
|
||||
}
|
||||
|
||||
static uint64_t weights_hash_mix(uint64_t h, uint64_t v) {
|
||||
h ^= v + 0x9e3779b97f4a7c15ULL + (h << 6) + (h >> 2);
|
||||
return h * 0x100000001b3ULL;
|
||||
}
|
||||
|
||||
static uint64_t weights_hash_bytes(uint64_t h, const void * data, size_t size) {
|
||||
const auto * p = static_cast<const unsigned char *>(data);
|
||||
size_t i = 0;
|
||||
for (; i + 8 <= size; i += 8) {
|
||||
uint64_t w;
|
||||
std::memcpy(&w, p + i, 8);
|
||||
h = (h ^ w) * 0x100000001b3ULL;
|
||||
}
|
||||
for (; i < size; ++i) {
|
||||
h = (h ^ p[i]) * 0x100000001b3ULL;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
// Content hash recorded in a weights file's size record, or false if it has none.
|
||||
static bool read_weights_hash(const std::string & path, uint64_t & hash) {
|
||||
std::ifstream meta(path + ".meta");
|
||||
std::string key, value;
|
||||
while (meta >> key >> value) {
|
||||
if (key == "hash") {
|
||||
hash = std::stoull(value, nullptr, 16);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::shared_ptr<ggml_openvino_weights_layout> ggml_openvino_model_cache_write_weights(
|
||||
const std::string & path,
|
||||
const std::vector<std::shared_ptr<ov::Model>> & models,
|
||||
bool keep_existing,
|
||||
ggml_openvino_weights_status * status) {
|
||||
struct region {
|
||||
const void * data;
|
||||
size_t size;
|
||||
size_t offset;
|
||||
};
|
||||
constexpr size_t alignment = 64;
|
||||
auto layout = std::make_shared<ggml_openvino_weights_layout>();
|
||||
layout->path = path;
|
||||
std::vector<region> regions;
|
||||
std::unordered_map<const void *, size_t> region_of;
|
||||
for (const auto & model : models) {
|
||||
if (!model) {
|
||||
continue;
|
||||
}
|
||||
for (const auto & node : model->get_ordered_ops()) {
|
||||
auto cnst = ov::as_type_ptr<ov::op::v0::Constant>(node);
|
||||
if (!cnst || cnst->get_rt_info().count(ov::WeightlessCacheAttribute::get_type_info_static()) == 0 ||
|
||||
cnst->get_data_ptr() == nullptr || cnst->get_byte_size() == 0) {
|
||||
continue;
|
||||
}
|
||||
layout->constants.push_back(cnst);
|
||||
auto [it, inserted] = region_of.emplace(cnst->get_data_ptr(), regions.size());
|
||||
if (inserted) {
|
||||
regions.push_back({ cnst->get_data_ptr(), cnst->get_byte_size(), 0 });
|
||||
} else {
|
||||
regions[it->second].size = std::max(regions[it->second].size, cnst->get_byte_size());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size_t total = 0;
|
||||
for (auto & r : regions) {
|
||||
r.offset = total;
|
||||
total = (total + r.size + alignment - 1) / alignment * alignment;
|
||||
layout->regions[r.data] = { r.offset, r.size };
|
||||
}
|
||||
for (const auto & cnst : layout->constants) {
|
||||
cnst->get_rt_info()[ov::WeightlessCacheAttribute::get_type_info_static()] = ov::WeightlessCacheAttribute(
|
||||
cnst->get_byte_size(), layout->regions.at(cnst->get_data_ptr()).first, cnst->get_element_type());
|
||||
}
|
||||
|
||||
uint64_t content_hash = weights_hash_mix(0xcbf29ce484222325ULL, total);
|
||||
for (const auto & r : regions) {
|
||||
content_hash = weights_hash_mix(content_hash, r.offset);
|
||||
content_hash = weights_hash_mix(content_hash, r.size);
|
||||
content_hash = weights_hash_bytes(content_hash, r.data, r.size);
|
||||
}
|
||||
|
||||
if (keep_existing && ggml_openvino_model_cache_weights_valid(path)) {
|
||||
uint64_t existing_hash = 0;
|
||||
if (read_weights_hash(path, existing_hash) && existing_hash == content_hash) {
|
||||
if (status) {
|
||||
*status = ggml_openvino_weights_status::reused;
|
||||
}
|
||||
GGML_LOG_INFO("ggml-openvino: model cache REUSED %s (%zu regions, %.1f MiB)\n", path.c_str(),
|
||||
regions.size(), total / 1024.0 / 1024.0);
|
||||
return layout;
|
||||
}
|
||||
if (status) {
|
||||
*status = ggml_openvino_weights_status::mismatch;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
if (status) {
|
||||
*status = ggml_openvino_weights_status::written;
|
||||
}
|
||||
|
||||
// Publish the data file first and its size record last, so a valid record always describes a complete file.
|
||||
const std::string data_tmp = path + ".tmp";
|
||||
const std::string meta_path = path + ".meta";
|
||||
const std::string meta_tmp = meta_path + ".tmp";
|
||||
{
|
||||
std::ofstream f(data_tmp, std::ios::binary | std::ios::trunc);
|
||||
if (!f.is_open()) {
|
||||
return nullptr;
|
||||
}
|
||||
static const char zeros[alignment] = {};
|
||||
size_t pos = 0;
|
||||
for (const auto & r : regions) {
|
||||
f.write(zeros, static_cast<std::streamsize>(r.offset - pos));
|
||||
f.write(static_cast<const char *>(r.data), static_cast<std::streamsize>(r.size));
|
||||
pos = r.offset + r.size;
|
||||
}
|
||||
f.write(zeros, static_cast<std::streamsize>(total - pos));
|
||||
f.close();
|
||||
if (!f.good()) {
|
||||
std::remove(data_tmp.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
{
|
||||
std::ofstream f(meta_tmp, std::ios::trunc);
|
||||
f << "size " << total << "\nregions " << regions.size() << "\nhash " << hex64(content_hash) << "\n";
|
||||
f.close();
|
||||
if (!f.good()) {
|
||||
std::remove(data_tmp.c_str());
|
||||
std::remove(meta_tmp.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
// std::rename does not replace an existing file on Windows.
|
||||
ggml_openvino_model_cache_remove_weights(path);
|
||||
if (std::rename(data_tmp.c_str(), path.c_str()) != 0 || std::rename(meta_tmp.c_str(), meta_path.c_str()) != 0) {
|
||||
std::remove(data_tmp.c_str());
|
||||
std::remove(meta_tmp.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
GGML_LOG_INFO("ggml-openvino: model cache WROTE %s (%zu regions, %.1f MiB)\n", path.c_str(), regions.size(),
|
||||
total / 1024.0 / 1024.0);
|
||||
return layout;
|
||||
}
|
||||
|
||||
size_t ggml_openvino_model_cache_retag_weights(const std::shared_ptr<ov::Model> & model,
|
||||
const ggml_openvino_weights_layout & layout) {
|
||||
size_t resolved = 0;
|
||||
for (const auto & node : model->get_ordered_ops()) {
|
||||
auto cnst = ov::as_type_ptr<ov::op::v0::Constant>(node);
|
||||
if (!cnst) {
|
||||
continue;
|
||||
}
|
||||
auto & rt_info = cnst->get_rt_info();
|
||||
auto attr = rt_info.find(ov::WeightlessCacheAttribute::get_type_info_static());
|
||||
if (attr == rt_info.end()) {
|
||||
continue;
|
||||
}
|
||||
auto it = layout.regions.find(cnst->get_data_ptr());
|
||||
if (it != layout.regions.end() && cnst->get_byte_size() <= it->second.second) {
|
||||
attr->second = ov::WeightlessCacheAttribute(cnst->get_byte_size(), it->second.first, cnst->get_element_type());
|
||||
++resolved;
|
||||
} else {
|
||||
rt_info.erase(attr);
|
||||
}
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
bool ggml_openvino_model_cache_weights_valid(const std::string & path) {
|
||||
std::ifstream meta(path + ".meta");
|
||||
std::string key;
|
||||
uint64_t size = 0;
|
||||
if (!(meta >> key >> size) || key != "size") {
|
||||
return false;
|
||||
}
|
||||
std::ifstream f(path, std::ios::binary | std::ios::ate);
|
||||
return f.is_open() && static_cast<uint64_t>(f.tellg()) == size;
|
||||
}
|
||||
|
||||
void ggml_openvino_model_cache_remove_weights(const std::string & path) {
|
||||
std::remove((path + ".meta").c_str());
|
||||
std::remove(path.c_str());
|
||||
}
|
||||
|
||||
@@ -1,13 +1,36 @@
|
||||
#pragma once
|
||||
|
||||
// Compiled blobs include weights. Cache-only execution skips weight uploads and graph compilation.
|
||||
// On NPU, GGML_OPENVINO_SELF_CONTAINED_BLOB=1 exports embedded-weight prefill/decode
|
||||
// blobs. A warm hit skips weight requantization, graph conversion, and compilation,
|
||||
// at the cost of roughly model-size storage per blob and separate NPUW weight banks.
|
||||
// After both blobs exist, GGML_OPENVINO_SELF_CONTAINED_MMAP=1 additionally wraps the
|
||||
// GGUF mmap instead of allocating the private OpenVINO model buffer. This mode is
|
||||
// import-only and fails clearly if either required blob is missing or invalid.
|
||||
// GGML_OPENVINO_NPU_WEIGHTLESS_BANK=1 is the same raw-weight flow, but exports weightless
|
||||
// prefill/decode blobs plus ONE <weights_fp>-<weights_fp>.weights.bin shared by every prefill/decode pair
|
||||
// of the model (pairs differ only in KV window). Warm import reads the weights from that file
|
||||
// (ov::weights_path) and NPUW deduplicates all models into one bank
|
||||
// (no OpenVINO patch needed), so the weights are stored once on disk and resident once.
|
||||
|
||||
#include "ggml.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace ov {
|
||||
class Model;
|
||||
namespace op {
|
||||
namespace v0 {
|
||||
class Constant;
|
||||
}
|
||||
} // namespace op
|
||||
} // namespace ov
|
||||
|
||||
bool ggml_openvino_model_cache_only();
|
||||
void ggml_openvino_model_cache_init();
|
||||
|
||||
@@ -16,8 +39,13 @@ struct ggml_openvino_source_mapping {
|
||||
uintptr_t end;
|
||||
uint64_t offset;
|
||||
uint64_t identity;
|
||||
// The mapping cannot be written, so its pages always hold the file's bytes.
|
||||
bool read_only;
|
||||
};
|
||||
|
||||
// Find the file mapping that holds [data, data + size), or nullptr. Cache mappings for one buffer lifetime.
|
||||
const ggml_openvino_source_mapping * ggml_openvino_find_source_mapping(const void * data, size_t size,
|
||||
std::vector<ggml_openvino_source_mapping> & mappings);
|
||||
// Identify mmap weights without reading their pages. Cache mappings for one buffer lifetime.
|
||||
uint64_t ggml_openvino_source_fingerprint(const void * data, size_t size, std::vector<ggml_openvino_source_mapping> & mappings);
|
||||
uint64_t ggml_backend_openvino_weight_fingerprint(const ggml_tensor * tensor);
|
||||
@@ -55,3 +83,39 @@ bool ggml_openvino_model_cache_verify_manifest(const std::string & path,
|
||||
uint64_t fingerprint,
|
||||
std::vector<std::string> & inputs,
|
||||
std::vector<std::string> & outputs);
|
||||
|
||||
// GGML_OPENVINO_NPU_WEIGHTLESS_BANK: a weights file shared by weightless blobs. Every weightless-tagged
|
||||
// Constant is one region keyed by its data pointer (NPUW maps a Constant to its file offset by pointer).
|
||||
struct ggml_openvino_weights_layout {
|
||||
std::string path;
|
||||
// data pointer -> (offset, size) of its region in the file.
|
||||
std::unordered_map<const void *, std::pair<size_t, size_t>> regions;
|
||||
// Keeps the laid-out Constants alive so no other Constant can reuse one of their data pointers.
|
||||
std::vector<std::shared_ptr<ov::op::v0::Constant>> constants;
|
||||
};
|
||||
|
||||
// <dir>/<a>-<b>.weights.bin (NPUW's weightless import needs the .bin extension).
|
||||
std::string ggml_openvino_model_cache_weights_path(const std::string & dir, uint64_t fp_a, uint64_t fp_b);
|
||||
|
||||
enum class ggml_openvino_weights_status { written, reused, mismatch };
|
||||
|
||||
// Lay out all weightless-tagged Constants of |models| in one file, retag their WeightlessCacheAttribute with the
|
||||
// real offsets and write the file. Call before compiling. Returns nullptr on I/O error.
|
||||
// With |keep_existing|, a valid file at |path| is never rewritten: it is reused if its layout and content hash
|
||||
// match (status reused), otherwise nullptr is returned (status mismatch).
|
||||
std::shared_ptr<ggml_openvino_weights_layout> ggml_openvino_model_cache_write_weights(
|
||||
const std::string & path,
|
||||
const std::vector<std::shared_ptr<ov::Model>> & models,
|
||||
bool keep_existing = false,
|
||||
ggml_openvino_weights_status * status = nullptr);
|
||||
|
||||
// Point the tagged Constants of |model| that live in |layout| at their regions and untag the rest, which NPUW
|
||||
// then embeds in the blob. Returns the number of Constants resolved through the file.
|
||||
size_t ggml_openvino_model_cache_retag_weights(const std::shared_ptr<ov::Model> & model,
|
||||
const ggml_openvino_weights_layout & layout);
|
||||
|
||||
// True if the weights file is complete (its size matches the record written after it).
|
||||
bool ggml_openvino_model_cache_weights_valid(const std::string & path);
|
||||
|
||||
// Remove a weights file and its size record.
|
||||
void ggml_openvino_model_cache_remove_weights(const std::string & path);
|
||||
|
||||
@@ -23,6 +23,9 @@ struct ModelExtraInputInfo {
|
||||
Shape shape;
|
||||
int64_t value;
|
||||
bool is_parameter;
|
||||
// when set, the value is read from dimension shape_axis of this model input instead of a Parameter
|
||||
std::string shape_source;
|
||||
int64_t shape_axis = -1;
|
||||
};
|
||||
|
||||
class GgmlDecoder : public DecoderBase {
|
||||
@@ -121,6 +124,8 @@ public:
|
||||
|
||||
virtual bool is_stateful() const = 0;
|
||||
|
||||
virtual bool is_prefill() const = 0;
|
||||
|
||||
virtual bool is_splited_model() const = 0;
|
||||
|
||||
virtual int is_swa_layer(int layer) const = 0;
|
||||
|
||||
@@ -162,6 +162,8 @@ public:
|
||||
|
||||
bool is_stateful() const { return m_decoder->is_stateful(); }
|
||||
|
||||
bool is_prefill() const { return m_decoder->is_prefill(); }
|
||||
|
||||
int get_ssm_state_size() const { return m_decoder->get_ssm_state_size(); }
|
||||
|
||||
private:
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
#include "../node_context.h"
|
||||
#include "../op_table.h"
|
||||
#include "../utils.h"
|
||||
|
||||
#include <cstring>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <vector>
|
||||
|
||||
namespace ov::frontend::ggml::op {
|
||||
OutputVector translate_arange(const NodeContext & context) {
|
||||
num_inputs_check(context, 0, 0);
|
||||
float params[3];
|
||||
memcpy(params, context.get_output_op_params(), sizeof(params));
|
||||
// GGML ARANGE has fixed op parameters, including for rollback slot order.
|
||||
std::vector<float> values(ov::shape_size(context.get_output_shape().to_shape()));
|
||||
for (size_t i = 0; i < values.size(); ++i) {
|
||||
values[i] = params[0] + params[2] * i;
|
||||
}
|
||||
auto result = ov::op::v0::Constant::create(context.get_output_type(), context.get_output_shape().to_shape(), values);
|
||||
return rename_outputs_with_suffix({result}, context.get_name());
|
||||
}
|
||||
} // namespace ov::frontend::ggml::op
|
||||
@@ -2,9 +2,16 @@
|
||||
#include "../op_table.h"
|
||||
#include "../utils.h"
|
||||
#include "ggml.h"
|
||||
#include "ggml-openvino/ggml-openvino-extra.h"
|
||||
|
||||
#include <climits>
|
||||
#include <cstdint>
|
||||
#include <openvino/frontend/exception.hpp>
|
||||
#include <openvino/op/add.hpp>
|
||||
#include <openvino/op/concat.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/gather_elements.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/squeeze.hpp>
|
||||
#include <openvino/op/topk.hpp>
|
||||
|
||||
@@ -45,6 +52,52 @@ OutputVector translate_argsort(const NodeContext & context) {
|
||||
return rename_outputs_with_suffix({topk->output(1)}, context.get_name());
|
||||
}
|
||||
|
||||
OutputVector translate_top_k(const NodeContext & context) {
|
||||
num_inputs_check(context, 1, 1);
|
||||
auto input = process_view_input_new(context, 0);
|
||||
const int64_t k = context.get_output_shape()[3].get_length();
|
||||
const auto shape = input.get_partial_shape();
|
||||
if (ggml_openvino_is_gpu() && shape.rank() == 4 && shape[3].is_static() &&
|
||||
shape[3].get_length() > 65535 && shape[3].get_length() <= INT32_MAX && k <= 64) {
|
||||
const int64_t width = shape[3].get_length();
|
||||
int64_t chunk_size = 4096;
|
||||
while (chunk_size >= 1024 && width % chunk_size != 0) {
|
||||
--chunk_size;
|
||||
}
|
||||
if (chunk_size >= 1024) {
|
||||
const int64_t chunks = width / chunk_size;
|
||||
auto chunked = std::make_shared<ov::op::v1::Reshape>(input,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{-1, 1, chunks, chunk_size}), false);
|
||||
// Each global top-k candidate is in its chunk's top-k set.
|
||||
// Value sorting selects the parallel GPU kernel; ggml allows any index order.
|
||||
auto local = std::make_shared<ov::op::v11::TopK>(chunked,
|
||||
ov::op::v0::Constant::create(ov::element::i32, {}, {k}), -1,
|
||||
ov::op::v11::TopK::Mode::MAX, ov::op::v11::TopK::SortType::SORT_VALUES, ov::element::i32, false);
|
||||
std::vector<int32_t> offsets(chunks);
|
||||
for (int64_t i = 0; i < chunks; ++i) {
|
||||
offsets[i] = i * chunk_size;
|
||||
}
|
||||
auto indices = std::make_shared<ov::op::v1::Add>(local->output(1),
|
||||
ov::op::v0::Constant::create(ov::element::i32, {1, 1, size_t(chunks), 1}, offsets));
|
||||
auto candidate_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, -1, chunks * k});
|
||||
auto values = std::make_shared<ov::op::v1::Reshape>(local->output(0), candidate_shape, false);
|
||||
auto candidate_indices = std::make_shared<ov::op::v1::Reshape>(indices, candidate_shape, false);
|
||||
auto selected = std::make_shared<ov::op::v11::TopK>(values,
|
||||
ov::op::v0::Constant::create(ov::element::i32, {}, {k}), -1,
|
||||
ov::op::v11::TopK::Mode::MAX, ov::op::v11::TopK::SortType::SORT_VALUES, ov::element::i32, false);
|
||||
auto result = std::make_shared<ov::op::v6::GatherElements>(candidate_indices, selected->output(1), 3);
|
||||
auto output_shape = std::make_shared<ov::op::v0::Concat>(OutputVector{
|
||||
get_dimensions(input.get_node_shared_ptr(), {0, 1, 2}),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {k})}, 0);
|
||||
return rename_outputs_with_suffix({std::make_shared<ov::op::v1::Reshape>(result, output_shape, false)}, context.get_name());
|
||||
}
|
||||
}
|
||||
auto topk = std::make_shared<ov::op::v11::TopK>(
|
||||
input, ov::op::v0::Constant::create(ov::element::i32, {}, {k}), -1,
|
||||
ov::op::v11::TopK::Mode::MAX, ov::op::v11::TopK::SortType::NONE, ov::element::i32, false);
|
||||
return rename_outputs_with_suffix({topk->output(1)}, context.get_name());
|
||||
}
|
||||
|
||||
} // namespace op
|
||||
} // namespace ggml
|
||||
} // namespace frontend
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
#include "../node_context.h"
|
||||
#include "../op_table.h"
|
||||
#include "../utils.h"
|
||||
|
||||
#include <climits>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
@@ -12,7 +11,9 @@
|
||||
#include <openvino/op/concat.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/convert.hpp>
|
||||
#include <openvino/op/divide.hpp>
|
||||
#include <openvino/op/gather.hpp>
|
||||
#include <openvino/op/mod.hpp>
|
||||
#include <openvino/op/multiply.hpp>
|
||||
#include <openvino/op/negative.hpp>
|
||||
#include <openvino/op/range.hpp>
|
||||
@@ -22,6 +23,7 @@
|
||||
#include <openvino/op/slice.hpp>
|
||||
#include <openvino/op/squeeze.hpp>
|
||||
#include <openvino/op/subtract.hpp>
|
||||
#include <openvino/op/unsqueeze.hpp>
|
||||
#include <vector>
|
||||
|
||||
namespace ov {
|
||||
@@ -34,6 +36,45 @@ OutputVector translate_cpy(const NodeContext & context) {
|
||||
auto input_shape = context.get_input_shape(0);
|
||||
auto output_shape = context.get_input_shape(1);
|
||||
|
||||
if (op_case == 11 || op_case == 12) {
|
||||
auto base = context.get_input(1);
|
||||
auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});
|
||||
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
|
||||
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
|
||||
auto feature = ov::op::v0::Constant::create(ov::element::i64, {4},
|
||||
std::vector<int64_t>{1, 1, -1, output_shape[3].get_length()});
|
||||
ov::Output<ov::Node> state_part = context.get_input(0);
|
||||
if (op_case == 11) {
|
||||
auto source_rows = std::make_shared<ov::op::v8::Gather>(
|
||||
std::make_shared<ov::op::v3::ShapeOf>(context.get_input(0), ov::element::i64), axis,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
|
||||
auto source_indices = std::make_shared<ov::op::v4::Range>(
|
||||
std::make_shared<ov::op::v0::Squeeze>(context.get_input("rs_src_begin_" + context.get_name())),
|
||||
std::make_shared<ov::op::v0::Squeeze>(source_rows),
|
||||
std::make_shared<ov::op::v0::Squeeze>(one), ov::element::i64);
|
||||
state_part = std::make_shared<ov::op::v8::Gather>(context.get_input(0), source_indices,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {}, {2}));
|
||||
}
|
||||
auto states = std::make_shared<ov::op::v1::Reshape>(state_part, feature, false);
|
||||
auto rows =
|
||||
std::make_shared<ov::op::v8::Gather>(std::make_shared<ov::op::v3::ShapeOf>(states, ov::element::i64), axis,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
|
||||
auto indices = std::make_shared<ov::op::v4::Range>(
|
||||
std::make_shared<ov::op::v0::Squeeze>(zero), std::make_shared<ov::op::v0::Squeeze>(rows),
|
||||
std::make_shared<ov::op::v0::Squeeze>(one), ov::element::i64);
|
||||
auto seq_count = context.get_input("rs_seq_count_" + context.get_name());
|
||||
auto plane = std::make_shared<ov::op::v1::Divide>(indices, seq_count);
|
||||
auto sequence = std::make_shared<ov::op::v1::Mod>(indices, seq_count);
|
||||
auto slots = std::make_shared<ov::op::v1::Add>(
|
||||
std::make_shared<ov::op::v1::Multiply>(plane, context.get_input("rs_snapshot_stride_" + context.get_name())),
|
||||
std::make_shared<ov::op::v1::Add>(sequence, context.get_input("rs_slot_begin_" + context.get_name())));
|
||||
auto updated = std::make_shared<ov::op::v3::ScatterUpdate>(
|
||||
std::make_shared<ov::op::v1::Reshape>(base, feature, false), slots, states, axis);
|
||||
auto result = std::make_shared<ov::op::v1::Reshape>(
|
||||
updated, std::make_shared<ov::op::v3::ShapeOf>(base, ov::element::i64), false);
|
||||
return rename_outputs_with_suffix({result}, context.get_name());
|
||||
}
|
||||
|
||||
if (op_case == 4) {
|
||||
auto src = process_view_input_new(context, 0);
|
||||
auto base = context.get_input(1);
|
||||
@@ -73,10 +114,9 @@ OutputVector translate_cpy(const NodeContext & context) {
|
||||
// runtime input, so the cached model works for any kv head and active sequence count. The
|
||||
// result is the full updated cache.
|
||||
// op_case 1: gated-delta-net state, op_case 2: conv state, op_case 3: defrag remainder.
|
||||
if (op_case == 3) {
|
||||
// With -np 1 (and generally whenever there is no defrag remainder) this GET_ROWS gathers
|
||||
// zero rows: nothing to write back, and the cache is unchanged. NPU rejects zero-size
|
||||
// tensors, so short-circuit instead of building a degenerate Slice/Concat chain.
|
||||
if (op_case == 3 && context.is_static()) {
|
||||
// A statically empty remainder leaves the cache unchanged at any slot capacity.
|
||||
// NPU rejects zero-size tensors, so avoid a degenerate Slice/Concat chain.
|
||||
bool is_empty = false;
|
||||
if (input_shape.rank().is_static()) {
|
||||
for (const auto & d : input_shape) {
|
||||
@@ -90,24 +130,13 @@ OutputVector translate_cpy(const NodeContext & context) {
|
||||
return {context.get_input(1)};
|
||||
}
|
||||
}
|
||||
// op_case 7/8/9 are the single-slot variants; op_case 10 writes native GDN state into a
|
||||
// multi-slot cache without rollback snapshots.
|
||||
const bool single_slot_assign = op_case >= 7 && op_case <= 9;
|
||||
const bool direct_gdn_state = op_case == 7 || op_case == 10;
|
||||
int writeback_case = op_case;
|
||||
if (op_case == 10) {
|
||||
writeback_case = 1;
|
||||
} else if (single_slot_assign) {
|
||||
writeback_case = op_case - 6;
|
||||
}
|
||||
// op_case 10 consumes native GDN state without rollback snapshots.
|
||||
const bool direct_gdn_state = op_case == 10;
|
||||
const int writeback_case = direct_gdn_state ? 1 : op_case;
|
||||
const std::string slot_begin_name = "rs_slot_begin_" + context.get_name();
|
||||
const bool slice_assign = writeback_case >= 1 && writeback_case <= 3 &&
|
||||
(single_slot_assign || context.has_input(slot_begin_name));
|
||||
context.has_input(slot_begin_name);
|
||||
if (slice_assign) {
|
||||
if (single_slot_assign && writeback_case == 3) {
|
||||
return {context.get_input(1)};
|
||||
}
|
||||
|
||||
const int64_t slot_axis = 2;
|
||||
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
|
||||
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
|
||||
@@ -117,15 +146,12 @@ OutputVector translate_cpy(const NodeContext & context) {
|
||||
std::vector<int64_t>{1, 1, -1, output_shape[3].get_length()});
|
||||
|
||||
ov::Output<ov::Node> src;
|
||||
ov::Output<ov::Node> begin;
|
||||
if (!single_slot_assign) {
|
||||
begin = context.get_input(slot_begin_name);
|
||||
}
|
||||
auto begin = context.get_input(slot_begin_name);
|
||||
auto base = context.get_input(1);
|
||||
if (writeback_case == 1) {
|
||||
if (direct_gdn_state) {
|
||||
// Non-rollback GDN publishes state directly as [active_slots, heads, value_dim,
|
||||
// key_dim]. Flatten each active slot before replacing or updating the cache.
|
||||
// key_dim]. Flatten each active slot before updating the cache.
|
||||
src = std::make_shared<ov::op::v1::Reshape>(context.get_input(0), feature, false);
|
||||
} else {
|
||||
// Multi-slot rollback still consumes GGML's packed [attention | state snapshots]
|
||||
@@ -145,9 +171,12 @@ OutputVector translate_cpy(const NodeContext & context) {
|
||||
const std::string src_begin_name = "rs_src_begin_" + context.get_name();
|
||||
if (context.has_input(src_begin_name)) {
|
||||
auto src_begin = context.get_input(src_begin_name);
|
||||
auto src_end = std::make_shared<ov::op::v1::Add>(
|
||||
src_begin, ov::op::v0::Constant::create(ov::element::i64, {1}, {window_size}));
|
||||
window = std::make_shared<ov::op::v8::Slice>(context.get_input(0), src_begin, src_end, one, col_axis);
|
||||
std::vector<int64_t> offsets(window_size);
|
||||
std::iota(offsets.begin(), offsets.end(), 0);
|
||||
auto indices = std::make_shared<ov::op::v1::Add>(
|
||||
src_begin, ov::op::v0::Constant::create(ov::element::i64, {size_t(window_size)}, offsets));
|
||||
window = std::make_shared<ov::op::v8::Gather>(context.get_input(0), indices,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {}, {3}));
|
||||
} else if (context.has_input("chunk_valid_len")) {
|
||||
std::vector<int64_t> offsets(window_size);
|
||||
std::iota(offsets.begin(), offsets.end(), 0);
|
||||
@@ -181,20 +210,16 @@ OutputVector translate_cpy(const NodeContext & context) {
|
||||
FRONT_END_OP_CONVERSION_CHECK(feature_begin >= 0 && feature_end <= full_feature_size,
|
||||
"CPY conv state cache update feature range is out of bounds");
|
||||
|
||||
auto partial_feature = ov::op::v0::Constant::create(
|
||||
ov::element::i64, {4}, std::vector<int64_t>{1, 1, -1, update_feature_size});
|
||||
auto partial_feature = ov::op::v0::Constant::create(ov::element::i64, {4},
|
||||
std::vector<int64_t>{1, 1, -1, update_feature_size});
|
||||
src = std::make_shared<ov::op::v1::Reshape>(window, partial_feature, false);
|
||||
if (src.get_element_type() != context.get_output_type()) {
|
||||
src = std::make_shared<ov::op::v0::Convert>(src, context.get_output_type());
|
||||
}
|
||||
|
||||
if (single_slot_assign) {
|
||||
return rename_outputs_with_suffix({src}, context.get_name());
|
||||
}
|
||||
|
||||
auto src_len = std::make_shared<ov::op::v8::Gather>(
|
||||
std::make_shared<ov::op::v3::ShapeOf>(src, ov::element::i64), axis,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
|
||||
auto src_len =
|
||||
std::make_shared<ov::op::v8::Gather>(std::make_shared<ov::op::v3::ShapeOf>(src, ov::element::i64), axis,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
|
||||
auto slot_end = std::make_shared<ov::op::v1::Add>(begin, src_len);
|
||||
auto active_slots = std::make_shared<ov::op::v8::Slice>(base, begin, slot_end, one, axis);
|
||||
|
||||
@@ -209,16 +234,26 @@ OutputVector translate_cpy(const NodeContext & context) {
|
||||
} else {
|
||||
// op_case 3: gathered remainder rows already have the cache slot layout [1, 1, extra, feature]
|
||||
src = context.get_input(0);
|
||||
if (context.get_view_input_size(0) > 0 && !context.is_static()) {
|
||||
// build_rs gathers active and extra rows together. The remainder boundary moves
|
||||
// with the active sequences, so do not capture the first batch's VIEW offset.
|
||||
|
||||
// gathered rows can be [1, rows, feature]; add the slot axis
|
||||
const auto src_rank = src.get_partial_shape().rank();
|
||||
if (src_rank.is_static() && src_rank.get_length() == 3) {
|
||||
src = std::make_shared<ov::op::v0::Unsqueeze>(src, zero);
|
||||
}
|
||||
src = std::make_shared<ov::op::v8::Slice>(src,
|
||||
context.get_input("rs_src_begin_" + context.get_name()), int_max, one, axis);
|
||||
} else {
|
||||
src = process_view_input_new(context, 0);
|
||||
}
|
||||
}
|
||||
|
||||
if (src.get_element_type() != context.get_output_type()) {
|
||||
src = std::make_shared<ov::op::v0::Convert>(src, context.get_output_type());
|
||||
}
|
||||
|
||||
if (single_slot_assign) {
|
||||
return rename_outputs_with_suffix({src}, context.get_name());
|
||||
}
|
||||
|
||||
auto src_len =
|
||||
std::make_shared<ov::op::v8::Gather>(std::make_shared<ov::op::v3::ShapeOf>(src, ov::element::i64), axis,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
|
||||
|
||||
@@ -66,6 +66,10 @@ ov::Output<ov::Node> repeat_input_to_match(const NodeContext & context,
|
||||
const int64_t input_dim = input_shape[axis].get_length();
|
||||
const int64_t target_dim = target_shape[axis].get_length();
|
||||
|
||||
// Empty token axes can occur in a MoE warmup with no logits.
|
||||
if (input_dim == target_dim) {
|
||||
continue;
|
||||
}
|
||||
FRONT_END_OP_CONVERSION_CHECK(input_dim > 0 && target_dim > 0 && target_dim % input_dim == 0,
|
||||
"DIV input shape ", input_shape, " cannot repeat to match ", target_shape);
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
#include <openvino/op/broadcast.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/shape_of.hpp>
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
@@ -15,14 +16,16 @@ namespace op {
|
||||
OutputVector translate_fill(const NodeContext & context) {
|
||||
num_inputs_check(context, 1, 1);
|
||||
|
||||
if (is_recurrent_cache_reset_case(context)) {
|
||||
return translate_recurrent_cache_reset(context);
|
||||
}
|
||||
|
||||
float c;
|
||||
memcpy(&c, context.get_output_op_params(), sizeof(float));
|
||||
|
||||
auto shape = context.get_input_shape(0).to_shape();
|
||||
|
||||
// the shape of the input, which keeps its dynamic token dim
|
||||
auto target_shape = std::make_shared<ov::op::v3::ShapeOf>(process_view_input_new(context, 0), ov::element::i64);
|
||||
auto val = ov::op::v0::Constant::create(context.get_output_type(), {}, {c});
|
||||
auto target_shape = ov::op::v0::Constant::create(ov::element::i64, {shape.size()},
|
||||
std::vector<int64_t>(shape.begin(), shape.end()));
|
||||
auto res = std::make_shared<ov::op::v3::Broadcast>(val, target_shape);
|
||||
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
|
||||
@@ -12,19 +12,69 @@
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/convert.hpp>
|
||||
#include <openvino/op/matmul.hpp>
|
||||
#include <openvino/op/maximum.hpp>
|
||||
#include <openvino/op/multiply.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/scaled_dot_product_attention.hpp>
|
||||
#include <openvino/op/scatter_update.hpp>
|
||||
#include <openvino/op/slice.hpp>
|
||||
#include <openvino/op/softmax.hpp>
|
||||
#include <openvino/op/tanh.hpp>
|
||||
#include <openvino/op/tile.hpp>
|
||||
#include <openvino/op/transpose.hpp>
|
||||
#include <openvino/op/unsqueeze.hpp>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
namespace ggml {
|
||||
namespace op {
|
||||
static std::optional<ov::Output<ov::Node>> find_scatter_updates(const ov::Output<ov::Node> & root) {
|
||||
std::vector<ov::Output<ov::Node>> pending{root};
|
||||
std::unordered_set<const ov::Node *> visited;
|
||||
std::optional<ov::Output<ov::Node>> updates;
|
||||
while (!pending.empty()) {
|
||||
auto output = pending.back();
|
||||
pending.pop_back();
|
||||
auto node = output.get_node_shared_ptr();
|
||||
if (!node || !visited.insert(node.get()).second) {
|
||||
continue;
|
||||
}
|
||||
if (auto scatter = ov::as_type_ptr<ov::op::v3::ScatterUpdate>(node)) {
|
||||
OPENVINO_ASSERT(!updates.has_value(), "HFA KV input contains multiple ScatterUpdate ancestors");
|
||||
updates = scatter->input_value(2);
|
||||
continue;
|
||||
}
|
||||
for (const auto & input : node->inputs()) {
|
||||
pending.push_back(input.get_source_output());
|
||||
}
|
||||
}
|
||||
return updates;
|
||||
}
|
||||
|
||||
static ov::Output<ov::Node> reshape_present_kv(const ov::Output<ov::Node> & updates,
|
||||
int64_t n_heads_kv,
|
||||
int64_t head_size) {
|
||||
auto shape = updates.get_partial_shape();
|
||||
OPENVINO_ASSERT(shape.rank().is_static() && shape.rank().get_length() == 4,
|
||||
"HFA present KV updates must be rank 4");
|
||||
auto reshaped = std::make_shared<ov::op::v1::Reshape>(
|
||||
updates,
|
||||
ov::op::v0::Constant::create(
|
||||
ov::element::i64, {4}, std::vector<int64_t>{1, -1, n_heads_kv, head_size}),
|
||||
false);
|
||||
return std::make_shared<ov::op::v1::Transpose>(
|
||||
reshaped, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
|
||||
}
|
||||
|
||||
static int extract_layer(const std::string & name) {
|
||||
const auto begin = name.find("_l");
|
||||
OPENVINO_ASSERT(begin != std::string::npos, "HFA KV input has no layer suffix: ", name);
|
||||
return std::stoi(name.substr(begin + 2));
|
||||
}
|
||||
|
||||
static ov::Output<ov::Node> reshape_flat_kv(const ov::Output<ov::Node> & kv_flat,
|
||||
size_t view_offset_bytes,
|
||||
size_t nb1_bytes,
|
||||
@@ -60,11 +110,16 @@ static ov::Output<ov::Node> reshape_flat_kv(const ov::Output<ov::Node> & kv_flat
|
||||
}
|
||||
|
||||
OutputVector translate_flash_attn_ext(const NodeContext & context) {
|
||||
num_inputs_check(context, 3, 4);
|
||||
const bool has_mask = context.get_input_size() == 4;
|
||||
num_inputs_check(context, 3, 5);
|
||||
// Inputs are q, k, v, then the optional mask and sinks; supports_op rejects sinks without a mask
|
||||
const bool has_mask = context.get_input_size() >= 4;
|
||||
const bool has_sink = context.get_input_size() == 5;
|
||||
|
||||
auto q_f32 = context.get_input(0);
|
||||
auto k = context.get_input(1);
|
||||
auto v = context.get_input(2);
|
||||
const auto k_cache = k;
|
||||
const auto v_cache = v;
|
||||
const int op_case = context.get_op_case();
|
||||
|
||||
if (op_case == 1 || op_case == 2) {
|
||||
@@ -85,34 +140,40 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
|
||||
float * params = reinterpret_cast<float *>(context.get_output_op_params());
|
||||
float scale = params[0];
|
||||
// float max_bias = params[1];
|
||||
// float logit_softcap = params[2];
|
||||
const float logit_softcap = params[2];
|
||||
|
||||
auto q = std::make_shared<ov::op::v0::Convert>(q_f32, ov::element::f16);
|
||||
auto scale_node = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{}, std::vector<float>{scale});
|
||||
|
||||
ov::Output<ov::Node> res;
|
||||
|
||||
// For stateful
|
||||
ov::Output<ov::Node> mask;
|
||||
if (has_mask) {
|
||||
mask = context.get_input(3);
|
||||
std::string mask_name = "KQ_mask_sliced";
|
||||
if (context.get_input_names()[3].find("swa") != std::string::npos) {
|
||||
mask_name = "KQ_mask_swa_sliced";
|
||||
}
|
||||
if (context.has_input(mask_name)) {
|
||||
mask = context.get_input(mask_name);
|
||||
}
|
||||
mask = get_kq_mask_input(context, 3);
|
||||
if (mask.get_element_type() != ov::element::f16) {
|
||||
mask = std::make_shared<ov::op::v0::Convert>(mask, ov::element::f16);
|
||||
}
|
||||
}
|
||||
|
||||
ov::Output<ov::Node> sink;
|
||||
if (has_sink) {
|
||||
sink = context.get_input(4);
|
||||
if (sink.get_element_type() != q->get_element_type()) {
|
||||
sink = std::make_shared<ov::op::v0::Convert>(sink, q->get_element_type());
|
||||
}
|
||||
sink = std::make_shared<ov::op::v1::Reshape>(
|
||||
sink,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4},
|
||||
std::vector<int64_t>{1, context.get_input_shape(0)[1].get_length(), 1, 1}),
|
||||
false);
|
||||
}
|
||||
|
||||
//auto tile_kv = [&](int64_t num_heads, int64_t num_heads_kv, int64_t head_size, ov::Output<Node> kv) {
|
||||
// int64_t factor = num_heads / num_heads_kv;
|
||||
// if (factor > 1 && num_heads_kv > 1) {
|
||||
auto q_shape = context.get_input_shape(0).to_shape();
|
||||
auto k_shape = context.get_input_shape(1).to_shape();
|
||||
auto v_shape = context.get_input_shape(2).to_shape();
|
||||
const int64_t num_heads = q_shape[1];
|
||||
const int64_t num_heads_kv = k_shape[1];
|
||||
const int64_t head_size = q_shape[3];
|
||||
@@ -128,54 +189,73 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
|
||||
}
|
||||
return ggml_openvino_is_gpu();
|
||||
}();
|
||||
// The manual path reshapes Q, K, V and the output with one head size; models whose K/V head sizes differ
|
||||
// from Q's (e.g. MLA) take the SDPA path.
|
||||
const bool same_head_size = k_shape[3] == q_shape[3] && v_shape[3] == q_shape[3];
|
||||
// SDPA has no logit soft-capping (gemma-2), so soft-capped attention always takes the manual path, also in
|
||||
// stateful execution. Only SDPA applies sinks, so attention with sinks never takes the manual path
|
||||
// (supports_op rejects sinks with soft-capping)
|
||||
const bool use_manual_gqa_attention =
|
||||
manual_gqa_enabled && factor > 1 && num_heads_kv > 1 && !context.is_stateful();
|
||||
!has_sink &&
|
||||
(logit_softcap != 0.0f || (manual_gqa_enabled && factor > 1 && num_heads_kv > 1 && !context.is_stateful())) &&
|
||||
same_head_size;
|
||||
FRONT_END_OP_CONVERSION_CHECK(logit_softcap == 0.0f || use_manual_gqa_attention,
|
||||
"FLASH_ATTN_EXT with logit soft-capping is not supported with different K/V head "
|
||||
"sizes");
|
||||
|
||||
if (use_manual_gqa_attention) {
|
||||
// Q, K, V arrive as [B, n_heads(_kv), S, head_size], where B is the active
|
||||
// batch (n_seq_active) and may be > 1 (llama-perplexity, llama-server -np > 1)
|
||||
// or dynamic. Reshape to
|
||||
// K_r: [B, num_heads_kv, 1, S, head_size]
|
||||
// Q_r: [B, num_heads_kv, factor, S_q, head_size]
|
||||
// and let MatMul broadcast across the factor dim without materialising
|
||||
// an expanded K/V. The leading 0 + special_zero=true copies B at runtime,
|
||||
// so this is correct for B == 1, B > 1, and dynamic B alike. Only the head
|
||||
// dims and head_size are baked in as literals; the sequence dim stays -1.
|
||||
auto k_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5},
|
||||
std::vector<int64_t>{0, num_heads_kv, 1, -1, head_size});
|
||||
auto v_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5},
|
||||
std::vector<int64_t>{0, num_heads_kv, 1, -1, head_size});
|
||||
auto q_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5},
|
||||
std::vector<int64_t>{0, num_heads_kv, factor, -1, head_size});
|
||||
// or dynamic. K and V are Transposes of the KV cache; they must feed the MatMuls
|
||||
// directly (4-D, no Reshape in between) so that the GPU plugin folds the transpose
|
||||
// into the Gemm instead of copying the whole transposed cache on every token.
|
||||
// The query heads that share a KV head are stacked along the row axis:
|
||||
// Q: [B, num_heads, S_q, head_size] -> [B, num_heads_kv, factor * S_q, head_size]
|
||||
// (a pure reshape: the query heads of one KV head are adjacent)
|
||||
// scores: [B, num_heads_kv, factor * S_q, S_k]
|
||||
// The leading 0 + special_zero=true copies B at runtime, so this is correct for
|
||||
// B == 1, B > 1, and dynamic B alike.
|
||||
auto q_r = std::make_shared<ov::op::v1::Reshape>(
|
||||
q,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, num_heads_kv, -1, head_size}),
|
||||
true);
|
||||
|
||||
auto k_r = std::make_shared<ov::op::v1::Reshape>(k, k_5d_shape, true);
|
||||
auto v_r = std::make_shared<ov::op::v1::Reshape>(v, v_5d_shape, true);
|
||||
auto q_r = std::make_shared<ov::op::v1::Reshape>(q, q_5d_shape, true);
|
||||
|
||||
// QK^T → [B, num_heads_kv, factor, S_q, S_k]
|
||||
auto qk = std::make_shared<ov::op::v0::MatMul>(q_r, k_r, /*tA=*/false, /*tB=*/true);
|
||||
auto qk_scaled = std::make_shared<ov::op::v1::Multiply>(qk, scale_node);
|
||||
|
||||
// Mask arrives as [B, 1, S_q, S_k]. Unsqueeze a factor axis at position 2 to
|
||||
// get [B, 1, 1, S_q, S_k], which NUMPY-broadcasts cleanly against the
|
||||
// [B, num_heads_kv, factor, S_q, S_k] scores: B==B, then 1→num_heads_kv and
|
||||
// 1→factor on the head dims.
|
||||
ov::Output<ov::Node> qk_masked;
|
||||
if (has_mask) {
|
||||
auto mask_unsq1 =
|
||||
std::make_shared<ov::op::v0::Unsqueeze>(mask, ov::op::v0::Constant::create(ov::element::i64, {1}, {2}));
|
||||
qk_masked = std::make_shared<ov::op::v1::Add>(qk_scaled, mask_unsq1);
|
||||
// QK^T -> [B, num_heads_kv, factor * S_q, S_k]
|
||||
auto qk = std::make_shared<ov::op::v0::MatMul>(q_r, k, /*tA=*/false, /*tB=*/true);
|
||||
ov::Output<ov::Node> probs;
|
||||
if (logit_softcap != 0.0f) {
|
||||
// as ggml: softcap * tanh(qk * scale / softcap)
|
||||
probs = std::make_shared<ov::op::v1::Multiply>(
|
||||
qk,
|
||||
ov::op::v0::Constant::create(ov::element::f16, ov::Shape{}, std::vector<float>{scale / logit_softcap}));
|
||||
probs = std::make_shared<ov::op::v1::Multiply>(
|
||||
std::make_shared<ov::op::v0::Tanh>(probs),
|
||||
ov::op::v0::Constant::create(ov::element::f16, ov::Shape{}, std::vector<float>{logit_softcap}));
|
||||
} else {
|
||||
qk_masked = qk_scaled;
|
||||
probs = std::make_shared<ov::op::v1::Multiply>(qk, scale_node);
|
||||
}
|
||||
|
||||
auto softmax = std::make_shared<ov::op::v8::Softmax>(qk_masked, /*axis=*/-1);
|
||||
if (has_mask) {
|
||||
// Mask arrives as [B, 1, S_q, S_k]. Row j * S_q + s of the scores belongs to query position s, so
|
||||
// repeat the mask factor times along the rows, [B, 1, factor * S_q, S_k], and add it to the 4-D
|
||||
// scores. Keeping the scores 4-D from the MatMul to the Softmax lets the GPU plugin fuse the scale
|
||||
// and the mask into the Gemm: a reshape of the scores in between costs a full pass over them per
|
||||
// layer, which is large in prefill. The tiled mask is the same in every layer.
|
||||
// The GPU plugin fuses this attention into its SDPA kernel, which works on blocks of keys: a block
|
||||
// fully masked with -inf for a row gives NaN (see the SDPA path below), so use a large finite value.
|
||||
auto mask_finite = std::make_shared<ov::op::v1::Maximum>(
|
||||
mask, ov::op::v0::Constant::create(ov::element::f16, ov::Shape{}, std::vector<float>{-30000.0f}));
|
||||
auto mask_tiled = std::make_shared<ov::op::v0::Tile>(
|
||||
mask_finite,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, factor, 1}));
|
||||
probs = std::make_shared<ov::op::v1::Add>(probs, mask_tiled);
|
||||
}
|
||||
probs = std::make_shared<ov::op::v8::Softmax>(probs, /*axis=*/-1);
|
||||
|
||||
// softmax @ V → [B, num_heads_kv, factor, S_q, head_size]
|
||||
auto attn = std::make_shared<ov::op::v0::MatMul>(softmax, v_r);
|
||||
// softmax @ V -> [B, num_heads_kv, factor * S_q, head_size]
|
||||
auto attn = std::make_shared<ov::op::v0::MatMul>(probs, v);
|
||||
|
||||
// Reshape back to [B, num_heads, S_q, head_size] (combine num_heads_kv * factor).
|
||||
// Leading 0 + special_zero=true copies B at runtime.
|
||||
auto out_4d_shape =
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, num_heads, -1, head_size});
|
||||
auto out_4d = std::make_shared<ov::op::v1::Reshape>(attn, out_4d_shape, true);
|
||||
@@ -191,7 +271,7 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
|
||||
// Default path: explicit Broadcast → SDPA. Kept as the fallback because
|
||||
// (a) it goes through the GPU plugin's micro-SDPA fast path (FlashAttention
|
||||
// tiles via DPAS), and (b) the manual path above is still being validated.
|
||||
auto tile_kv = [&](int64_t n_heads, int64_t n_heads_kv, int64_t hs, ov::Output<Node> kv) {
|
||||
auto tile_kv = [&](int64_t n_heads, int64_t n_heads_kv, int64_t hs, ov::Output<Node> kv, bool transposed = false) {
|
||||
int64_t f = n_heads / n_heads_kv;
|
||||
if (f > 1 && n_heads_kv > 1) {
|
||||
ov::Output<ov::Node> kv_broadcast_shape;
|
||||
@@ -202,8 +282,9 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
|
||||
|
||||
kv_broadcast_shape = ov::op::v0::Constant::create(ov::element::i64, {5},
|
||||
{(int64_t) 1, (int64_t) 1, f, (int64_t) 1, (int64_t) 1});
|
||||
new_kv_shape =
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, n_heads, (int64_t) -1, hs});
|
||||
new_kv_shape = ov::op::v0::Constant::create(
|
||||
ov::element::i64, {4},
|
||||
transposed ? std::vector<int64_t>{0, n_heads, hs, -1} : std::vector<int64_t>{0, n_heads, -1, hs});
|
||||
// ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1});
|
||||
//new_kv_shape =
|
||||
// ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, num_heads, (int64_t) -1, head_size});
|
||||
@@ -215,15 +296,112 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) {
|
||||
return kv;
|
||||
};
|
||||
|
||||
// Transposed V cache [1, heads, head_size, rows] (GenAI's NPUW_LLM_OPTIMIZE_V_TENSORS layout).
|
||||
const bool v_transposed = ggml_openvino_npu_kv_head_major_enabled() && context.is_static() &&
|
||||
context.get_input_names()[2].find("cache_v_l") != std::string::npos;
|
||||
|
||||
// the HFA prefill path has no sinks
|
||||
const bool use_hfa_gate =
|
||||
ggml_openvino_npu_hfa_prefill_enabled() && context.is_static() && context.is_prefill() && has_mask && !has_sink;
|
||||
if (use_hfa_gate) {
|
||||
const int layer = extract_layer(context.get_input_names()[1]);
|
||||
auto past_k = context.get_input("ggml_hfa_past_cache_k_l" + std::to_string(layer));
|
||||
auto past_v = context.get_input("ggml_hfa_past_cache_v_l" + std::to_string(layer));
|
||||
auto hfa_mask = context.get_input("ggml_hfa_mask");
|
||||
auto present_k_updates = find_scatter_updates(k_cache);
|
||||
auto present_v_updates = find_scatter_updates(v_cache);
|
||||
if (!present_v_updates && v_transposed) {
|
||||
// Transposed V prefill outputs the chunk rows themselves (see set_rows); the window Slice wraps them.
|
||||
ov::Output<ov::Node> rows = v_cache;
|
||||
while (auto slice = ov::as_type_ptr<ov::op::v8::Slice>(rows.get_node_shared_ptr())) {
|
||||
rows = slice->input_value(0);
|
||||
}
|
||||
present_v_updates = rows;
|
||||
}
|
||||
OPENVINO_ASSERT(present_k_updates.has_value() && present_v_updates.has_value(),
|
||||
"HFA prefill requires ScatterUpdate-backed K/V cache inputs");
|
||||
if (ggml_openvino_npu_kv_head_major_enabled()) {
|
||||
// Head-major cache: past and the new rows are already [1, heads, rows, head_size]
|
||||
// (V: [1, heads, head_size, rows] when transposed).
|
||||
k = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{past_k, *present_k_updates}, 2);
|
||||
v = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{past_v, *present_v_updates},
|
||||
v_transposed ? 3 : 2);
|
||||
} else {
|
||||
auto present_k = reshape_present_kv(*present_k_updates, num_heads_kv, head_size);
|
||||
auto present_v = reshape_present_kv(*present_v_updates, num_heads_kv, head_size);
|
||||
// Past K/V come in ggml cache layout [1, past, heads, head_size]; NPUW HFA reads tiles from it directly.
|
||||
auto to_bhsd = ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3});
|
||||
k = std::make_shared<ov::op::v0::Concat>(
|
||||
ov::OutputVector{std::make_shared<ov::op::v1::Transpose>(past_k, to_bhsd), present_k}, 2);
|
||||
v = std::make_shared<ov::op::v0::Concat>(
|
||||
ov::OutputVector{std::make_shared<ov::op::v1::Transpose>(past_v, to_bhsd), present_v}, 2);
|
||||
}
|
||||
k = tile_kv(num_heads, num_heads_kv, head_size, k);
|
||||
v = tile_kv(num_heads, num_heads_kv, head_size, v, v_transposed);
|
||||
|
||||
auto q_scaled = std::make_shared<ov::op::v1::Multiply>(q, scale_node);
|
||||
auto scores = std::make_shared<ov::op::v0::MatMul>(q_scaled, k, false, true);
|
||||
auto masked_scores = std::make_shared<ov::op::v1::Add>(scores, hfa_mask);
|
||||
auto probs = std::make_shared<ov::op::v8::Softmax>(masked_scores, -1);
|
||||
auto attended = std::make_shared<ov::op::v0::MatMul>(probs, v, false, v_transposed);
|
||||
auto transposed = std::make_shared<ov::op::v1::Transpose>(
|
||||
attended, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
|
||||
auto output_shape = ov::op::v0::Constant::create(
|
||||
ov::element::i64, {3}, std::vector<int64_t>{0, -1, num_heads * head_size});
|
||||
res = std::make_shared<ov::op::v1::Reshape>(transposed, output_shape, true);
|
||||
auto restored_shape = ov::op::v0::Constant::create(
|
||||
ov::element::i64, {4}, std::vector<int64_t>{0, -1, num_heads, head_size});
|
||||
res = std::make_shared<ov::op::v1::Reshape>(res, restored_shape, true);
|
||||
res = std::make_shared<ov::op::v0::Convert>(res, ov::element::f32);
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
|
||||
//auto q_shape = context.get_input_shape(0).to_shape();
|
||||
//auto k_shape = context.get_input_shape(1).to_shape();
|
||||
//k = tile_kv(q_shape[1], k_shape[1], q_shape[3], k);
|
||||
//v = tile_kv(q_shape[1], k_shape[1], q_shape[3], v);
|
||||
k = tile_kv(num_heads, num_heads_kv, head_size, k);
|
||||
v = tile_kv(num_heads, num_heads_kv, head_size, v);
|
||||
v = tile_kv(num_heads, num_heads_kv, head_size, v, v_transposed);
|
||||
|
||||
// The NPU 4000 compiler (Lunar Lake) computes ScaledDotProductAttention wrongly when K has one KV head and is
|
||||
// square, that is when the attended rows equal the head size (gemma-4 E2B: 512 rows with its 512-wide
|
||||
// full-attention heads, 256 rows with its 256-wide sliding-window heads). The same attention as separate MatMuls
|
||||
// is correct there. Remove when the NPU compiler computes the op correctly for this shape.
|
||||
const auto k_dims = k.get_partial_shape();
|
||||
const bool npu_square_k = ggml_openvino_npu_is_4000() && !has_sink && num_heads_kv == 1 &&
|
||||
k_dims.rank().is_static() && k_dims.size() == 4 && k_dims[2].is_static() &&
|
||||
k_dims[3].is_static() && k_dims[2].get_length() == k_dims[3].get_length();
|
||||
|
||||
if (v_transposed || npu_square_k) {
|
||||
// The attention as separate ops. SDPA has no transposed-V form: as GenAI does, that case uses
|
||||
// MatMul(probs, V^T, transpose_b).
|
||||
auto q_scaled = std::make_shared<ov::op::v1::Multiply>(q, scale_node);
|
||||
ov::Output<ov::Node> scores = std::make_shared<ov::op::v0::MatMul>(q_scaled, k, false, true);
|
||||
if (has_mask) {
|
||||
scores = std::make_shared<ov::op::v1::Add>(scores, mask);
|
||||
}
|
||||
auto probs = std::make_shared<ov::op::v8::Softmax>(scores, -1);
|
||||
auto attended = std::make_shared<ov::op::v0::MatMul>(probs, v, false, v_transposed);
|
||||
res = std::make_shared<ov::op::v1::Transpose>(
|
||||
attended, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
|
||||
res = std::make_shared<ov::op::v0::Convert>(res, ov::element::f32);
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
|
||||
// With n_seq_active read from the mask shape, the GPU plugin knows Q's head size and uses its fused SDPA kernel.
|
||||
// That kernel works on blocks of keys: a block fully masked with -inf for a row gives exp(-inf - -inf) = NaN,
|
||||
// which spreads to the row's result, so mask with a large finite value instead
|
||||
if (has_mask && ggml_openvino_is_gpu() && !context.is_stateful() && ggml_openvino_shape_from_mask_enabled()) {
|
||||
mask = std::make_shared<ov::op::v1::Maximum>(
|
||||
mask, ov::op::v0::Constant::create(ov::element::f16, ov::Shape{}, std::vector<float>{-30000.0f}));
|
||||
}
|
||||
|
||||
constexpr auto causal = false;
|
||||
if (has_mask) {
|
||||
if (has_mask && has_sink) {
|
||||
auto sdpa = std::make_shared<ov::op::v13::ScaledDotProductAttention>(q, k, v, mask, scale_node, sink, causal);
|
||||
res = std::make_shared<ov::op::v1::Transpose>(
|
||||
sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
|
||||
} else if (has_mask) {
|
||||
auto sdpa = std::make_shared<ov::op::v13::ScaledDotProductAttention>(q, k, v, mask, scale_node, causal);
|
||||
res = std::make_shared<ov::op::v1::Transpose>(
|
||||
sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
|
||||
|
||||
@@ -3,11 +3,17 @@
|
||||
#include "../node_context.h"
|
||||
#include "../op_table.h"
|
||||
#include "../utils.h"
|
||||
#ifdef GGML_OPENVINO_HAS_CACHE_UPDATE
|
||||
#include "recurrent_cache_update.hpp"
|
||||
#endif
|
||||
#include "ggml-openvino/ggml-openvino-extra.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <numeric>
|
||||
#include <openvino/op/add.hpp>
|
||||
#include <openvino/op/broadcast.hpp>
|
||||
@@ -20,15 +26,23 @@
|
||||
#include <openvino/op/less.hpp>
|
||||
#include <openvino/op/loop.hpp>
|
||||
#include <openvino/op/matmul.hpp>
|
||||
#include <openvino/op/maximum.hpp>
|
||||
#include <openvino/op/minimum.hpp>
|
||||
#include <openvino/op/multiply.hpp>
|
||||
#include <openvino/op/range.hpp>
|
||||
#include <openvino/op/reduce_mean.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/squeeze.hpp>
|
||||
#include <openvino/op/select.hpp>
|
||||
#include <openvino/op/shape_of.hpp>
|
||||
#include <openvino/op/slice.hpp>
|
||||
#include <openvino/op/split.hpp>
|
||||
#include <openvino/op/sqrt.hpp>
|
||||
#include <openvino/op/squeeze.hpp>
|
||||
#include <openvino/op/subtract.hpp>
|
||||
#include <openvino/op/tile.hpp>
|
||||
#include <openvino/op/transpose.hpp>
|
||||
#include <openvino/op/unsqueeze.hpp>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace ov {
|
||||
@@ -92,7 +106,276 @@ static bool match_gdn_l2_norm(const Output<Node> & normalized, Output<Node> & in
|
||||
return true;
|
||||
}
|
||||
|
||||
OutputVector translate_gated_delta_net(const NodeContext & context) {
|
||||
// A [CS, CS] mask of the chunked GatedDeltaNet, one instance per content for the whole process. The NPUW weights
|
||||
// bank, which all compiled models share (NPUW_WEIGHTS_BANK=shared), identifies a constant by its data pointer and
|
||||
// keeps its tensor after the model that owned it is gone. A mask of a later model that got the address of a freed
|
||||
// mask would then be read as that other mask, so a mask is never freed. Not needed once NPUW stops matching a new
|
||||
// constant to a detached one by pointer (OpenVINO 2026.4 does).
|
||||
static std::shared_ptr<ov::op::v0::Constant> gdn_chunk_mask(int64_t CS, const std::vector<float> & values) {
|
||||
static std::mutex mutex;
|
||||
static std::map<std::vector<float>, std::shared_ptr<ov::op::v0::Constant>> cache;
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
auto & mask = cache[values];
|
||||
if (!mask) {
|
||||
mask = ov::op::v0::Constant::create(ov::element::f32, {(size_t) CS, (size_t) CS}, values);
|
||||
}
|
||||
return mask;
|
||||
}
|
||||
|
||||
// Chunked GatedDeltaNet built from MatMuls, for devices where the GatedDeltaNet op cannot run (NPU4000).
|
||||
// Same contract as the op with equal q/k/v heads and no fused l2 norm:
|
||||
// q, k [B, T, H, Dk], v [B, T, H, Dv], g, beta [B, T, H], state [B, H, Dk, Dv]
|
||||
// returns attn [B, T, H, Dv] and the new state [B, H, Dk, Dv].
|
||||
// Per chunk of CS tokens (gc = cumsum of g in the chunk, decay[i][j] = exp(gc_i - gc_j) for j <= i, else 0):
|
||||
// P = (I + strict_lower(beta k k^T * decay))^-1
|
||||
// u = P (beta v), w = P (beta k exp(gc))
|
||||
// v' = u - w S
|
||||
// out = (q exp(gc)) S + lower(q k^T * decay) v'
|
||||
// S = S exp(gc_last) + (k exp(gc_last - gc))^T v'
|
||||
// The chunks run one after another; everything inside a chunk is batched over all chunks and heads.
|
||||
// P is built block-recursively, [[A, 0], [C, D]]^-1 = [[A^-1, 0], [-D^-1 C A^-1, D^-1]], for block sizes
|
||||
// 1, 2, 4, ..., CS / 2. Every intermediate is a part of the true inverse, so values stay small in f16. A Neumann
|
||||
// product (I - L)(I + L^2)... is exact too, but powers of L grow large and cancel, which f16 cannot hold.
|
||||
// A single token (decode) is a chunk of one token, where P = 1 and decay = 1:
|
||||
// v' = beta (v - exp(g) k S)
|
||||
// S = S exp(g) + k^T v'
|
||||
// out = q S
|
||||
// No Loop op is used, so the GatedDeltaNet loop fusion does not turn this back into the op.
|
||||
static std::pair<Output<Node>, Output<Node>> make_gated_delta_net_chunked(Output<Node> q, Output<Node> k,
|
||||
Output<Node> v, Output<Node> g,
|
||||
Output<Node> beta, Output<Node> state,
|
||||
int64_t B, int64_t T, int64_t H, int64_t Dk,
|
||||
int64_t Dv) {
|
||||
const int64_t CS = ggml_openvino_gdn_decompose_chunk();
|
||||
const int64_t max_dim = std::numeric_limits<int64_t>::max();
|
||||
FRONT_END_OP_CONVERSION_CHECK(B > 0 && T > 0 && H > 0 && Dk > 0 && Dv > 0,
|
||||
"GATED_DELTA_NET decomposition requires positive dimensions");
|
||||
const int64_t pad = (CS - T % CS) % CS;
|
||||
FRONT_END_OP_CONVERSION_CHECK(T <= max_dim - pad && B <= max_dim / H,
|
||||
"GATED_DELTA_NET decomposition dimensions overflow");
|
||||
const int64_t N = (T + pad) / CS;
|
||||
const int64_t X = B * H;
|
||||
FRONT_END_OP_CONVERSION_CHECK(X <= max_dim / N / CS / CS && X <= max_dim / N / CS / Dk &&
|
||||
X <= max_dim / N / CS / Dv && X <= max_dim / Dk / Dv,
|
||||
"GATED_DELTA_NET decomposition tensor size overflows");
|
||||
|
||||
auto i64 = [](const std::vector<int64_t> & v) {
|
||||
return ov::op::v0::Constant::create(ov::element::i64, {v.size()}, v);
|
||||
};
|
||||
auto f32 = [](float x) { return ov::op::v0::Constant::create(ov::element::f32, {}, {x}); };
|
||||
auto reshape = [&](const Output<Node> & x, const std::vector<int64_t> & shape) -> Output<Node> {
|
||||
return std::make_shared<ov::op::v1::Reshape>(x, i64(shape), false);
|
||||
};
|
||||
auto transpose = [&](const Output<Node> & x, const std::vector<int64_t> & perm) -> Output<Node> {
|
||||
return std::make_shared<ov::op::v1::Transpose>(x, i64(perm));
|
||||
};
|
||||
auto matmul = [](const Output<Node> & a, const Output<Node> & b, bool ta = false, bool tb = false) -> Output<Node> {
|
||||
return std::make_shared<ov::op::v0::MatMul>(a, b, ta, tb);
|
||||
};
|
||||
auto mul = [](const Output<Node> & a, const Output<Node> & b) -> Output<Node> {
|
||||
return std::make_shared<ov::op::v1::Multiply>(a, b);
|
||||
};
|
||||
auto add = [](const Output<Node> & a, const Output<Node> & b) -> Output<Node> {
|
||||
return std::make_shared<ov::op::v1::Add>(a, b);
|
||||
};
|
||||
auto sub = [](const Output<Node> & a, const Output<Node> & b) -> Output<Node> {
|
||||
return std::make_shared<ov::op::v1::Subtract>(a, b);
|
||||
};
|
||||
auto exp_op = [](const Output<Node> & x) -> Output<Node> { return std::make_shared<ov::op::v0::Exp>(x); };
|
||||
|
||||
if (T == 1) {
|
||||
// [B, 1, H, D] -> [1, X, 1, D], batch 1 as below
|
||||
q = mul(reshape(q, {1, X, 1, Dk}), f32(1.0f / std::sqrt((float) Dk)));
|
||||
k = reshape(k, {1, X, 1, Dk});
|
||||
v = reshape(v, {1, X, 1, Dv});
|
||||
auto state_decay = exp_op(reshape(g, {1, X, 1, 1}));
|
||||
beta = reshape(beta, {1, X, 1, 1});
|
||||
Output<Node> s = reshape(state, {1, X, Dk, Dv});
|
||||
auto v_new = mul(sub(v, matmul(mul(k, state_decay), s)), beta); // [1, X, 1, Dv]
|
||||
s = add(mul(s, state_decay), matmul(k, v_new, true, false)); // [1, X, Dk, Dv]
|
||||
return {reshape(matmul(q, s), {B, 1, H, Dv}), reshape(s, {B, H, Dk, Dv})};
|
||||
}
|
||||
|
||||
// Pad T to a multiple of CS. A padded token has g = 0 and beta = k = 0, so it leaves the state unchanged.
|
||||
if (pad > 0) {
|
||||
auto pad_tokens = [&](const Output<Node> & x, std::vector<size_t> zero_shape) -> Output<Node> {
|
||||
zero_shape[1] = pad;
|
||||
auto zeros = ov::op::v0::Constant::create(ov::element::f32, zero_shape, std::vector<float>{0.0f});
|
||||
return std::make_shared<ov::op::v0::Concat>(ov::OutputVector{x, zeros}, 1);
|
||||
};
|
||||
q = pad_tokens(q, {(size_t) B, 0, (size_t) H, (size_t) Dk});
|
||||
k = pad_tokens(k, {(size_t) B, 0, (size_t) H, (size_t) Dk});
|
||||
v = pad_tokens(v, {(size_t) B, 0, (size_t) H, (size_t) Dv});
|
||||
g = pad_tokens(g, {(size_t) B, 0, (size_t) H});
|
||||
beta = pad_tokens(beta, {(size_t) B, 0, (size_t) H});
|
||||
}
|
||||
|
||||
// Batch stays 1 for the NPU (a batched eltwise is split per batch and mis-broadcasts): X = B * H.
|
||||
// [B, T, H, D] -> [1, X * N, CS, D]
|
||||
auto to_chunks = [&](const Output<Node> & x, int64_t d) {
|
||||
return reshape(transpose(x, {0, 2, 1, 3}), {1, X * N, CS, d});
|
||||
};
|
||||
q = mul(to_chunks(q, Dk), f32(1.0f / std::sqrt((float) Dk)));
|
||||
k = to_chunks(k, Dk);
|
||||
v = to_chunks(v, Dv);
|
||||
g = reshape(transpose(g, {0, 2, 1}), {1, X * N, CS, 1});
|
||||
beta = reshape(transpose(beta, {0, 2, 1}), {1, X * N, CS, 1});
|
||||
|
||||
std::vector<float> lower_incl(CS * CS), lower_strict(CS * CS), identity(CS * CS);
|
||||
for (int64_t i = 0; i < CS; i++) {
|
||||
for (int64_t j = 0; j < CS; j++) {
|
||||
lower_incl[i * CS + j] = j <= i ? 1.0f : 0.0f;
|
||||
lower_strict[i * CS + j] = j < i ? 1.0f : 0.0f;
|
||||
identity[i * CS + j] = i == j ? 1.0f : 0.0f;
|
||||
}
|
||||
}
|
||||
auto lower_incl_mask = gdn_chunk_mask(CS, lower_incl);
|
||||
auto lower_strict_mask = gdn_chunk_mask(CS, lower_strict);
|
||||
auto eye = gdn_chunk_mask(CS, identity);
|
||||
|
||||
// cumsum of g inside a chunk as a MatMul with the lower triangle of ones: gc [1, X * N, CS, 1]
|
||||
auto gc_col = matmul(lower_incl_mask, g);
|
||||
auto gc_row = reshape(gc_col, {1, X * N, 1, CS});
|
||||
auto causal = std::make_shared<ov::op::v0::Convert>(lower_incl_mask, ov::element::boolean);
|
||||
auto log_decay = std::make_shared<ov::op::v1::Select>(causal, sub(gc_col, gc_row), f32(0.0f));
|
||||
auto decay = mul(exp_op(log_decay), lower_incl_mask);
|
||||
|
||||
auto kb = mul(k, beta);
|
||||
auto vb = mul(v, beta);
|
||||
|
||||
// P = (I + L)^-1 with L = strict_lower(beta k k^T * decay)
|
||||
auto neg_l = mul(mul(mul(matmul(kb, k, false, true), decay), lower_strict_mask), f32(-1.0f));
|
||||
// p holds the inverses of the diagonal b x b blocks; merging pairs of them only needs the -L entries
|
||||
// in the lower off-diagonal b x b block of each 2b x 2b block: p += p (-L masked) p
|
||||
auto block_mask = [&](int64_t b) {
|
||||
std::vector<float> m(CS * CS);
|
||||
for (int64_t i = 0; i < CS; i++) {
|
||||
for (int64_t j = 0; j < CS; j++) {
|
||||
m[i * CS + j] = (i / (2 * b) == j / (2 * b) && i / b > j / b) ? 1.0f : 0.0f;
|
||||
}
|
||||
}
|
||||
return gdn_chunk_mask(CS, m);
|
||||
};
|
||||
Output<Node> p = add(eye, mul(neg_l, block_mask(1)));
|
||||
for (int64_t b = 2; b < CS; b *= 2) {
|
||||
p = add(p, matmul(matmul(p, mul(neg_l, block_mask(b))), p));
|
||||
}
|
||||
|
||||
auto gc_exp = exp_op(gc_col); // [1, X * N, CS, 1]
|
||||
auto u = matmul(p, vb); // [1, X * N, CS, Dv]
|
||||
auto w = matmul(p, mul(kb, gc_exp)); // [1, X * N, CS, Dk]
|
||||
auto qg = mul(q, gc_exp); // [1, X * N, CS, Dk]
|
||||
auto qk = mul(matmul(q, k, false, true), decay); // [1, X * N, CS, CS]
|
||||
auto gc_last = std::make_shared<ov::op::v8::Slice>(gc_col, i64({CS - 1}), i64({CS}), i64({1}), i64({2}));
|
||||
auto kg = mul(k, exp_op(sub(gc_last, gc_col))); // [1, X * N, CS, Dk]
|
||||
auto state_decay = exp_op(gc_last); // [1, X * N, 1, 1]
|
||||
|
||||
// Per-chunk slices, reshaped to [X, rows, cols]
|
||||
auto chunk_axis = ov::op::v0::Constant::create(ov::element::i64, {}, {1});
|
||||
auto split_chunks = [&](const Output<Node> & x, int64_t rows, int64_t cols) {
|
||||
auto split = std::make_shared<ov::op::v1::Split>(reshape(x, {X, N, rows * cols}), chunk_axis,
|
||||
static_cast<size_t>(N));
|
||||
std::vector<Output<Node>> parts;
|
||||
for (int64_t n = 0; n < N; n++) {
|
||||
parts.push_back(reshape(split->output(n), {X, rows, cols}));
|
||||
}
|
||||
return parts;
|
||||
};
|
||||
auto u_n = split_chunks(u, CS, Dv);
|
||||
auto w_n = split_chunks(w, CS, Dk);
|
||||
auto qg_n = split_chunks(qg, CS, Dk);
|
||||
auto qk_n = split_chunks(qk, CS, CS);
|
||||
auto kg_n = split_chunks(kg, CS, Dk);
|
||||
auto state_decay_n = split_chunks(state_decay, 1, 1);
|
||||
|
||||
Output<Node> s = reshape(state, {X, Dk, Dv});
|
||||
ov::OutputVector outs;
|
||||
for (int64_t n = 0; n < N; n++) {
|
||||
auto v_new = sub(u_n[n], matmul(w_n[n], s)); // [X, CS, Dv]
|
||||
outs.push_back(add(matmul(qg_n[n], s), matmul(qk_n[n], v_new))); // [X, CS, Dv]
|
||||
s = add(mul(s, state_decay_n[n]), matmul(kg_n[n], v_new, true, false)); // [X, Dk, Dv]
|
||||
}
|
||||
|
||||
Output<Node> attn = N > 1 ? std::make_shared<ov::op::v0::Concat>(outs, 1)->output(0) : outs[0];
|
||||
if (pad > 0) {
|
||||
attn = std::make_shared<ov::op::v8::Slice>(attn, i64({0}), i64({T}), i64({1}), i64({1}));
|
||||
}
|
||||
attn = reshape(attn, {B, H, T, Dv});
|
||||
return {transpose(attn, {0, 2, 1, 3}), reshape(s, {B, H, Dk, Dv})};
|
||||
}
|
||||
|
||||
static OutputVector make_gated_delta_net(const Output<Node> & q,
|
||||
const Output<Node> & k,
|
||||
const Output<Node> & v,
|
||||
const Output<Node> & state,
|
||||
const Output<Node> & g,
|
||||
const Output<Node> & beta,
|
||||
bool norm,
|
||||
float q_eps,
|
||||
float k_eps) {
|
||||
const auto & q_shape = q.get_partial_shape();
|
||||
const auto & v_shape = v.get_partial_shape();
|
||||
if (ggml_openvino_gdn_decompose_enabled() && !norm && q_shape.is_static() && v_shape.is_static()) {
|
||||
auto outputs = make_gated_delta_net_chunked(q, k, v, g, beta, state,
|
||||
q_shape[0].get_length(), q_shape[1].get_length(),
|
||||
v_shape[2].get_length(), q_shape[3].get_length(),
|
||||
v_shape[3].get_length());
|
||||
return {outputs.first, outputs.second};
|
||||
}
|
||||
auto gdn = std::make_shared<ov::op::internal::GatedDeltaNet>(q, k, v, state, g, beta, norm, q_eps, k_eps);
|
||||
return {gdn->output(0), gdn->output(1)};
|
||||
}
|
||||
|
||||
static OutputVector translate_gdn_snapshots(const Output<Node> & q,
|
||||
const Output<Node> & k,
|
||||
const Output<Node> & v,
|
||||
const Output<Node> & state,
|
||||
const Output<Node> & g,
|
||||
const Output<Node> & beta,
|
||||
int64_t features,
|
||||
bool norm,
|
||||
float q_eps,
|
||||
float k_eps,
|
||||
int64_t snapshot_count) {
|
||||
// Unmatched graphs need state outputs. Ordinary GDN prefixes preserve that contract.
|
||||
auto i64 = [](const Shape & shape, const std::vector<int64_t> & values) {
|
||||
return ov::op::v0::Constant::create(ov::element::i64, shape, values);
|
||||
};
|
||||
auto shape = std::make_shared<ov::op::v3::ShapeOf>(q, ov::element::i64);
|
||||
auto batch = std::make_shared<ov::op::v8::Gather>(shape, i64({1}, {0}), i64({}, {0}));
|
||||
auto tokens = std::make_shared<ov::op::v8::Gather>(shape, i64({1}, {1}), i64({}, {0}));
|
||||
auto perm = i64({4}, {0, 1, 3, 2});
|
||||
auto initial = std::make_shared<ov::op::v1::Transpose>(state, perm);
|
||||
Output<Node> attention;
|
||||
OutputVector snapshots;
|
||||
for (int64_t offset = 0; offset < snapshot_count; ++offset) {
|
||||
auto end = std::make_shared<ov::op::v1::Maximum>(
|
||||
std::make_shared<ov::op::v1::Subtract>(tokens, i64({1}, {offset})), i64({1}, {1}));
|
||||
auto prefix = [&](const Output<Node> & value) -> Output<Node> {
|
||||
if (offset == 0) {
|
||||
return value;
|
||||
}
|
||||
// Materialize prefixes to avoid the GPU's dynamic Slice alias optimization.
|
||||
auto indices = std::make_shared<ov::op::v4::Range>(
|
||||
i64({}, {0}), std::make_shared<ov::op::v0::Squeeze>(end), i64({}, {1}), ov::element::i64);
|
||||
return std::make_shared<ov::op::v8::Gather>(value, indices, i64({}, {1}));
|
||||
};
|
||||
auto gdn = std::make_shared<ov::op::internal::GatedDeltaNet>(
|
||||
prefix(q), prefix(k), prefix(v), initial, prefix(g), prefix(beta), norm, q_eps, k_eps);
|
||||
if (offset == 0) {
|
||||
attention = gdn->output(0);
|
||||
}
|
||||
snapshots.push_back(std::make_shared<ov::op::v1::Reshape>(
|
||||
std::make_shared<ov::op::v1::Transpose>(gdn->output(1), perm), i64({4}, {1, 1, -1, features}), false));
|
||||
}
|
||||
auto valid_rows = std::make_shared<ov::op::v1::Multiply>(
|
||||
std::make_shared<ov::op::v1::Minimum>(tokens, i64({1}, {snapshot_count})), batch);
|
||||
auto states = std::make_shared<ov::op::v8::Slice>(
|
||||
std::make_shared<ov::op::v0::Concat>(snapshots, 2), i64({1}, {0}), valid_rows, i64({1}, {1}), i64({1}, {2}));
|
||||
return {attention, states};
|
||||
}
|
||||
|
||||
static OutputVector translate_gdn(const NodeContext & context, const OutputVector * cache_inputs) {
|
||||
auto v_shape = context.get_input_shape(2).to_shape(); // [B, T, H_v, S_v]
|
||||
auto q_shape = context.get_input_shape(0).to_shape(); // [B, T, H_k, S_k]
|
||||
|
||||
@@ -107,6 +390,7 @@ OutputVector translate_gated_delta_net(const NodeContext & context) {
|
||||
const int64_t H_v = v_shape[2];
|
||||
const int64_t S_v = v_shape[3];
|
||||
const int64_t H_k = q_shape[2];
|
||||
const int64_t snapshot_count = context.get_output_op_params()[0];
|
||||
// const int64_t S_k = q_shape[3];
|
||||
|
||||
auto q = context.get_input(0);
|
||||
@@ -126,6 +410,12 @@ OutputVector translate_gated_delta_net(const NodeContext & context) {
|
||||
k = raw_k;
|
||||
}
|
||||
|
||||
if (context.get_op_case() == 1 && ggml_openvino_gdn_prescale_enabled()) {
|
||||
// the output is linear in q and the state does not depend on q; the next rms_norm removes the scale
|
||||
q = std::make_shared<ov::op::v1::Multiply>(
|
||||
q, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {ggml_openvino_gdn_q_scale()}));
|
||||
}
|
||||
|
||||
// ggml maps GQA heads in tiled order, while OV GDN maps repeated heads in grouped order.
|
||||
if (H_v != H_k) {
|
||||
const int64_t repeat = H_v / H_k;
|
||||
@@ -145,8 +435,13 @@ OutputVector translate_gated_delta_net(const NodeContext & context) {
|
||||
|
||||
// ggml state layout (OV notation): [B, H_v, value_dim, key_dim]
|
||||
// GatedDeltaNet op expects: [B, H_v, key_dim, value_dim]
|
||||
// With native state the buffer already holds the op layout, so skip both transposes. This is safe because
|
||||
// resets and slot copies move whole states, and supports_op aborts instead of falling back to CPU.
|
||||
const bool native_state = ggml_openvino_gdn_native_state_enabled();
|
||||
auto state_perm = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, 1, 3, 2});
|
||||
state = std::make_shared<ov::op::v1::Transpose>(state, state_perm);
|
||||
if (!native_state && !cache_inputs && snapshot_count == 1) {
|
||||
state = std::make_shared<ov::op::v1::Transpose>(state, state_perm);
|
||||
}
|
||||
|
||||
g = std::make_shared<ov::op::v0::Squeeze>(g, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
|
||||
beta = std::make_shared<ov::op::v0::Squeeze>(beta, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
|
||||
@@ -177,33 +472,72 @@ OutputVector translate_gated_delta_net(const NodeContext & context) {
|
||||
// << ", v=" << v.get_partial_shape() << ", g=" << g.get_partial_shape()
|
||||
// << ", beta=" << beta.get_partial_shape() << ", state=" << state.get_partial_shape() << std::endl;
|
||||
|
||||
auto gdn = std::make_shared<ov::op::internal::GatedDeltaNet>(q, k, v, state, g, beta, fuse_qk_l2norm, q_eps, k_eps);
|
||||
auto attn_4d = gdn->output(0);
|
||||
auto state_4d = gdn->output(1); // [B, H_v, key_dim, value_dim]
|
||||
#ifdef GGML_OPENVINO_HAS_CACHE_UPDATE
|
||||
if (cache_inputs) {
|
||||
OutputVector args{q, k, v, (*cache_inputs)[0], g, beta};
|
||||
args.insert(args.end(), cache_inputs->begin() + 1, cache_inputs->end());
|
||||
auto fused = std::make_shared<ov::op::internal::RecurrentGatedDeltaNetUpdate>(
|
||||
args, fuse_qk_l2norm, q_eps, k_eps, snapshot_count);
|
||||
fused->set_friendly_name(context.get_name() + "/RecurrentGatedDeltaNetUpdate");
|
||||
// The matched state VIEW is consumed only by the overridden full-cache
|
||||
// writeback. Route that cache directly instead of materializing snapshots.
|
||||
if (context.get_output_names().size() == 3) {
|
||||
return {fused->output(0), fused->output(0), (*cache_inputs)[0]};
|
||||
}
|
||||
return {fused->output(0), (*cache_inputs)[0]};
|
||||
}
|
||||
#endif
|
||||
|
||||
// std::cout << "GatedDeltaNet output shapes: attn=" << gdn->output(0).get_partial_shape()
|
||||
// << ", new_state=" << gdn->output(1).get_partial_shape() << std::endl;
|
||||
|
||||
// Transpose output state back to ggml layout [B, H_v, value_dim, key_dim]
|
||||
auto state_transposed = std::make_shared<ov::op::v1::Transpose>(state_4d, state_perm);
|
||||
if (context.get_output_names().size() == 2) {
|
||||
Output<Node> attn_4d, states;
|
||||
if (snapshot_count > 1) {
|
||||
auto outputs = translate_gdn_snapshots(q, k, v, state, g, beta, H_v * S_v * q_shape[3],
|
||||
fuse_qk_l2norm, q_eps, k_eps, snapshot_count);
|
||||
attn_4d = outputs[0];
|
||||
states = outputs[1];
|
||||
} else {
|
||||
auto outputs = make_gated_delta_net(q, k, v, state, g, beta, fuse_qk_l2norm, q_eps, k_eps);
|
||||
attn_4d = outputs[0];
|
||||
if (native_state) {
|
||||
states = outputs[1];
|
||||
} else {
|
||||
states = std::make_shared<ov::op::v1::Transpose>(outputs[1], state_perm);
|
||||
}
|
||||
}
|
||||
if (snapshot_count == 1 && context.get_output_names().size() == 2) {
|
||||
// The canonical graph consumes the packed GGML result only through separate attention
|
||||
// and state VIEWs. Publish the native outputs under those VIEW names to avoid
|
||||
// flatten -> concat -> reshape -> slice -> reshape chains. This also works for B > 1.
|
||||
return rename_outputs_with_suffix({attn_4d, state_transposed}, context.get_name());
|
||||
return rename_outputs_with_suffix({attn_4d, states}, context.get_name());
|
||||
}
|
||||
|
||||
auto flat_shape_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
|
||||
auto attn = std::make_shared<ov::op::v1::Reshape>(attn_4d, flat_shape_1d, false);
|
||||
auto new_state = std::make_shared<ov::op::v1::Reshape>(state_transposed, flat_shape_1d, false);
|
||||
auto new_state = std::make_shared<ov::op::v1::Reshape>(states, flat_shape_1d, false);
|
||||
auto packed = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{attn, new_state}, 0);
|
||||
auto out_shape = ov::op::v0::Constant::create(ov::element::i64, {4},
|
||||
std::vector<int64_t>{1, 1, -1 /*T * B + S_v * B*/, S_v * H_v});
|
||||
auto res = std::make_shared<ov::op::v1::Reshape>(packed, out_shape, false);
|
||||
|
||||
if (context.get_output_names().size() == 3) {
|
||||
auto flat_states = std::make_shared<ov::op::v1::Reshape>(
|
||||
states,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, -1, S_v * S_v * H_v}),
|
||||
false);
|
||||
return rename_outputs_with_suffix({res, attn_4d, flat_states}, context.get_name());
|
||||
}
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
|
||||
OutputVector translate_gated_delta_net(const NodeContext & context) {
|
||||
return translate_gdn(context, nullptr);
|
||||
}
|
||||
|
||||
#ifdef GGML_OPENVINO_HAS_CACHE_UPDATE
|
||||
OutputVector translate_recurrent_gated_delta_net(const NodeContext & context, const OutputVector & cache_inputs) {
|
||||
return translate_gdn(context, &cache_inputs);
|
||||
}
|
||||
#endif
|
||||
|
||||
static OutputVector translate_gated_delta_net_ref(const NodeContext & context) {
|
||||
num_inputs_check(context, 6, 6);
|
||||
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
// Copyright (C) 2018-2026 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
#pragma once
|
||||
|
||||
#include "openvino/op/op.hpp"
|
||||
|
||||
@@ -11,9 +11,9 @@
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/convert.hpp>
|
||||
#include <openvino/op/gather.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/shape_of.hpp>
|
||||
#include <openvino/op/slice.hpp>
|
||||
#include <openvino/op/squeeze.hpp>
|
||||
#include <openvino/op/unsqueeze.hpp>
|
||||
|
||||
namespace ov {
|
||||
@@ -39,15 +39,14 @@ OutputVector translate_get_rows(const NodeContext & context) {
|
||||
data.get_partial_shape()[1].compatible(view_shape[3]),
|
||||
"GET_ROWS: view over a weight must be a row range");
|
||||
row_offset = static_cast<int64_t>(view_offset / row_bytes);
|
||||
} else {
|
||||
} else if (context.get_view_input_size(0) > 0) {
|
||||
data = process_view_input_new(context, 0);
|
||||
} else {
|
||||
// gemma-4 gathers the mask rows of its output tokens for the KV-shared layers
|
||||
data = get_kq_mask_input(context, 0);
|
||||
}
|
||||
|
||||
auto op_case = context.get_op_case();
|
||||
if (op_case == 3 || op_case == 4) {
|
||||
return {data};
|
||||
}
|
||||
|
||||
ov::Output<ov::Node> indices;
|
||||
if ((op_case == 1 || op_case == 2) && context.has_input("s_copy_active_slot_len")) {
|
||||
// Recurrent state reorder (inp->s_copy): slice the active (op_case 1) or extra (op_case 2)
|
||||
@@ -68,20 +67,41 @@ OutputVector translate_get_rows(const NodeContext & context) {
|
||||
indices = process_view_input_new(context, 1);
|
||||
}
|
||||
|
||||
if (op_case == 5) {
|
||||
// data [b3, b2, rows, row_size], indices [b3, b2, k]: one gather per (b3, b2) batch. The leading dims reach
|
||||
// here in either layout (stateful models merge b3 into b2), so restore them from the static inner dims
|
||||
const auto data_shape = context.get_input_shape(0);
|
||||
const auto indices_shape = context.get_input_shape(1);
|
||||
data = std::make_shared<ov::op::v1::Reshape>(
|
||||
data,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4},
|
||||
{(int64_t) -1, data_shape[1].get_length(), data_shape[2].get_length(),
|
||||
data_shape[3].get_length()}),
|
||||
false);
|
||||
indices = std::make_shared<ov::op::v1::Reshape>(
|
||||
indices,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {3},
|
||||
{(int64_t) -1, indices_shape[2].get_length(), indices_shape[3].get_length()}),
|
||||
false);
|
||||
auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {2});
|
||||
res = std::make_shared<ov::op::v8::Gather>(data, indices, axis, 2);
|
||||
if (res.get_element_type() != context.get_output_type()) {
|
||||
res = std::make_shared<ov::op::v0::Convert>(res, context.get_output_type());
|
||||
}
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
|
||||
// data[1,b,x,y] ind[1,1,b,x'] test-backend-ops case
|
||||
// data[x,y] ind[1,1,1,x'] normal case
|
||||
// Squeeze the leading dims down to [b,x']. Stateful models drop one rank, so a hardcoded
|
||||
// Drop the leading dims down to [b,x']. Stateful models drop one rank, so a hardcoded
|
||||
// {0,1} would also strip the batch dim whenever b == 1 (every decode step).
|
||||
const auto indices_rank = indices.get_partial_shape().rank();
|
||||
FRONT_END_OP_CONVERSION_CHECK(indices_rank.is_static(), "Expected static rank for GET_ROWS indices");
|
||||
std::vector<int64_t> indices_squeeze_axes;
|
||||
for (int64_t i = 0; i + 2 < indices_rank.get_length(); ++i) {
|
||||
indices_squeeze_axes.push_back(i);
|
||||
}
|
||||
if (!indices_squeeze_axes.empty()) {
|
||||
indices = std::make_shared<ov::op::v0::Squeeze>(
|
||||
indices, ov::op::v0::Constant::create(ov::element::i64, {indices_squeeze_axes.size()},
|
||||
indices_squeeze_axes));
|
||||
if (indices_rank.get_length() > 2) {
|
||||
// Keep zero-token batches: GPU Squeeze-to-Reshape can treat a zero dimension as a copy marker.
|
||||
indices = std::make_shared<ov::op::v1::Reshape>(
|
||||
indices, get_dimensions(std::make_shared<ov::op::v3::ShapeOf>(indices),
|
||||
{int(indices_rank.get_length() - 2), int(indices_rank.get_length() - 1)}), false);
|
||||
}
|
||||
if (row_offset != 0) {
|
||||
indices = std::make_shared<ov::op::v1::Add>(
|
||||
@@ -90,14 +110,14 @@ OutputVector translate_get_rows(const NodeContext & context) {
|
||||
if (data.get_partial_shape().rank() == 4) {
|
||||
if (!(data.get_partial_shape()[1].is_dynamic()) && data.get_partial_shape()[1].get_length() == 1) {
|
||||
// Work-around for a bug in ov cpu plugin for test-backend-ops
|
||||
data = std::make_shared<ov::op::v0::Squeeze>(data,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {2}, {0, 1}));
|
||||
data = std::make_shared<ov::op::v1::Reshape>(
|
||||
data, get_dimensions(data.get_node_shared_ptr(), {2, 3}), false);
|
||||
auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0});
|
||||
res = std::make_shared<ov::op::v8::Gather>(data, indices, axis);
|
||||
} else {
|
||||
auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1});
|
||||
data =
|
||||
std::make_shared<ov::op::v0::Squeeze>(data, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));
|
||||
data = std::make_shared<ov::op::v1::Reshape>(
|
||||
data, get_dimensions(data.get_node_shared_ptr(), {1, 2, 3}), false);
|
||||
// data: [batch, rows, ...], indices: [batch, n] - this is a batched gather
|
||||
// (batch_dims=1) along the rows axis. The data and indices batch dims are
|
||||
// logically equal (both == n_tokens) but reach this node through independent
|
||||
@@ -159,8 +179,15 @@ OutputVector translate_get_rows(const NodeContext & context) {
|
||||
auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1});
|
||||
res = std::make_shared<ov::op::v8::Gather>(data, indices, axis, 1);
|
||||
} else {
|
||||
auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0});
|
||||
res = std::make_shared<ov::op::v8::Gather>(data, indices, axis);
|
||||
// For a tied embedding the same weight also feeds the lm_head matmul. Gathering the
|
||||
// dequantized table would give that dequantization chain a second consumer, forcing the
|
||||
// full table to be materialized; gather the packed rows instead and dequantize those.
|
||||
if (auto compressed = gather_compressed_rows(data, indices)) {
|
||||
res = compressed;
|
||||
} else {
|
||||
auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0});
|
||||
res = std::make_shared<ov::op::v8::Gather>(data, indices, axis);
|
||||
}
|
||||
}
|
||||
|
||||
if (res.get_element_type() != context.get_output_type()) {
|
||||
|
||||
@@ -8,20 +8,36 @@
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <numeric>
|
||||
#include <openvino/op/add.hpp>
|
||||
#include <openvino/op/bitwise_and.hpp>
|
||||
#include <openvino/op/bitwise_right_shift.hpp>
|
||||
#include <openvino/op/broadcast.hpp>
|
||||
#include <openvino/op/concat.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/convert.hpp>
|
||||
#include <openvino/op/cum_sum.hpp>
|
||||
#include <openvino/op/equal.hpp>
|
||||
#include <openvino/op/gather.hpp>
|
||||
#include <openvino/op/greater_eq.hpp>
|
||||
#include <openvino/op/less.hpp>
|
||||
#include <openvino/op/logical_and.hpp>
|
||||
#include <openvino/op/logical_not.hpp>
|
||||
#include <openvino/op/matmul.hpp>
|
||||
#include <openvino/op/maximum.hpp>
|
||||
#include <openvino/op/multiply.hpp>
|
||||
#include <openvino/op/reduce_logical_or.hpp>
|
||||
#include <openvino/op/reduce_sum.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/select.hpp>
|
||||
#include <openvino/op/shape_of.hpp>
|
||||
#include <openvino/op/slice.hpp>
|
||||
#include <openvino/op/squeeze.hpp>
|
||||
#include <openvino/op/strided_slice.hpp>
|
||||
#include <openvino/op/transpose.hpp>
|
||||
#include <openvino/op/unsqueeze.hpp>
|
||||
#include <openvino/op/util/topk_base.hpp>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
namespace ov {
|
||||
@@ -71,6 +87,79 @@ ov::Output<ov::Node> static_shape_dims_or_shapeof(const ov::Output<ov::Node> & i
|
||||
return get_dimensions(shape, dims);
|
||||
}
|
||||
|
||||
// TopK indices are never -1. Model graphs take the ids from the router's TopK through views and
|
||||
// reshapes, so they skip the -1 handling below and keep the GatherMatmul pattern that the GPU MoE
|
||||
// fusion matches.
|
||||
bool ids_from_topk(ov::Output<ov::Node> ids) {
|
||||
for (;;) {
|
||||
const auto node = ids.get_node_shared_ptr();
|
||||
if (ov::is_type<ov::op::util::TopKBase>(node)) {
|
||||
return ids.get_index() == 1;
|
||||
}
|
||||
if (!ov::is_type<ov::op::v1::Reshape>(node) && !ov::is_type<ov::op::v0::Unsqueeze>(node) &&
|
||||
!ov::is_type<ov::op::v0::Squeeze>(node) && !ov::is_type<ov::op::v0::Convert>(node) &&
|
||||
!ov::is_type<ov::op::v1::Transpose>(node) && !ov::is_type<ov::op::v8::Slice>(node) &&
|
||||
!ov::is_type<ov::op::v1::StridedSlice>(node)) {
|
||||
return false;
|
||||
}
|
||||
ids = node->input_value(0);
|
||||
}
|
||||
}
|
||||
|
||||
// An id of -1 skips the slot and zeroes the matching row of the result. The skipped slot looks up a
|
||||
// valid expert, and the result is multiplied by this mask: 1 for a kept slot, 0 for a skipped one,
|
||||
// shaped [ids..., 1] to broadcast over the row.
|
||||
ov::Output<ov::Node> kept_slots_mask(const ov::Output<ov::Node> & ids, const ov::element::Type & type) {
|
||||
auto zero = ov::op::v0::Constant::create(ids.get_element_type(), ov::Shape{}, {0});
|
||||
ov::Output<ov::Node> keep = std::make_shared<ov::op::v1::GreaterEqual>(ids, zero);
|
||||
keep = std::make_shared<ov::op::v0::Convert>(keep, type);
|
||||
return std::make_shared<ov::op::v0::Unsqueeze>(keep, const_i64({-1}));
|
||||
}
|
||||
|
||||
ov::Output<ov::Node> clamp_skipped_ids(const ov::Output<ov::Node> & ids) {
|
||||
auto zero = ov::op::v0::Constant::create(ids.get_element_type(), ov::Shape{}, {0});
|
||||
return std::make_shared<ov::op::v1::Maximum>(ids, zero);
|
||||
}
|
||||
|
||||
// GatherMatmul computes a wrong row when two slots of a token name the same expert (ggml leaves
|
||||
// that undefined too, see TAG_MUL_MAT_ID_DUPLICATE_ID), so a skipped slot cannot simply look up
|
||||
// expert 0 while another slot of the row keeps it. A row with S skipped slots keeps at most
|
||||
// n_used - S experts, so [0, n_used) holds at least S unused ones: the r-th skipped slot of a row
|
||||
// looks up the r-th of them. ids is [n_tokens, n_used].
|
||||
ov::Output<ov::Node> distinct_ids_for_skipped_slots(const ov::Output<ov::Node> & ids) {
|
||||
using namespace ov::op;
|
||||
const auto & ps = ids.get_partial_shape();
|
||||
if (ps.rank().is_dynamic() || ps[ps.rank().get_length() - 1].is_dynamic()) {
|
||||
return clamp_skipped_ids(ids);
|
||||
}
|
||||
const auto type = ids.get_element_type();
|
||||
const int64_t n_used = ps[ps.rank().get_length() - 1].get_length();
|
||||
std::vector<int64_t> range(n_used);
|
||||
std::iota(range.begin(), range.end(), 0);
|
||||
auto experts = v0::Constant::create(type, ov::Shape{static_cast<size_t>(n_used)}, range);
|
||||
auto zero = v0::Constant::create(type, ov::Shape{}, {0});
|
||||
auto one = v0::Constant::create(type, ov::Shape{}, {1});
|
||||
auto last_axis = v0::Constant::create(ov::element::i64, ov::Shape{}, {-1});
|
||||
|
||||
// free[t, e]: no slot of row t uses expert e, for e in [0, n_used)
|
||||
auto uses = std::make_shared<v1::Equal>(std::make_shared<v0::Unsqueeze>(ids, const_i64({-1})), experts);
|
||||
auto free = std::make_shared<v1::LogicalNot>(std::make_shared<v1::ReduceLogicalOr>(uses, const_i64({-2}), false));
|
||||
// 1-based rank of each free expert in its row, and of each skipped slot in its row
|
||||
auto free_rank = std::make_shared<v0::CumSum>(std::make_shared<v0::Convert>(free, type), last_axis);
|
||||
auto skip = std::make_shared<v1::Less>(ids, zero);
|
||||
auto skip_rank = std::make_shared<v1::Add>(
|
||||
std::make_shared<v0::CumSum>(std::make_shared<v0::Convert>(skip, type), last_axis, true), one);
|
||||
|
||||
// pick[t, j, e]: expert e is the skip_rank[t, j]-th free expert of row t
|
||||
auto pick = std::make_shared<v1::LogicalAnd>(
|
||||
std::make_shared<v1::Equal>(std::make_shared<v0::Unsqueeze>(skip_rank, const_i64({-1})),
|
||||
std::make_shared<v0::Unsqueeze>(free_rank, const_i64({-2}))),
|
||||
std::make_shared<v0::Unsqueeze>(free, const_i64({-2})));
|
||||
auto picked = std::make_shared<v1::ReduceSum>(
|
||||
std::make_shared<v1::Multiply>(std::make_shared<v0::Convert>(pick, type), experts), const_i64({-1}), false);
|
||||
return std::make_shared<v1::Select>(skip, picked, ids);
|
||||
}
|
||||
|
||||
ov::Output<ov::Node> translate_mul_mat_id_mxfp4_packed(const NodeContext & context,
|
||||
ov::Output<ov::Node> expert_weights,
|
||||
ov::Output<ov::Node> activations,
|
||||
@@ -98,6 +187,13 @@ ov::Output<ov::Node> translate_mul_mat_id_mxfp4_packed(const NodeContext & conte
|
||||
if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) {
|
||||
ids = std::make_shared<ov::op::v0::Convert>(ids, ov::element::i32);
|
||||
}
|
||||
const auto output_type = context.get_output_type();
|
||||
// Gather and MatMul handle repeated ids, so clamping a skipped slot to expert 0 is enough here.
|
||||
std::optional<ov::Output<ov::Node>> keep;
|
||||
if (!ids_from_topk(ids)) {
|
||||
keep = kept_slots_mask(ids, output_type);
|
||||
ids = clamp_skipped_ids(ids);
|
||||
}
|
||||
|
||||
auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0});
|
||||
|
||||
@@ -156,10 +252,12 @@ ov::Output<ov::Node> translate_mul_mat_id_mxfp4_packed(const NodeContext & conte
|
||||
ov::OutputVector{batch_dim, get_dimensions(ids_shape, {0, 1}), row_dim}, 0);
|
||||
result = std::make_shared<ov::op::v1::Reshape>(result, result_target_dims, false);
|
||||
|
||||
const auto output_type = context.get_output_type();
|
||||
if (result.get_element_type() != output_type) {
|
||||
result = std::make_shared<ov::op::v0::Convert>(result, output_type);
|
||||
}
|
||||
if (keep) {
|
||||
result = std::make_shared<ov::op::v1::Multiply>(result, *keep);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -219,6 +317,12 @@ OutputVector translate_mul_mat_id(const NodeContext & context) {
|
||||
}
|
||||
|
||||
const auto output_type = context.get_output_type();
|
||||
std::optional<ov::Output<ov::Node>> keep;
|
||||
if (!ids_from_topk(ids)) {
|
||||
keep = kept_slots_mask(ids, output_type);
|
||||
ids = distinct_ids_for_skipped_slots(ids);
|
||||
}
|
||||
|
||||
const auto activations_type = ggml_openvino_is_gpu() ? ov::element::f16 : ov::element::f32;
|
||||
if (activations.get_element_type() != activations_type) {
|
||||
activations = std::make_shared<ov::op::v0::Convert>(activations, activations_type);
|
||||
@@ -244,6 +348,9 @@ OutputVector translate_mul_mat_id(const NodeContext & context) {
|
||||
if (result.get_element_type() != output_type) {
|
||||
result = std::make_shared<ov::op::v0::Convert>(result, output_type);
|
||||
}
|
||||
if (keep) {
|
||||
result = std::make_shared<ov::op::v1::Multiply>(result, *keep);
|
||||
}
|
||||
|
||||
return rename_outputs_with_suffix({result}, context.get_name());
|
||||
}
|
||||
|
||||
@@ -31,6 +31,8 @@ OutputVector translate_mulmat(const NodeContext & context) {
|
||||
|
||||
ov::Output<ov::Node> B = process_view_input_new(context, 0);
|
||||
ov::Output<ov::Node> A = process_view_input_new(context, 1);
|
||||
// the stateful V cache with `-fa off` is not transposed (see permute)
|
||||
const bool transpose_b = B.get_node()->get_rt_info().count(GGML_OV_V_TOKEN_MAJOR) == 0;
|
||||
|
||||
if (A.get_element_type() != B.get_element_type()) {
|
||||
B = std::make_shared<ov::op::v0::Convert>(B, context.get_input_type(1));
|
||||
@@ -48,6 +50,37 @@ OutputVector translate_mulmat(const NodeContext & context) {
|
||||
Output<Node> Z = A_batch_larger ? B : A;
|
||||
auto Z_shape = A_batch_larger ? B_shape : A_shape;
|
||||
int64_t factor = batch_large / batch_small;
|
||||
if (factor > 1 && batch_small > 1 && A_batch_larger && A_shape[0] == B_shape[0]) {
|
||||
// B is shared by groups of factor consecutive A matrices (grouped-query attention with `-fa off`: K and V
|
||||
// have fewer heads than Q and KQ). Stack each group along the rows of A instead of repeating B, which would
|
||||
// copy the whole K or V cache factor times per layer and token:
|
||||
// A [n, batch_large, rows, k] -> [n, batch_small, factor * rows, k] (a pure reshape: the group is adjacent)
|
||||
// A x B^T [n, batch_small, factor * rows, cols] -> [n, batch_large, rows, cols]
|
||||
// The zeros copy the input dimensions (special_zero), which may be dynamic.
|
||||
ov::Output<Node> A_grouped = std::make_shared<ov::op::v1::Reshape>(
|
||||
A, ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, batch_small, -1, 0}), true);
|
||||
// A is often a Reshape of the grouped scores (see softmax): take them directly. With one Reshape less, the
|
||||
// OpenVINO SDPAFusion can match the attention
|
||||
const auto * A_reshape = ov::as_type<ov::op::v1::Reshape>(A.get_node());
|
||||
const auto * A_target =
|
||||
A_reshape != nullptr ? ov::as_type<ov::op::v0::Constant>(A_reshape->get_input_node_ptr(1)) : nullptr;
|
||||
if (context.is_stateful() && A_target != nullptr && A_reshape->get_special_zero() &&
|
||||
A_target->cast_vector<int64_t>() == std::vector<int64_t>{0, batch_large, -1, 0} &&
|
||||
A_reshape->get_input_partial_shape(0)[1] == batch_small) {
|
||||
A_grouped = A_reshape->input_value(0);
|
||||
}
|
||||
ov::Output<Node> res = std::make_shared<ov::op::v0::MatMul>(A_grouped, B, false, transpose_b);
|
||||
res = std::make_shared<ov::op::v1::Reshape>(
|
||||
res, ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, batch_large, -1, 0}),
|
||||
true);
|
||||
// lets SOFT_MAX work on the grouped scores (see softmax)
|
||||
res.get_node()->get_rt_info()[GGML_OV_GROUPED_ROWS] = factor;
|
||||
const auto output_type = context.get_output_type();
|
||||
if (res.get_element_type() != output_type) {
|
||||
res = std::make_shared<ov::op::v0::Convert>(res, output_type);
|
||||
}
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
if (factor > 1 && batch_small > 1) {
|
||||
auto batch_large_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector<int64_t>{batch_large});
|
||||
auto batch_small_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector<int64_t>{batch_small});
|
||||
@@ -75,7 +108,6 @@ OutputVector translate_mulmat(const NodeContext & context) {
|
||||
A = Z;
|
||||
}
|
||||
|
||||
bool transpose_b = true;
|
||||
ov::Output<Node> res = std::make_shared<ov::op::v0::MatMul>(A, B, false, transpose_b);
|
||||
|
||||
const auto output_type = context.get_output_type();
|
||||
|
||||
@@ -3,11 +3,13 @@
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <openvino/op/concat.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/gather.hpp>
|
||||
#include <openvino/op/pad.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/shape_of.hpp>
|
||||
#include <openvino/op/subtract.hpp>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
@@ -20,16 +22,25 @@ namespace {
|
||||
|
||||
ov::Output<ov::Node> translate_circular_pad(ov::Output<ov::Node> input,
|
||||
const std::array<int32_t, 8> & pads,
|
||||
const ov::Shape & input_shape) {
|
||||
const ov::Shape & input_shape,
|
||||
bool stateful) {
|
||||
ov::Output<ov::Node> result = std::move(input);
|
||||
|
||||
const std::array<int32_t, 4> pads_begin = {pads[6], pads[4], pads[2], pads[0]};
|
||||
const std::array<int32_t, 4> pads_end = {pads[7], pads[5], pads[3], pads[1]};
|
||||
// Stateful models drop leading dims, so the input can have rank < 4; align pads to the trailing input axes
|
||||
const size_t dropped =
|
||||
stateful && input_shape.size() < pads_begin.size() ? pads_begin.size() - input_shape.size() : 0;
|
||||
|
||||
for (size_t i = 0; i < dropped; ++i) {
|
||||
FRONT_END_CHECK_IMPLEMENTED(pads_begin[i] == 0 && pads_end[i] == 0,
|
||||
"Circular PAD cannot pad a dimension dropped from the input rank");
|
||||
}
|
||||
|
||||
for (size_t axis = 0; axis < input_shape.size(); ++axis) {
|
||||
const int64_t input_dim = static_cast<int64_t>(input_shape[axis]);
|
||||
const int64_t pad_begin = pads_begin[axis];
|
||||
const int64_t pad_end = pads_end[axis];
|
||||
const int64_t pad_begin = pads_begin[dropped + axis];
|
||||
const int64_t pad_end = pads_end[dropped + axis];
|
||||
|
||||
if (pad_begin == 0 && pad_end == 0) {
|
||||
continue;
|
||||
@@ -72,15 +83,44 @@ OutputVector translate_pad(const NodeContext & context) {
|
||||
const bool circular = op_params[8] != 0;
|
||||
|
||||
if (circular) {
|
||||
auto res = translate_circular_pad(input, pads, context.get_input_shape(0).to_shape());
|
||||
auto res = translate_circular_pad(input, pads, context.get_input_shape(0).to_shape(), context.is_stateful());
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
|
||||
const std::vector<int64_t> pads_begin = {pads[6], pads[4], pads[2], pads[0]};
|
||||
const std::vector<int64_t> pads_end = {pads[7], pads[5], pads[3], pads[1]};
|
||||
std::vector<int64_t> pads_begin = {pads[6], pads[4], pads[2], pads[0]};
|
||||
std::vector<int64_t> pads_end = {pads[7], pads[5], pads[3], pads[1]};
|
||||
|
||||
// Stateful models drop leading dims, so the input can have rank < 4.
|
||||
const auto input_rank = input.get_partial_shape().rank();
|
||||
if (context.is_stateful() && input_rank.is_static() && input_rank.get_length() < 4) {
|
||||
const size_t dropped = 4 - static_cast<size_t>(input_rank.get_length());
|
||||
for (size_t i = 0; i < dropped; ++i) {
|
||||
FRONT_END_OP_CONVERSION_CHECK(pads_begin[i] == 0 && pads_end[i] == 0,
|
||||
"PAD: cannot pad a dimension dropped from the input rank");
|
||||
}
|
||||
pads_begin.erase(pads_begin.begin(), pads_begin.begin() + dropped);
|
||||
pads_end.erase(pads_end.begin(), pads_end.begin() + dropped);
|
||||
}
|
||||
|
||||
auto pads_begin_node = ov::op::v0::Constant::create(ov::element::i64, {pads_begin.size()}, pads_begin);
|
||||
auto pads_end_node = ov::op::v0::Constant::create(ov::element::i64, {pads_end.size()}, pads_end);
|
||||
ov::Output<ov::Node> pads_end_node = ov::op::v0::Constant::create(ov::element::i64, {pads_end.size()}, pads_end);
|
||||
// No-cache attention pads K and V on the token axis up to the mask width (TAG_NO_CACHE_PAD), which changes with
|
||||
// the token count, so compute the end padding from the mask at run time
|
||||
const int dyn = context.get_op_dynamic_dim();
|
||||
const int axis = dyn >= 0 ? static_cast<int>(pads_end.size()) - 1 - dyn : -1;
|
||||
if (axis >= 0 && pads_end[axis] != 0 && context.has_input("self_kq_mask")) {
|
||||
auto mask = context.get_input("self_kq_mask");
|
||||
auto mask_len = get_dimensions(std::make_shared<ov::op::v3::ShapeOf>(mask),
|
||||
{static_cast<int>(mask.get_partial_shape().rank().get_length()) - 1});
|
||||
auto len = get_dimensions(std::make_shared<ov::op::v3::ShapeOf>(input), {axis});
|
||||
auto pad = std::make_shared<ov::op::v1::Subtract>(mask_len, len);
|
||||
ov::OutputVector parts;
|
||||
for (int i = 0; i < static_cast<int>(pads_end.size()); ++i) {
|
||||
parts.push_back(i == axis ? ov::Output<ov::Node>(pad) :
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {pads_end[i]})->output(0));
|
||||
}
|
||||
pads_end_node = std::make_shared<ov::op::v0::Concat>(parts, 0);
|
||||
}
|
||||
auto pad_value = ov::op::v0::Constant::create(context.get_input_type(0), ov::Shape{}, {0});
|
||||
auto res =
|
||||
std::make_shared<ov::op::v1::Pad>(input, pads_begin_node, pads_end_node, pad_value, ov::op::PadMode::CONSTANT);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#include "../node_context.h"
|
||||
#include "../op_table.h"
|
||||
#include "../utils.h"
|
||||
#include "ggml-openvino/ggml-openvino-extra.h"
|
||||
|
||||
#include <climits>
|
||||
#include <cstdint>
|
||||
@@ -12,6 +13,8 @@
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/slice.hpp>
|
||||
#include <openvino/op/transpose.hpp>
|
||||
#include <openvino/op/unsqueeze.hpp>
|
||||
#include <openvino/op/variadic_split.hpp>
|
||||
#include <vector>
|
||||
|
||||
namespace ov {
|
||||
@@ -45,7 +48,45 @@ OutputVector translate_permute(const NodeContext & context) {
|
||||
static_cast<int64_t>(perm_values.size() - 1 - input_axis);
|
||||
}
|
||||
}
|
||||
if (op_case == 1 || context.is_stateful()) {
|
||||
if (context.is_stateful() && (op_case == 5 || op_case == 6)) {
|
||||
// `-fa off` reads V transposed. The stateful V cache is token-major, [1, seq, n_heads_kv, head_size]
|
||||
const auto output_shape = context.get_output_shape().to_shape();
|
||||
const int64_t n_heads = output_shape[1];
|
||||
const int64_t head_size = output_shape[2];
|
||||
const auto & state_shape = src.get_partial_shape();
|
||||
const bool single_head = n_heads == 1 && state_shape[2].get_length() == 1;
|
||||
if (!single_head && state_shape[2].get_length() * state_shape[3].get_length() != n_heads * head_size) {
|
||||
// llama.cpp pads the V cache rows to the widest layer: keep this layer's rows
|
||||
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
|
||||
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
|
||||
auto rows = std::make_shared<ov::op::v1::Reshape>(
|
||||
src, ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector<int64_t>{0, 0, -1}), true);
|
||||
auto sliced = std::make_shared<ov::op::v8::Slice>(
|
||||
rows, zero, ov::op::v0::Constant::create(ov::element::i64, {1}, {n_heads * head_size}), one,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {2}));
|
||||
src = std::make_shared<ov::op::v1::Reshape>(
|
||||
sliced,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, 0, n_heads, head_size}),
|
||||
true);
|
||||
}
|
||||
// keep V as [1, n_heads_kv, kv, head_size]: MUL_MAT then multiplies without transposing it
|
||||
res = std::make_shared<ov::op::v1::Transpose>(
|
||||
src, ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, 2, 1, 3}));
|
||||
if (single_head && state_shape[3].get_length() != head_size) {
|
||||
// the padded rows, as above. After the Transpose, which then cancels the state relayout, so the Slice reads
|
||||
// the KV cache in place: between the two they become copies of the cache, recompiled for every length
|
||||
res = std::make_shared<ov::op::v8::Slice>(res, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {head_size}),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
|
||||
}
|
||||
res.get_node()->get_rt_info()[GGML_OV_V_TOKEN_MAJOR] = true;
|
||||
} else if (op_case == 1 || context.is_stateful()) {
|
||||
if (op_case == 2 && src.get_partial_shape().rank() == 3) {
|
||||
// Stateful RoPE lifts Q to rank 4 (see translate_rope). Do the same for Q of a layer without RoPE (NoPE)
|
||||
src = std::make_shared<ov::op::v0::Unsqueeze>(src,
|
||||
ov::op::v0::Constant::create(ov::element::i64, Shape{}, {0}));
|
||||
}
|
||||
// The stateful path carries hidden-state tensors in a rank-3 layout (the
|
||||
// leading batch dim is dropped, e.g. Gemma4's per-layer-embedding path). The
|
||||
// perm above is rank-4; when the actual input is rank-3, drop the batch axis
|
||||
@@ -78,6 +119,27 @@ OutputVector translate_permute(const NodeContext & context) {
|
||||
|
||||
auto reshaped = std::make_shared<ov::op::v1::Reshape>(src, new_shape, true);
|
||||
res = std::make_shared<ov::op::v1::Transpose>(reshaped, perm);
|
||||
} else if ((op_case == 3 || op_case == 4) && is_kvcache_4d_shape(src.get_partial_shape())) {
|
||||
// cache [1, ctx, n_heads_kv, head_size] (see set_rows): take the first attention_size tokens with
|
||||
// VariadicSplit, the read side of the GPU plugin's StatelessKVFusion. The fused StatelessKV then hands
|
||||
// attention a view of those tokens, with no copy
|
||||
FRONT_END_OP_CONVERSION_CHECK((perm_values == std::vector<int64_t>{0, 2, 1, 3}),
|
||||
"Unsupported permutation of a 4-D KV cache");
|
||||
Output<Node> attention_size;
|
||||
if (!context.has_input("attention_size")) {
|
||||
attention_size =
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {context.get_output_shape().to_shape()[2]});
|
||||
} else if (op_case == 3) {
|
||||
attention_size = context.get_input("attention_size");
|
||||
} else {
|
||||
attention_size = context.get_input("attention_size_swa");
|
||||
}
|
||||
auto split_lengths = std::make_shared<ov::op::v0::Concat>(
|
||||
ov::OutputVector{attention_size, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1})}, 0);
|
||||
auto tokens = std::make_shared<ov::op::v1::VariadicSplit>(
|
||||
src, ov::op::v0::Constant::create(ov::element::i64, {}, {1}), split_lengths);
|
||||
res = std::make_shared<ov::op::v1::Transpose>(tokens->output(0),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values));
|
||||
} else {
|
||||
auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values);
|
||||
auto cache_shape = src.get_partial_shape();
|
||||
@@ -93,7 +155,9 @@ OutputVector translate_permute(const NodeContext & context) {
|
||||
|
||||
Output<Node> attention_size;
|
||||
if (!context.has_input("attention_size")) {
|
||||
attention_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[2]});
|
||||
// the attended tokens are the last axis of the transposed V cache (`-fa off`)
|
||||
const size_t tokens_axis = (op_case == 5 || op_case == 6) ? 3 : 2;
|
||||
attention_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[tokens_axis]});
|
||||
} else if (op_case == 3 || op_case == 5) {
|
||||
attention_size = context.get_input("attention_size");
|
||||
} else {
|
||||
@@ -121,7 +185,37 @@ OutputVector translate_permute(const NodeContext & context) {
|
||||
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
|
||||
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
|
||||
|
||||
if (op_case == 3 || op_case == 4) {
|
||||
if ((op_case == 3 || op_case == 4) && context.is_static() && ggml_openvino_npu_kv_head_major_enabled() &&
|
||||
cache_shape.rank().get_length() == 4 && cache_shape[1].is_static() &&
|
||||
cache_shape[1].get_length() == n_heads && n_heads > 1) {
|
||||
// Head-major cache [1, heads, rows, head_size] is already in attention layout: only cut the window.
|
||||
// A transposed V cache [1, heads, head_size, rows] is cut on axis 3 and stays transposed.
|
||||
const bool v_transposed = context.get_input_names()[0].find("cache_v_l") != std::string::npos;
|
||||
auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {v_transposed ? 3 : 2});
|
||||
res = std::make_shared<ov::op::v8::Slice>(src, zero, attention_size, one, axis);
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
|
||||
if ((op_case == 3 || op_case == 4) && n_seq == 1 && perm_values == std::vector<int64_t>{0, 2, 1, 3}) {
|
||||
// one sequence: slice the cache [1, 1, ctx, n_heads * head_size] on its context axis, then split the
|
||||
// heads. The reshape after the slice also restates the head count and head size: the GPU plugin loses
|
||||
// all dimensions of a slice with a runtime end, and SDPA would then run its reference kernel
|
||||
auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});
|
||||
auto sliced = std::make_shared<ov::op::v8::Slice>(src, zero, attention_size, one, two);
|
||||
if (n_heads == 1) {
|
||||
res = std::make_shared<ov::op::v1::Reshape>(
|
||||
sliced,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{1, 1, -1, head_size}),
|
||||
false);
|
||||
} else {
|
||||
auto split = std::make_shared<ov::op::v1::Reshape>(
|
||||
sliced,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4},
|
||||
std::vector<int64_t>{1, -1, n_heads, head_size}),
|
||||
false);
|
||||
res = std::make_shared<ov::op::v1::Transpose>(split, perm);
|
||||
}
|
||||
} else if (op_case == 3 || op_case == 4) {
|
||||
auto src_reshaped = std::make_shared<ov::op::v1::Reshape>(
|
||||
src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, ctx_per_seq, n_heads, head_size}),
|
||||
false);
|
||||
@@ -133,11 +227,26 @@ OutputVector translate_permute(const NodeContext & context) {
|
||||
std::make_shared<ov::op::v8::Slice>(src_reshaped, seq_active_start, seq_active_end, one, zero);
|
||||
}
|
||||
auto slice2 = std::make_shared<ov::op::v8::Slice>(after_seq_slice, zero, attention_size, one, one);
|
||||
res = std::make_shared<ov::op::v1::Transpose>(slice2, perm);
|
||||
// restate the head count and head size, see above
|
||||
auto restated = std::make_shared<ov::op::v1::Reshape>(
|
||||
slice2,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector<int64_t>{0, -1, n_heads, head_size}),
|
||||
true);
|
||||
res = std::make_shared<ov::op::v1::Transpose>(restated, perm);
|
||||
} else {
|
||||
auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3});
|
||||
// the transposed V cache holds one row per value element, [n_seq, rows, ctx]. A model whose layers have
|
||||
// different V sizes allocates the rows for the widest layer (gemma-4: 512, sliding-window layers use the
|
||||
// first 256), so keep this layer's n_heads * head_size rows before splitting them into heads
|
||||
const int64_t cache_rows = cache_shape[3].get_length();
|
||||
ov::Output<ov::Node> v_rows = std::make_shared<ov::op::v1::Reshape>(
|
||||
src, ov::op::v0::Constant::create(ov::element::i64, {3}, {n_seq, cache_rows, ctx_per_seq}), false);
|
||||
if (cache_rows != n_heads * head_size) {
|
||||
v_rows = std::make_shared<ov::op::v8::Slice>(
|
||||
v_rows, zero, ov::op::v0::Constant::create(ov::element::i64, {1}, {n_heads * head_size}), one, one);
|
||||
}
|
||||
auto src_reshaped = std::make_shared<ov::op::v1::Reshape>(
|
||||
src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, n_heads, head_size, ctx_per_seq}),
|
||||
v_rows, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, n_heads, head_size, ctx_per_seq}),
|
||||
false);
|
||||
ov::Output<ov::Node> after_seq_slice;
|
||||
if (n_seq == 1) {
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
#include "../node_context.h"
|
||||
#include "../op_table.h"
|
||||
#include "../utils.h"
|
||||
|
||||
#include <openvino/op/avg_pool.hpp>
|
||||
#include <openvino/op/convert.hpp>
|
||||
#include <openvino/op/max_pool.hpp>
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
namespace ggml {
|
||||
namespace op {
|
||||
|
||||
OutputVector translate_pool_1d(const NodeContext & context) {
|
||||
num_inputs_check(context, 1, 1);
|
||||
const int32_t * params = context.get_output_op_params();
|
||||
|
||||
const int k0 = params[1];
|
||||
const int s0 = params[2];
|
||||
const int p0 = params[3];
|
||||
|
||||
const int op_case = context.get_op_case();
|
||||
FRONT_END_OP_CONVERSION_CHECK(op_case == 1 || op_case == 2, "Unsupported POOL_1D mode");
|
||||
|
||||
ov::Output<Node> input = process_view_input_new(context, 0);
|
||||
|
||||
// OV pools over the last two axes; pool only along last one
|
||||
ov::Strides strides{1, static_cast<size_t>(s0)};
|
||||
ov::Shape pads{0, static_cast<size_t>(p0)};
|
||||
ov::Shape kernel{1, static_cast<size_t>(k0)};
|
||||
ov::Output<Node> res;
|
||||
if (op_case == 1) {
|
||||
res = std::make_shared<ov::op::v1::MaxPool>(input, strides, pads, pads, kernel);
|
||||
} else {
|
||||
// ggml divides by the count of valid (not padded) cells
|
||||
res = std::make_shared<ov::op::v1::AvgPool>(input, strides, pads, pads, kernel, true);
|
||||
}
|
||||
|
||||
if (res.get_element_type() != context.get_output_type()) {
|
||||
res = std::make_shared<ov::op::v0::Convert>(res, context.get_output_type());
|
||||
}
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
|
||||
} // namespace op
|
||||
} // namespace ggml
|
||||
} // namespace frontend
|
||||
} // namespace ov
|
||||
@@ -0,0 +1,65 @@
|
||||
// Copyright (C) 2026 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
#pragma once
|
||||
#include "openvino/op/op.hpp"
|
||||
|
||||
namespace ov::op::internal {
|
||||
/// Qwen3.5/Qwen3.6 llama.cpp convolution and generic cache_r maintenance.
|
||||
/// Inputs: original cache, active rows, extra rows, reset begin/length,
|
||||
/// extra offsets, QKV [1,B,C,T], windows and depthwise weights.
|
||||
/// Nine inputs; output 0 is convolution [1,B,T,C].
|
||||
/// Cache input 0 is a writable GPU RemoteTensor, updated in place.
|
||||
/// Other inputs and output 0 must not overlap its allocation.
|
||||
/// The cache must be a Parameter or an exclusive Reshape alias; no other op
|
||||
/// may read it in the same inference. Required original rows are staged before
|
||||
/// reset, reorder and window writes, preserving cycles and duplicate sources.
|
||||
/// A failed inference may partially modify the cache; clear or restore it before retrying.
|
||||
class OPENVINO_API RecurrentConvCacheUpdate : public Op {
|
||||
public:
|
||||
OPENVINO_OP("RecurrentConvCacheUpdate", "internal")
|
||||
RecurrentConvCacheUpdate() = default;
|
||||
RecurrentConvCacheUpdate(const OutputVector & args, int64_t history);
|
||||
void validate_and_infer_types() override;
|
||||
bool visit_attributes(AttributeVisitor &) override;
|
||||
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector &) const override;
|
||||
|
||||
int64_t get_history() const { return m_history; }
|
||||
|
||||
private:
|
||||
int64_t m_history = 0;
|
||||
};
|
||||
|
||||
/// Qwen3.5/Qwen3.6 llama.cpp GDN and generic cache_s maintenance.
|
||||
/// Twelve inputs: Q/K/V, cache, gate/beta,
|
||||
/// active/extra rows, reset begin/length, extra offsets, ordered write control.
|
||||
/// The only output is attention [B,T,H,V]. Cache input 3 is updated in place
|
||||
/// under the same writable-input contract as RecurrentConvCacheUpdate.
|
||||
/// snapshot_count saves the newest min(T,count) states in cache, newest first.
|
||||
/// Active/nonidentity rows are staged before writes; identity rows need no copy.
|
||||
class OPENVINO_API RecurrentGatedDeltaNetUpdate : public Op {
|
||||
public:
|
||||
OPENVINO_OP("RecurrentGatedDeltaNetUpdate", "internal")
|
||||
RecurrentGatedDeltaNetUpdate() = default;
|
||||
RecurrentGatedDeltaNetUpdate(const OutputVector & args,
|
||||
bool fuse_qk_l2norm = false,
|
||||
float q_l2_norm_eps = 1e-6f,
|
||||
float k_l2_norm_eps = 1e-6f,
|
||||
int64_t snapshot_count = 1);
|
||||
void validate_and_infer_types() override;
|
||||
bool visit_attributes(AttributeVisitor &) override;
|
||||
std::shared_ptr<Node> clone_with_new_inputs(const OutputVector &) const override;
|
||||
|
||||
bool get_fuse_qk_l2norm() const { return m_fuse_qk_l2norm; }
|
||||
|
||||
float get_q_l2_norm_eps() const { return m_q_l2_norm_eps; }
|
||||
|
||||
float get_k_l2_norm_eps() const { return m_k_l2_norm_eps; }
|
||||
|
||||
int64_t get_snapshot_count() const { return m_snapshot_count; }
|
||||
|
||||
private:
|
||||
bool m_fuse_qk_l2norm = false;
|
||||
float m_q_l2_norm_eps = 1e-6f, m_k_l2_norm_eps = 1e-6f;
|
||||
int64_t m_snapshot_count = 1;
|
||||
};
|
||||
} // namespace ov::op::internal
|
||||
@@ -70,6 +70,21 @@ OutputVector translate_reshape(const NodeContext & context) {
|
||||
case 8:
|
||||
shape[0] = -1;
|
||||
break;
|
||||
case 10:
|
||||
if (context.has_input("s_copy_active_slot_len")) {
|
||||
new_shape_node = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{
|
||||
context.get_input("s_copy_active_slot_len"),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {3},
|
||||
std::vector<int64_t>{-1, shape[2], shape[3]})}, 0);
|
||||
} else {
|
||||
shape[1] = -1;
|
||||
}
|
||||
break;
|
||||
case 11:
|
||||
// ROPE positions keep the layout of inp_pos. gemma-4 reshapes the positions of its output rows, and a
|
||||
// batch can have no output rows, so the size checks of the other cases cannot place the dim.
|
||||
shape = {1, 1, 1, -1};
|
||||
break;
|
||||
default:
|
||||
FRONT_END_OP_CONVERSION_CHECK(false, "Unsupported RESHAPE case: ", op_case);
|
||||
}
|
||||
|
||||
@@ -1,18 +1,23 @@
|
||||
#include "../node_context.h"
|
||||
#include "../op_table.h"
|
||||
#include "../utils.h"
|
||||
#include "ggml-openvino/ggml-openvino-extra.h"
|
||||
|
||||
#include <memory>
|
||||
#include <openvino/op/add.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/concat.hpp>
|
||||
#include <openvino/op/divide.hpp>
|
||||
#include <openvino/op/gather.hpp>
|
||||
#include <openvino/op/multiply.hpp>
|
||||
#include <openvino/op/negative.hpp>
|
||||
#include <openvino/op/power.hpp>
|
||||
#include <openvino/op/range.hpp>
|
||||
#include <openvino/op/reduce_mean.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/slice.hpp>
|
||||
#include <openvino/op/sqrt.hpp>
|
||||
#include <openvino/op/squeeze.hpp>
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
@@ -25,23 +30,20 @@ OutputVector translate_rms_norm(const NodeContext & context) {
|
||||
auto op_case = context.get_op_case();
|
||||
|
||||
ov::Output<ov::Node> input_node;
|
||||
if (op_case == 3) {
|
||||
// Flatten sequence and token dimensions to match the gate layout.
|
||||
auto input_shape = context.get_input_shape(0).to_shape();
|
||||
input_node = std::make_shared<ov::op::v1::Reshape>(
|
||||
context.get_input(0),
|
||||
ov::op::v0::Constant::create(
|
||||
ov::element::i64, {4}, std::vector<int64_t>{1, -1, (int64_t) input_shape[2], (int64_t) input_shape[3]}),
|
||||
false);
|
||||
if (op_case == 3 || op_case == 4) {
|
||||
input_node = context.get_input(0);
|
||||
} else if (op_case == 1) {
|
||||
input_node = process_view_input_new(context, 0);
|
||||
} else if (op_case == 2) {
|
||||
} else if (op_case == 2 || op_case == 3) {
|
||||
auto ssm_state_size = context.get_ssm_state_size();
|
||||
// The GDN op packs [attn | new_state] along the row axis; the state occupies the last
|
||||
// ssm_state_size * n_seqs rows. Slice it off (scaling by the active sequence count) to keep
|
||||
// just the attention output.
|
||||
ov::Output<ov::Node> state_end;
|
||||
if (context.has_input("s_copy_active_slot_len")) {
|
||||
const auto attention_rows_name = "gdn_attention_rows_" + context.get_view_input_src_name(0, 0);
|
||||
if (context.has_input(attention_rows_name)) {
|
||||
state_end = context.get_input(attention_rows_name);
|
||||
} else if (context.has_input("s_copy_active_slot_len")) {
|
||||
auto len = context.get_input("s_copy_active_slot_len");
|
||||
auto state_rows = std::make_shared<ov::op::v1::Multiply>(
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {ssm_state_size}), len);
|
||||
@@ -49,29 +51,49 @@ OutputVector translate_rms_norm(const NodeContext & context) {
|
||||
} else {
|
||||
state_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {-ssm_state_size});
|
||||
}
|
||||
auto gdn_attn_output = std::make_shared<ov::op::v8::Slice>(
|
||||
context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {1}, {0}), state_end,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {2}));
|
||||
ov::Output<ov::Node> gdn_attn_output;
|
||||
if (context.has_input(attention_rows_name)) {
|
||||
auto indices = std::make_shared<ov::op::v4::Range>(
|
||||
ov::op::v0::Constant::create(ov::element::i64, {}, {0}),
|
||||
std::make_shared<ov::op::v0::Squeeze>(state_end),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {}, {1}), ov::element::i64);
|
||||
gdn_attn_output = std::make_shared<ov::op::v8::Gather>(context.get_input(0), indices,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {}, {2}));
|
||||
} else {
|
||||
gdn_attn_output = std::make_shared<ov::op::v8::Slice>(
|
||||
context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {1}, {0}), state_end,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {2}));
|
||||
}
|
||||
|
||||
auto input_shape = context.get_input_shape(0).to_shape();
|
||||
input_node = std::make_shared<ov::op::v1::Reshape>(
|
||||
gdn_attn_output,
|
||||
ov::op::v0::Constant::create(
|
||||
ov::element::i64, {4}, std::vector<int64_t>{1, -1, (int64_t) input_shape[2], (int64_t) input_shape[3]}),
|
||||
false);
|
||||
ov::Output<ov::Node> sequences = ov::op::v0::Constant::create(ov::element::i64, {1}, {input_shape[0]});
|
||||
if (context.has_input("s_copy_active_slot_len")) {
|
||||
sequences = context.get_input("s_copy_active_slot_len");
|
||||
}
|
||||
auto shape = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{
|
||||
sequences, ov::op::v0::Constant::create(ov::element::i64, {3},
|
||||
std::vector<int64_t>{-1, (int64_t) input_shape[2], (int64_t) input_shape[3]})}, 0);
|
||||
input_node = std::make_shared<ov::op::v1::Reshape>(gdn_attn_output, shape, false);
|
||||
|
||||
} else {
|
||||
input_node = process_view_input_new(context, 0);
|
||||
}
|
||||
|
||||
float eps;
|
||||
memcpy(&eps, context.get_output_op_params(), sizeof(float));
|
||||
|
||||
if (op_case == 3 && ggml_openvino_gdn_prescale_enabled()) {
|
||||
// input is the GDN output pre-scaled by ggml_openvino_gdn_q_scale(): rms_norm(s*x, s*s*eps) == rms_norm(x, eps)
|
||||
const float q_scale = ggml_openvino_gdn_q_scale();
|
||||
eps *= q_scale * q_scale;
|
||||
}
|
||||
|
||||
auto square = std::make_shared<ov::op::v1::Multiply>(input_node, input_node);
|
||||
|
||||
auto mean = std::make_shared<ov::op::v1::ReduceMean>(
|
||||
square, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true);
|
||||
|
||||
float eps;
|
||||
memcpy(&eps, context.get_output_op_params(), sizeof(float));
|
||||
|
||||
auto rms = std::make_shared<ov::op::v0::Sqrt>(
|
||||
std::make_shared<ov::op::v1::Add>(mean, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps})));
|
||||
|
||||
|
||||
@@ -60,6 +60,8 @@ OutputVector translate_rope(const NodeContext & context) {
|
||||
if (context.get_input_size() == 3) {
|
||||
cache_key += "_ff_" + context.get_input_names()[2];
|
||||
}
|
||||
// Not every ROPE takes inp_pos: gemma-4 rotates its KV-shared layers at the positions of the output rows only
|
||||
cache_key += "_pos_" + context.get_input_names()[1];
|
||||
if (context.has_input(cache_key + "_cos")) {
|
||||
cos_theta_node = context.get_input(cache_key + "_cos");
|
||||
sin_theta_node = context.get_input(cache_key + "_sin");
|
||||
|
||||
@@ -2,24 +2,9 @@
|
||||
#include "../op_table.h"
|
||||
#include "../utils.h"
|
||||
|
||||
#include <openvino/core/except.hpp>
|
||||
#include <openvino/op/add.hpp>
|
||||
#include <openvino/op/concat.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/convert.hpp>
|
||||
#include <openvino/op/equal.hpp>
|
||||
#include <openvino/op/gather.hpp>
|
||||
#include <openvino/op/greater_eq.hpp>
|
||||
#include <openvino/op/if.hpp>
|
||||
#include <openvino/op/less.hpp>
|
||||
#include <openvino/op/logical_or.hpp>
|
||||
#include <openvino/op/multiply.hpp>
|
||||
#include <openvino/op/range.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/shape_of.hpp>
|
||||
#include <openvino/op/slice.hpp>
|
||||
#include <openvino/op/squeeze.hpp>
|
||||
#include <openvino/op/unsqueeze.hpp>
|
||||
#include <vector>
|
||||
|
||||
namespace ov {
|
||||
@@ -37,37 +22,8 @@ OutputVector translate_scale(const NodeContext & context) {
|
||||
|
||||
auto scale_node = std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{}, std::vector<float>{scale});
|
||||
|
||||
if (context.get_op_case() == 2) {
|
||||
return {context.get_input(0)};
|
||||
}
|
||||
|
||||
if (context.get_op_case() == 1 && context.has_input("cache_rs_reset_len")) {
|
||||
auto cache_rs_reset_idx = context.get_input("cache_rs_reset_idx");
|
||||
auto cache_rs_reset_len = context.get_input("cache_rs_reset_len");
|
||||
|
||||
auto cache_rs = context.get_input(0);
|
||||
|
||||
auto cache_shape = std::make_shared<ov::op::v3::ShapeOf>(cache_rs, ov::element::i64);
|
||||
auto n_slots_1d = std::make_shared<ov::op::v8::Gather>(
|
||||
cache_shape, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {2}),
|
||||
ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {0}));
|
||||
auto n_slots = std::make_shared<ov::op::v0::Squeeze>(n_slots_1d);
|
||||
|
||||
auto iota = std::make_shared<ov::op::v4::Range>(
|
||||
ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {0}), n_slots,
|
||||
ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {1}), ov::element::i64);
|
||||
|
||||
auto idx_plus_len = std::make_shared<ov::op::v1::Add>(cache_rs_reset_idx, cache_rs_reset_len);
|
||||
auto less_than_idx = std::make_shared<ov::op::v1::Less>(iota, cache_rs_reset_idx);
|
||||
auto greater_equal_idx_plus_len = std::make_shared<ov::op::v1::GreaterEqual>(iota, idx_plus_len);
|
||||
auto keep_mask = std::make_shared<ov::op::v1::LogicalOr>(less_than_idx, greater_equal_idx_plus_len);
|
||||
|
||||
auto keep_mask_f32 = std::make_shared<ov::op::v0::Convert>(keep_mask, ov::element::f32);
|
||||
auto keep_mask_reshape = std::make_shared<ov::op::v0::Unsqueeze>(
|
||||
keep_mask_f32, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {1}));
|
||||
|
||||
auto cleared_cache_rs = std::make_shared<ov::op::v1::Multiply>(cache_rs, keep_mask_reshape);
|
||||
return rename_outputs_with_suffix({cleared_cache_rs}, context.get_name());
|
||||
if (is_recurrent_cache_reset_case(context)) {
|
||||
return translate_recurrent_cache_reset(context);
|
||||
}
|
||||
|
||||
auto scaled = std::make_shared<ov::op::v1::Multiply>(process_view_input_new(context, 0), scale_node);
|
||||
|
||||
@@ -1,26 +1,35 @@
|
||||
#include "../node_context.h"
|
||||
#include "../op_table.h"
|
||||
#include "../utils.h"
|
||||
#include "ggml-openvino/ggml-openvino-extra.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <numeric>
|
||||
#include <openvino/core/node.hpp>
|
||||
#include <openvino/core/node_output.hpp>
|
||||
#include <openvino/core/type.hpp>
|
||||
#include <openvino/frontend/exception.hpp>
|
||||
#include <openvino/op/add.hpp>
|
||||
#include <openvino/op/broadcast.hpp>
|
||||
#include <openvino/op/concat.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/convert.hpp>
|
||||
#include <openvino/op/divide.hpp>
|
||||
#include <openvino/op/gather.hpp>
|
||||
#include <openvino/op/mod.hpp>
|
||||
#include <openvino/op/multiply.hpp>
|
||||
#include <openvino/op/range.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/scatter_elements_update.hpp>
|
||||
#include <openvino/op/scatter_update.hpp>
|
||||
#include <openvino/op/shape_of.hpp>
|
||||
#include <openvino/op/slice.hpp>
|
||||
#include <openvino/op/squeeze.hpp>
|
||||
#include <openvino/op/subtract.hpp>
|
||||
#include <openvino/op/transpose.hpp>
|
||||
#include <openvino/op/unsqueeze.hpp>
|
||||
#include <vector>
|
||||
|
||||
namespace ov {
|
||||
@@ -28,12 +37,68 @@ namespace frontend {
|
||||
namespace ggml {
|
||||
namespace op {
|
||||
|
||||
static OutputVector translate_conv_snapshots(const NodeContext & context) {
|
||||
using namespace ov::op;
|
||||
FRONT_END_OP_CONVERSION_CHECK(!context.is_static(), "Convolution snapshots require dynamic execution");
|
||||
auto constant = [](std::initializer_list<size_t> shape, std::initializer_list<int64_t> values) {
|
||||
return v0::Constant::create(element::i64, Shape(shape), std::vector<int64_t>(values));
|
||||
};
|
||||
const auto history = context.get_input_shape(0)[3].get_length();
|
||||
const auto features = history * context.get_input_shape(0)[1].get_length();
|
||||
const auto name = context.get_name();
|
||||
auto count = context.get_input("rs_window_count_" + name);
|
||||
auto seqs = context.get_input("rs_seq_count_" + name);
|
||||
auto zero = constant({}, {0});
|
||||
auto one = constant({}, {1});
|
||||
auto range = [&](const Output<Node> & end) {
|
||||
return std::make_shared<v4::Range>(zero, std::make_shared<v0::Squeeze>(end), one, element::i64);
|
||||
};
|
||||
|
||||
// VIEW passes through conv_x; gather overlapping windows in [window, seq, channel, history] order.
|
||||
// Return the full cache to preserve the in-place alias chain.
|
||||
auto begin = context.get_input("rs_src_begin_" + name);
|
||||
auto positions = range(std::make_shared<v1::Multiply>(count, constant({}, {history})));
|
||||
auto columns = std::make_shared<v1::Add>(begin,
|
||||
std::make_shared<v1::Add>(std::make_shared<v1::Divide>(positions, constant({}, {history})),
|
||||
std::make_shared<v1::Mod>(positions, constant({}, {history}))));
|
||||
auto column_matrix = std::make_shared<v1::Reshape>(columns, constant({2}, {-1, history}), false);
|
||||
auto windows = std::make_shared<v8::Gather>(context.get_input(0), column_matrix, constant({}, {3}));
|
||||
auto ordered = std::make_shared<v1::Transpose>(
|
||||
std::make_shared<v0::Squeeze>(windows, constant({1}, {0})), constant({4}, {2, 0, 1, 3}));
|
||||
Output<Node> updates = std::make_shared<v1::Reshape>(ordered, constant({4}, {1, 1, -1, features}), false);
|
||||
updates = std::make_shared<v0::Convert>(updates, context.get_output_type());
|
||||
|
||||
auto row = range(std::make_shared<v1::Multiply>(count, seqs));
|
||||
auto plane = std::make_shared<v1::Subtract>(std::make_shared<v1::Subtract>(count, one),
|
||||
std::make_shared<v1::Divide>(row, seqs));
|
||||
auto slots = std::make_shared<v1::Add>(context.get_input("rs_slot_begin_" + name),
|
||||
std::make_shared<v1::Add>(std::make_shared<v1::Mod>(row, seqs),
|
||||
std::make_shared<v1::Multiply>(plane,
|
||||
context.get_input("rs_snapshot_stride_" + name))));
|
||||
auto base = context.get_input(2);
|
||||
const auto full_features = base.get_partial_shape()[3].get_length();
|
||||
const auto feature_begin = (context.get_view_input_offset(2, 0) / context.get_output_type().size()) % full_features;
|
||||
if (features != full_features) {
|
||||
auto old_rows = std::make_shared<v8::Gather>(base, slots, constant({}, {2}));
|
||||
auto head = std::make_shared<v8::Slice>(old_rows, constant({1}, {0}),
|
||||
constant({1}, {int64_t(feature_begin)}), constant({1}, {1}), constant({1}, {3}));
|
||||
auto tail = std::make_shared<v8::Slice>(old_rows, constant({1}, {int64_t(feature_begin) + features}),
|
||||
constant({1}, {INT64_MAX}), constant({1}, {1}), constant({1}, {3}));
|
||||
updates = std::make_shared<v0::Concat>(OutputVector{head, updates, tail}, 3);
|
||||
}
|
||||
auto result = std::make_shared<v3::ScatterUpdate>(base, slots, updates, constant({}, {2}));
|
||||
return rename_outputs_with_suffix({result}, name);
|
||||
}
|
||||
|
||||
OutputVector translate_set_rows(const NodeContext & context) {
|
||||
num_inputs_check(context, 3, 3);
|
||||
|
||||
if (context.get_op_case() == 3) {
|
||||
return translate_conv_snapshots(context);
|
||||
}
|
||||
|
||||
auto data = process_view_input_new(context, 0);
|
||||
auto indices = process_view_input_new(context, 1);
|
||||
auto dst = process_view_input_new(context, 2);
|
||||
|
||||
data = std::make_shared<ov::op::v0::Convert>(data, context.get_output_type());
|
||||
|
||||
@@ -43,41 +108,143 @@ OutputVector translate_set_rows(const NodeContext & context) {
|
||||
((indices_shape[1].is_static() && indices_shape[1].get_length() > 1) ||
|
||||
(indices_shape[2].is_static() && indices_shape[2].get_length() > 1));
|
||||
|
||||
// On NPU ScatterUpdate is not in-place: it copies the whole destination to write
|
||||
// one KV row, which dominates decode once ctx grows. ScatterElementsUpdate is
|
||||
// measurably cheaper for the same single-row write.
|
||||
const bool use_elements_update =
|
||||
!multidim_indices && !context.is_stateful() && ggml_openvino_getenv_int("GGML_OPENVINO_KV_SCATTER_ELEMENTS");
|
||||
|
||||
auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {2});
|
||||
|
||||
const bool is_cache_write = context.get_op_case() != 2;
|
||||
|
||||
if (is_cache_write && context.is_static() && !context.is_stateful() && !multidim_indices &&
|
||||
ggml_openvino_npu_kv_head_major_enabled()) {
|
||||
// Head-major cache [1, heads, rows, head_size]: write the new rows along axis 2 without reshaping the cache.
|
||||
ov::Output<Node> cache = context.get_input(2);
|
||||
while (auto reshape = ov::as_type_ptr<ov::op::v1::Reshape>(cache.get_node_shared_ptr())) {
|
||||
cache = reshape->input_value(0);
|
||||
}
|
||||
const auto cache_shape = cache.get_partial_shape();
|
||||
if (cache_shape.rank().is_static() && cache_shape.rank().get_length() == 4 && cache_shape[1].is_static() &&
|
||||
cache_shape[3].is_static() && cache_shape[1].get_length() > 1) {
|
||||
const int64_t heads = cache_shape[1].get_length();
|
||||
// Transposed V cache is [1, heads, head_size, rows]: rows go along axis 3.
|
||||
const bool v_transposed = cache.get_node()->get_friendly_name().rfind("cache_v_l", 0) == 0;
|
||||
const int64_t head_size = (v_transposed ? cache_shape[2] : cache_shape[3]).get_length();
|
||||
auto ind_squeezed = std::make_shared<ov::op::v0::Squeeze>(
|
||||
indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2}));
|
||||
auto rows = std::make_shared<ov::op::v1::Reshape>(
|
||||
data,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) -1, heads, head_size}),
|
||||
false);
|
||||
auto rows_bhsd = std::make_shared<ov::op::v1::Transpose>(
|
||||
rows, ov::op::v0::Constant::create(ov::element::i64, {4},
|
||||
v_transposed ? std::vector<int64_t>{0, 2, 3, 1} :
|
||||
std::vector<int64_t>{0, 2, 1, 3}));
|
||||
if (v_transposed && context.is_prefill() && ggml_openvino_npu_hfa_prefill_enabled()) {
|
||||
// On NPU a chunk-sized scatter along the innermost axis of [1, heads, head_size, rows] is very slow:
|
||||
// the HFA prefill outputs the chunk's V^T rows [1, heads, head_size, n] instead and the host copies
|
||||
// them into the cache (as GenAI does for its prefill KV).
|
||||
return rename_outputs_with_suffix({rows_bhsd}, context.get_name());
|
||||
}
|
||||
Output<Node> res = std::make_shared<ov::op::v3::ScatterUpdate>(
|
||||
cache, ind_squeezed, rows_bhsd,
|
||||
v_transposed ? ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {3}) : axes);
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
}
|
||||
|
||||
auto dst = process_view_input_new(context, 2);
|
||||
Output<Node> res;
|
||||
if (context.is_stateful()) {
|
||||
if (is_cache_write && context.is_stateful()) {
|
||||
int concat_axis = 1;
|
||||
if (context.get_op_case() == 1) {
|
||||
// `-fa off` writes V transposed through a [1, n] reshape of the cache. The state keeps V token-major
|
||||
// like K, and the new values are already in token order, so append them to the state itself
|
||||
dst = dst.get_node()->input_value(0);
|
||||
}
|
||||
FRONT_END_OP_CONVERSION_CHECK(dst.get_partial_shape()[2].is_static() && dst.get_partial_shape()[3].is_static(),
|
||||
"Unsupported KV cache layout for stateful execution");
|
||||
int64_t dim2 = dst.get_partial_shape()[2].get_length();
|
||||
int64_t dim3 = dst.get_partial_shape()[3].get_length();
|
||||
data = std::make_shared<ov::op::v1::Reshape>(
|
||||
data, ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) -1, dim2, dim3}), false);
|
||||
res = std::make_shared<ov::op::v0::Concat>(OutputVector{dst, data}, concat_axis);
|
||||
} else if (multidim_indices) {
|
||||
} else if (is_cache_write && is_kvcache_4d_shape(dst.get_partial_shape())) {
|
||||
// cache [1, ctx, n_heads_kv, head_size]: write whole tokens on axis 1. This is the pattern of the GPU
|
||||
// plugin's StatelessKVFusion (1-D axis, read with VariadicSplit, see permute), which writes the new rows in
|
||||
// place instead of copying the whole cache first
|
||||
FRONT_END_OP_CONVERSION_CHECK(!multidim_indices, "SET_ROWS into a 4-D KV cache needs one index per token");
|
||||
const int64_t n_heads_kv = dst.get_partial_shape()[2].get_length();
|
||||
const int64_t head_size = dst.get_partial_shape()[3].get_length();
|
||||
auto data_tokens = std::make_shared<ov::op::v1::Reshape>(
|
||||
data,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) -1, n_heads_kv, head_size}),
|
||||
false);
|
||||
auto ind_squeezed = std::make_shared<ov::op::v0::Squeeze>(
|
||||
indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2}));
|
||||
res = std::make_shared<ov::op::v3::ScatterUpdate>(dst, ind_squeezed, data_tokens,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {1}));
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
} else if (multidim_indices || !is_cache_write) {
|
||||
// updates [.., n_rows, row_size] with one index per row; stateful models may drop the leading 1 of the
|
||||
// updates but not of the destination
|
||||
const auto dst_rank = dst.get_partial_shape().rank();
|
||||
const auto data_rank = data.get_partial_shape().rank();
|
||||
FRONT_END_OP_CONVERSION_CHECK(dst_rank.is_static() && data_rank.is_static() &&
|
||||
data_rank.get_length() <= dst_rank.get_length(),
|
||||
"SET_ROWS with per-row indices: updates ", data.get_partial_shape(),
|
||||
" do not fit the destination ", dst.get_partial_shape());
|
||||
if (data_rank.get_length() < dst_rank.get_length()) {
|
||||
std::vector<int64_t> lead_axes(dst_rank.get_length() - data_rank.get_length());
|
||||
std::iota(lead_axes.begin(), lead_axes.end(), 0);
|
||||
data = std::make_shared<ov::op::v0::Unsqueeze>(
|
||||
data, ov::op::v0::Constant::create(ov::element::i64, {lead_axes.size()}, lead_axes));
|
||||
}
|
||||
auto updates_shape = std::make_shared<ov::op::v3::ShapeOf>(data, ov::element::i64);
|
||||
|
||||
auto indices_rank3 = std::make_shared<ov::op::v0::Squeeze>(
|
||||
indices, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));
|
||||
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
|
||||
auto indices_rank4_shape = std::make_shared<ov::op::v0::Concat>(OutputVector{get_dimensions(updates_shape, {0, 1, 2}), one}, 0);
|
||||
auto indices_rank4 = std::make_shared<ov::op::v1::Reshape>(indices_rank3, indices_rank4_shape, false);
|
||||
auto broadcasted_indices = std::make_shared<ov::op::v3::Broadcast>(indices_rank4, updates_shape);
|
||||
std::vector<int> row_dims(dst_rank.get_length() - 1);
|
||||
std::iota(row_dims.begin(), row_dims.end(), 0);
|
||||
auto indices_rows_shape =
|
||||
std::make_shared<ov::op::v0::Concat>(OutputVector{get_dimensions(updates_shape, row_dims), one}, 0);
|
||||
auto indices_rows = std::make_shared<ov::op::v1::Reshape>(indices_rank3, indices_rows_shape, false);
|
||||
auto broadcasted_indices = std::make_shared<ov::op::v3::Broadcast>(indices_rows, updates_shape);
|
||||
auto rows_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-2});
|
||||
|
||||
res = std::make_shared<ov::op::v3::ScatterElementsUpdate>(dst, broadcasted_indices, data, axes);
|
||||
res = std::make_shared<ov::op::v3::ScatterElementsUpdate>(dst, broadcasted_indices, data, rows_axis);
|
||||
} else {
|
||||
auto row_size = context.get_input_shape(2)[3].get_length();
|
||||
auto ind_squeezed = std::make_shared<ov::op::v0::Squeeze>(
|
||||
indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2}));
|
||||
auto data_reshaped = std::make_shared<ov::op::v1::Reshape>(
|
||||
data,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4},
|
||||
{(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}),
|
||||
false);
|
||||
res = std::make_shared<ov::op::v3::ScatterUpdate>(dst, ind_squeezed, data_reshaped, axes);
|
||||
// skip the reshape when the rows already have that shape: it costs a GPU primitive per cache write
|
||||
const auto & data_shape = data.get_partial_shape();
|
||||
const bool data_is_rows = data_shape.rank().is_static() && data_shape.rank().get_length() == 4 &&
|
||||
data_shape[0] == 1 && data_shape[1] == 1 && data_shape[3] == row_size;
|
||||
ov::Output<ov::Node> data_reshaped = data;
|
||||
if (!data_is_rows) {
|
||||
data_reshaped = std::make_shared<ov::op::v1::Reshape>(
|
||||
data,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {4},
|
||||
{(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}),
|
||||
false);
|
||||
}
|
||||
if (use_elements_update) {
|
||||
auto updates_shape = std::make_shared<ov::op::v3::ShapeOf>(data_reshaped, ov::element::i64);
|
||||
auto ind_rank4 = std::make_shared<ov::op::v1::Reshape>(
|
||||
ind_squeezed, ov::op::v0::Constant::create(ov::element::i64, {4}, {1, 1, -1, 1}), false);
|
||||
auto broadcasted_indices = std::make_shared<ov::op::v3::Broadcast>(ind_rank4, updates_shape);
|
||||
res = std::make_shared<ov::op::v3::ScatterElementsUpdate>(dst, broadcasted_indices, data_reshaped, axes);
|
||||
} else {
|
||||
res = std::make_shared<ov::op::v3::ScatterUpdate>(dst, ind_squeezed, data_reshaped, axes);
|
||||
}
|
||||
}
|
||||
|
||||
auto dst_reshape = ov::as_type_ptr<ov::op::v1::Reshape>(dst.get_node_shared_ptr());
|
||||
if (!multidim_indices && dst_reshape) {
|
||||
if (!multidim_indices && is_cache_write && dst_reshape) {
|
||||
// Fix the case of multiple sequences, reshape back to original shape [1, n_seq, ctx_per_seq, emb]
|
||||
// ctx_per_seq is not fixed due to llama-bench compatibility
|
||||
auto dst_shape_partial = dst_reshape->get_input_partial_shape(0);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user