mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-19 17:27:41 -06:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b94959a64d | ||
|
|
50518b64c6 | ||
|
|
c406f4367e | ||
|
|
7fce5439fe | ||
|
|
53ecea05a0 | ||
|
|
9a080f44b5 | ||
|
|
c7468107ef | ||
|
|
ac830a9cbb | ||
|
|
8af68d486a | ||
|
|
f8caa5ea30 | ||
|
|
4db8b7638f | ||
|
|
01592dc58b | ||
|
|
8bf2132c3f | ||
|
|
5812201a81 | ||
|
|
41d8297fe4 | ||
|
|
51a77c4f24 | ||
|
|
76f186df71 | ||
|
|
95b09b62be | ||
|
|
b5dbca1e44 | ||
|
|
80437488a4 | ||
|
|
39039aa27f | ||
|
|
7a4117eef9 | ||
|
|
573aa89de6 | ||
|
|
ecafc67145 | ||
|
|
d4f5c26049 | ||
|
|
08c252e221 | ||
|
|
e4b620af19 | ||
|
|
b001d7bb22 | ||
|
|
0a76757469 | ||
|
|
b044890408 | ||
|
|
78e05298f0 | ||
|
|
785f6bac00 | ||
|
|
edec462cc0 | ||
|
|
4483b2b987 | ||
|
|
bafa0420c1 | ||
|
|
2cbfc3cf34 | ||
|
|
6f311d7ad6 | ||
|
|
ff6b8d1761 | ||
|
|
98d67c8d53 | ||
|
|
387395aa8d | ||
|
|
a693a5f66f | ||
|
|
b069e526de | ||
|
|
ffc87c3a79 | ||
|
|
10f173b253 | ||
|
|
97069ff6c4 | ||
|
|
bba8408821 | ||
|
|
48a32194ee | ||
|
|
6da36139e8 | ||
|
|
61f182fcde |
@@ -25,7 +25,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-11.7.91-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-11.7.101-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-11.7.91-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-11.7.91-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvml_dev-windows-x86_64-11.7.91-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_cudart-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvcc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7\cuda_nvrtc-windows-x86_64-11.7.99-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.7" /E /I /H /Y
|
||||
@@ -54,7 +56,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvprof/windows-x86_64/cuda_nvprof-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-12.4.127-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-12.4.127-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvml_dev-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_cudart-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvcc-windows-x86_64-12.4.131-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4\cuda_nvrtc-windows-x86_64-12.4.127-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.4" /E /I /H /Y
|
||||
@@ -85,7 +89,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.1.80-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.1.68-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-13.1.78-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.1.68-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvml_dev-windows-x86_64-13.1.68-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_crt-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_cudart-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1\cuda_nvcc-windows-x86_64-13.1.80-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.1" /E /I /H /Y
|
||||
@@ -116,7 +122,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_profiler_api/windows-x86_64/cuda_profiler_api-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/visual_studio_integration/windows-x86_64/visual_studio_integration-windows-x86_64-13.5.25-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cccl/windows-x86_64/cccl-windows-x86_64-13.3.5.0.6-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-x86_64/cuda_nvml_dev-windows-x86_64-13.5.37-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_cudart-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
@@ -143,7 +151,9 @@ runs:
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libnvvm/windows-x86_64/libnvvm-windows-x86_64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-arm64/cuda_cudart-windows-arm64-13.5.37-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/libcublas/windows-arm64/libcublas-windows-arm64-13.10.0.20-archive.zip"
|
||||
curl -O "https://developer.download.nvidia.com/compute/cuda/redist/cuda_nvml_dev/windows-arm64/cuda_nvml_dev-windows-arm64-13.5.37-archive.zip"
|
||||
unzip '*.zip' -d "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5"
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvml_dev-windows-arm64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cccl-windows-x86_64-13.3.5.0.6-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_crt-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
xcopy "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5\cuda_nvcc-windows-x86_64-13.5.37-archive\*" "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.5" /E /I /H /Y
|
||||
|
||||
@@ -140,6 +140,11 @@ jobs:
|
||||
source .venv/Scripts/activate
|
||||
pip install -r requirements/requirements-convert_hf_to_gguf.txt
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
shell: cmd
|
||||
|
||||
@@ -187,6 +187,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
|
||||
@@ -61,6 +61,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Install OpenCL Headers and Libs
|
||||
id: install_opencl
|
||||
run: |
|
||||
|
||||
@@ -213,6 +213,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
run: |
|
||||
|
||||
@@ -900,6 +900,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: ccache
|
||||
uses: ggml-org/ccache-action@v1.2.24
|
||||
with:
|
||||
@@ -1146,6 +1151,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
# TODO: these jobs need to use llvm toolchain in order to utilize the ccache
|
||||
#- name: ccache
|
||||
# uses: ggml-org/ccache-action@v1.2.24
|
||||
|
||||
@@ -175,6 +175,11 @@ jobs:
|
||||
evict-old-files: 1d
|
||||
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Build
|
||||
id: cmake_build
|
||||
shell: cmd
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
name: UI Chat Publish
|
||||
|
||||
# Publishes Llama Chat to npm: @ggml-org/llama-chat (ui/apps/chat, the app and its CLI) and
|
||||
# llama-chat (ui/apps/chat/alias, the short name for npx), always at the same version.
|
||||
#
|
||||
# The version in ui/apps/chat/package.json is the release, set by hand, as for @ggml-org/llama-ui
|
||||
# in ui-lib-publish.yml. A commit that reaches master with a version npm does not have builds the
|
||||
# app and publishes both packages; any other commit publishes nothing. Bump both package.json
|
||||
# files in the PR that should ship.
|
||||
#
|
||||
# Auth: no npm token is stored here. The job exchanges its OIDC identity for one (npm trusted
|
||||
# publishing) and signs the tarballs with provenance. One-time setup, before this can publish: on
|
||||
# each of the two packages, Publishing settings, Trusted Publisher, GitHub Actions, with
|
||||
# organization ggml-org, repository llama.cpp, workflow file ui-chat-publish.yml and Environment
|
||||
# left blank. Until then the publish step fails on master; re-running the job publishes it.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: 'Build and pack, but do not publish'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths: [
|
||||
'.github/workflows/ui-chat-publish.yml',
|
||||
'ui/apps/chat/package.json',
|
||||
'ui/apps/chat/alias/package.json'
|
||||
]
|
||||
# a dry run for PRs that change what goes into the packages
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
paths: [
|
||||
'.github/workflows/ui-chat-publish.yml',
|
||||
'ui/apps/chat/package.json',
|
||||
'ui/apps/chat/alias/**',
|
||||
'ui/apps/chat/bin/**',
|
||||
'ui/apps/chat/cli/**'
|
||||
]
|
||||
|
||||
cache-mode: none
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
|
||||
concurrency:
|
||||
# never cancel a run mid-publish: a version half-published cannot be taken back
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
APP_DIR: ui/apps/chat
|
||||
ALIAS_DIR: ui/apps/chat/alias
|
||||
VERSION_CHECK: ui/lib/ui/scripts/npm-version-check.mjs
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Publish @ggml-org/llama-chat and llama-chat
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: ubuntu-26.04-arm
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: "24"
|
||||
registry-url: 'https://registry.npmjs.org'
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Check npm can publish with an OIDC token
|
||||
run: |
|
||||
node -e '
|
||||
const v = process.argv[1];
|
||||
const [major, minor] = v.split(".").map(Number);
|
||||
if (major < 11 || (major === 11 && minor < 5)) {
|
||||
console.error(`npm ${v} cannot publish with trusted publishing: 11.5.1 or newer is needed`);
|
||||
process.exit(1);
|
||||
}
|
||||
' "$(npm -v)"
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Build the app
|
||||
# the same build.json version as the UI that llama-server embeds, for the app's update check.
|
||||
# This folder's .npmrc sets ignore-scripts, so npm never builds it: this step does.
|
||||
run: LLAMA_BUILD_NUMBER="b$(git rev-list --count HEAD)" npm run build
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Test the CLI and the alias version
|
||||
run: npx vitest --project=unit --run tests/unit/cli.test.ts
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Check what goes into the package
|
||||
run: |
|
||||
npm pack --dry-run --json > "$RUNNER_TEMP/pack.json"
|
||||
node -e '
|
||||
const [{ files }] = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
|
||||
const paths = files.map((file) => file.path);
|
||||
const maps = paths.filter((path) => path.endsWith(".map"));
|
||||
if (!paths.includes("dist/index.html")) {
|
||||
console.error("dist/index.html is not in the package: nothing would be served");
|
||||
process.exit(1);
|
||||
}
|
||||
if (maps.length > 0) {
|
||||
console.error(`source maps in the package: ${maps.join(", ")}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`${paths.length} files, no source maps`);
|
||||
' "$RUNNER_TEMP/pack.json"
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Compare @ggml-org/llama-chat with npm
|
||||
id: app
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node "$VERSION_CHECK" "$APP_DIR"
|
||||
|
||||
- name: Compare llama-chat with npm
|
||||
id: alias
|
||||
continue-on-error: ${{ github.event_name == 'pull_request' }}
|
||||
run: node "$VERSION_CHECK" "$ALIAS_DIR"
|
||||
|
||||
# master only: npm's trusted publisher matches the repository and the workflow file, not the
|
||||
# branch, so this condition is what keeps a manual run on a PR branch from publishing.
|
||||
# The app goes first, since the alias depends on it at the same version.
|
||||
- name: Publish @ggml-org/llama-chat
|
||||
if: github.ref == 'refs/heads/master' && steps.app.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
TAG: ${{ steps.app.outputs.tag }}
|
||||
run: npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.APP_DIR }}
|
||||
|
||||
- name: Publish llama-chat
|
||||
if: github.ref == 'refs/heads/master' && steps.alias.outputs.should_publish == 'true' && inputs.dry_run != true
|
||||
env:
|
||||
TAG: ${{ steps.alias.outputs.tag }}
|
||||
run: npm publish --access public --provenance --ignore-scripts --tag "$TAG"
|
||||
working-directory: ${{ env.ALIAS_DIR }}
|
||||
|
||||
- name: Report
|
||||
if: always()
|
||||
# values reach the shell through env, so a name from a PR's package.json cannot inject here
|
||||
env:
|
||||
APP: ${{ steps.app.outputs.name }}@${{ steps.app.outputs.version }}
|
||||
APP_VERDICT: ${{ steps.app.outputs.verdict || 'not reached' }}
|
||||
APP_NOTE: ${{ steps.app.outputs.note }}
|
||||
ALIAS: ${{ steps.alias.outputs.name }}@${{ steps.alias.outputs.version }}
|
||||
ALIAS_VERDICT: ${{ steps.alias.outputs.verdict || 'not reached' }}
|
||||
ALIAS_NOTE: ${{ steps.alias.outputs.note }}
|
||||
DRY: ${{ inputs.dry_run }}
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
{
|
||||
echo "### Llama Chat on npm"
|
||||
echo
|
||||
echo "| package | verdict | note |"
|
||||
echo "| --- | --- | --- |"
|
||||
echo "| \`$APP\` | \`$APP_VERDICT\` | $APP_NOTE |"
|
||||
echo "| \`$ALIAS\` | \`$ALIAS_VERDICT\` | $ALIAS_NOTE |"
|
||||
echo
|
||||
if [ "$EVENT" = "pull_request" ] || [ "$DRY" = "true" ]; then echo "Dry run: nothing was published."; fi
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Mind the verdict
|
||||
# a version going backwards is a mistake; an unchanged version is a package.json edit that
|
||||
# is not a release, such as a dependency update
|
||||
if: always() && github.event_name != 'pull_request'
|
||||
env:
|
||||
VERDICTS: ${{ steps.app.outputs.verdict }} ${{ steps.alias.outputs.verdict }}
|
||||
run: |
|
||||
case " $VERDICTS " in
|
||||
*" stale "*)
|
||||
echo "::error::a version in package.json is older than the one on npm"
|
||||
exit 1 ;;
|
||||
esac
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
deploy:
|
||||
name: Deploy to Hugging Face Spaces
|
||||
if: github.repository == 'ggml-org/llama.cpp'
|
||||
runs-on: ubuntu-24.04
|
||||
runs-on: hf-jobs-cpu-upgrade:ubuntu26_04
|
||||
|
||||
env:
|
||||
HF_SPACE: ${{ vars.HF_SPACE_UI_STORYBOOK || 'ggml-org/llama-ui-storybook' }}
|
||||
@@ -67,12 +67,15 @@ jobs:
|
||||
|
||||
- name: Install Hugging Face Hub CLI
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y python3 python3-venv
|
||||
python3 -m venv .venv-hf
|
||||
.venv-hf/bin/pip install -U huggingface_hub==1.28.0
|
||||
|
||||
- name: Upload to the Space
|
||||
env:
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_UI_STATIC_OUTPUT }}
|
||||
# a token with write access to the Space; HF_TOKEN_UI_STATIC_OUTPUT if that one has it
|
||||
HF_TOKEN: ${{ secrets.HF_TOKEN_UI_STORYBOOK || secrets.HF_TOKEN_UI_STATIC_OUTPUT }}
|
||||
run: |
|
||||
# --delete '*' removes files the new build no longer has; the Space's .gitattributes is always kept
|
||||
.venv-hf/bin/hf upload "$HF_SPACE" ui/apps/chat/storybook-static . \
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<b>LLM inference in C/C++</b>
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v1)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v1)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=b)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
|
||||
@@ -1313,6 +1313,11 @@ common_init_result::common_init_result(common_params & params, bool model_only)
|
||||
LOG_INF("%s", "decision model reads the embeddings output, enabling embedding mode\n");
|
||||
}
|
||||
|
||||
// 0 is n_batch
|
||||
if ((decision_type == COMMON_DECISION_TYPE_LFM3_D1 || decision_type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) && cparams.n_outputs_max != 0) {
|
||||
cparams.n_outputs_max = std::max(cparams.n_outputs_max, (uint32_t) COMMON_DECISION_LFM3_D1_SET_SIZE);
|
||||
}
|
||||
|
||||
if (llama_model_classifier_free_guidance(model)) {
|
||||
params.kv_unified = true;
|
||||
cparams.kv_unified = true;
|
||||
@@ -2334,6 +2339,9 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
|
||||
if (t.decision_order != 0) {
|
||||
llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order);
|
||||
}
|
||||
if (t.decision_group != 0 && !llama_batch_ext_set_decision_group(res, idx, t.decision_group)) {
|
||||
GGML_ABORT("%s: failed to set the decision group of the entry at index %d\n", __func__, i);
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
|
||||
@@ -1002,6 +1002,9 @@ enum common_decision_type {
|
||||
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
|
||||
};
|
||||
|
||||
// lfm3-d1: max questions packed in one prompt
|
||||
#define COMMON_DECISION_LFM3_D1_SET_SIZE 16
|
||||
|
||||
common_decision_type common_get_decision_type(const struct llama_model * model);
|
||||
|
||||
// metadata of a GGUF file, read without loading the model
|
||||
@@ -1117,6 +1120,7 @@ struct common_batch {
|
||||
llama_embd state; // non-owning view of the data passed to set_embd_state(), data == NULL if none
|
||||
std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
|
||||
int32_t decision_order = 0; // see llama_batch_ext_set_decision_order()
|
||||
int32_t decision_group = 0; // see llama_batch_ext_set_decision_group()
|
||||
int32_t embd_row = -1; // see add_embd_row(), -1 if none
|
||||
int32_t state_row = -1; // see set_embd_state_row(), -1 if none
|
||||
};
|
||||
|
||||
@@ -287,6 +287,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"TinyVocaModel": "tiny_voca",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"VoxCPM2Model": "voxcpm",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Qwen3TTSForConditionalGeneration": "qwen3tts",
|
||||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
@@ -340,6 +341,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
||||
"XverseForCausalLM": "xverse",
|
||||
"YoutuForCausalLM": "deepseek",
|
||||
"YoutuVLForConditionalGeneration": "deepseek",
|
||||
"ZayaForCausalLM": "zaya1",
|
||||
"fish_qwen3_omni": "fish_s2",
|
||||
"modeling_grove_moe.GroveMoeForCausalLM": "grovemoe",
|
||||
"modeling_sarvam_moe.SarvamMoEForCausalLM": "bailingmoe",
|
||||
@@ -366,6 +368,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
||||
"FishOmniModel": "drama3",
|
||||
"Zonos3Model": "zonos3",
|
||||
"YuE2ForCausalLM": "yue2",
|
||||
"VoxCPM2Model": "voxcpm",
|
||||
"MiniMaxMusic3ForConditionalGeneration": "minimax_music3",
|
||||
"Gemma3ForConditionalGeneration": "gemma",
|
||||
"Gemma3nForConditionalGeneration": "gemma",
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, MmprojModel, TextModel, LazyTorchTensor, gguf, logger
|
||||
from .drama3 import fold_snake
|
||||
from .minicpm import MiniCPMModel
|
||||
|
||||
# sample rate of the AudioVAE decoder output, picks the row of the sample-rate conditioning
|
||||
VAE_OUT_SAMPLE_RATE = 48000
|
||||
|
||||
|
||||
def _load_config(dir_model: Path) -> dict[str, Any] | None:
|
||||
path = dir_model / "config.json"
|
||||
if not path.is_file():
|
||||
return None
|
||||
with open(path, encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
return config if config.get("architecture") == "voxcpm2" else None
|
||||
|
||||
|
||||
def _is_voxcpm2_checkpoint(dir_model: Path) -> bool:
|
||||
return _load_config(dir_model) is not None
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_voxcpm2_checkpoint)
|
||||
def _load_voxcpm2_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected VoxCPM2 checkpoint")
|
||||
config = _load_config(dir_model)
|
||||
assert config is not None
|
||||
if config["lm_config"].get("use_mup"):
|
||||
raise NotImplementedError("VoxCPM2: use_mup = true is not supported")
|
||||
if config["dit_config"].get("mean_mode"):
|
||||
raise NotImplementedError("VoxCPM2: dit mean_mode = true is not supported")
|
||||
if not config.get("residual_lm_no_rope"):
|
||||
raise NotImplementedError("VoxCPM2: residual LM with RoPE is not supported")
|
||||
for key in ("hidden_dim", "ffn_dim", "num_heads", "num_layers", "kv_channels"):
|
||||
if config["encoder_config"][key] != config["dit_config"][key]:
|
||||
raise NotImplementedError(f"VoxCPM2: LocEnc and LocDiT differ in {key}")
|
||||
return {**config, "architectures": ["VoxCPM2Model"], "text_config": config["lm_config"], "audio_config": config["dit_config"]}
|
||||
|
||||
|
||||
@ModelBase.register("VoxCPM2Model")
|
||||
@ModelBase.example("openbmb/VoxCPM2")
|
||||
class VoxCPM2TextModel(MiniCPMModel):
|
||||
model_arch = gguf.MODEL_ARCH.MINICPM
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
# the text-semantic LM is a plain MiniCPM, the rest goes to the mmproj
|
||||
if not name.startswith("base_lm."):
|
||||
return None
|
||||
return super().filter_tensors(("model." + name.removeprefix("base_lm."), gen))
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
TextModel.set_gguf_parameters(self)
|
||||
# use_mup = false: no embedding, residual or logit scaling
|
||||
self.gguf_writer.add_embedding_scale(1.0)
|
||||
self.gguf_writer.add_residual_scale(1.0)
|
||||
self.gguf_writer.add_logit_scale(1.0)
|
||||
|
||||
def set_vocab(self):
|
||||
# the checkpoint has no tokenizer.model and its tokenizer class is remote code, so read tokenizer.json
|
||||
# SPM-style BPE: a merged token scores minus its merge rank, so the SPM tokenizer merges in the BPE order
|
||||
with open(self.dir_model / "tokenizer.json", encoding="utf-8") as f:
|
||||
tok = json.load(f)
|
||||
with open(self.dir_model / "tokenizer_config.json", encoding="utf-8") as f:
|
||||
tok_cfg = json.load(f)
|
||||
model = tok["model"]
|
||||
if model["type"] != "BPE" or not model.get("byte_fallback"):
|
||||
raise NotImplementedError("VoxCPM2: unexpected tokenizer type")
|
||||
|
||||
vocab: dict[str, int] = model["vocab"]
|
||||
vocab_size = self.hparams["vocab_size"]
|
||||
tokens = [f"[PAD{i}]" for i in range(vocab_size)]
|
||||
scores = [-1e9] * vocab_size
|
||||
toktypes = [gguf.TokenType.UNUSED] * vocab_size
|
||||
for text, i in vocab.items():
|
||||
tokens[i] = text
|
||||
toktypes[i] = gguf.TokenType.BYTE if re.fullmatch(r"<0x[0-9A-Fa-f]{2}>", text) else gguf.TokenType.NORMAL
|
||||
for rank, merge in enumerate(model["merges"]):
|
||||
a, b = merge.split(" ") if isinstance(merge, str) else merge
|
||||
i = vocab[a + b]
|
||||
scores[i] = max(scores[i], -float(rank))
|
||||
|
||||
# the special tokens (e.g. <|audio_start|>) are only listed in tokenizer_config.json
|
||||
specials = {int(i): t["content"] for i, t in tok_cfg.get("added_tokens_decoder", {}).items() if t.get("special")}
|
||||
specials.update({t["id"]: t["content"] for t in tok["added_tokens"] if t.get("special")})
|
||||
for i, text in specials.items():
|
||||
tokens[i] = text
|
||||
toktypes[i] = gguf.TokenType.CONTROL
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("llama")
|
||||
self.gguf_writer.add_tokenizer_pre("default")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores(scores)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens))
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
|
||||
|
||||
@ModelBase.register("VoxCPM2Model")
|
||||
@ModelBase.example("openbmb/VoxCPM2")
|
||||
class VoxCPM2MmprojModel(MmprojModel):
|
||||
has_vision_encoder = False
|
||||
has_audio_encoder = True # AudioVAE encoder, for voice cloning
|
||||
|
||||
_LAYER_MAP = {
|
||||
"input_layernorm.weight": "ln1",
|
||||
"self_attn.q_proj.weight": "attn_q",
|
||||
"self_attn.k_proj.weight": "attn_k",
|
||||
"self_attn.v_proj.weight": "attn_v",
|
||||
"self_attn.o_proj.weight": "attn_out",
|
||||
"post_attention_layernorm.weight": "ln2",
|
||||
"mlp.gate_proj.weight": "ffn_gate",
|
||||
"mlp.up_proj.weight": "ffn_up",
|
||||
"mlp.down_proj.weight": "ffn_down",
|
||||
}
|
||||
|
||||
# libllama names, mtmd loads the residual LM as a text model
|
||||
_RES_LAYER_MAP = {
|
||||
"input_layernorm.weight": "attn_norm",
|
||||
"self_attn.q_proj.weight": "attn_q",
|
||||
"self_attn.k_proj.weight": "attn_k",
|
||||
"self_attn.v_proj.weight": "attn_v",
|
||||
"self_attn.o_proj.weight": "attn_output",
|
||||
"post_attention_layernorm.weight": "ffn_norm",
|
||||
"mlp.gate_proj.weight": "ffn_gate",
|
||||
"mlp.up_proj.weight": "ffn_up",
|
||||
"mlp.down_proj.weight": "ffn_down",
|
||||
}
|
||||
|
||||
_PROJ_MAP = {
|
||||
"fsq_layer.in_proj": "a.gen.fsq.in",
|
||||
"fsq_layer.out_proj": "a.gen.fsq.out",
|
||||
"enc_to_lm_proj": "a.gen.enc_to_lm",
|
||||
"lm_to_dit_proj": "a.gen.lm_to_dit",
|
||||
"res_to_dit_proj": "a.gen.res_to_dit",
|
||||
"fusion_concat_proj": "a.gen.fusion",
|
||||
"stop_proj": "a.gen.stop_proj",
|
||||
"stop_head": "a.gen.stop_head",
|
||||
"feat_encoder.in_proj": "a.gen.enc.in_proj",
|
||||
"feat_decoder.estimator.in_proj": "a.gen.dit.in_proj",
|
||||
"feat_decoder.estimator.cond_proj": "a.gen.dit.cond_proj",
|
||||
"feat_decoder.estimator.out_proj": "a.gen.dit.out_proj",
|
||||
"feat_decoder.estimator.time_mlp.linear_1": "a.gen.dit.time_mlp.0",
|
||||
"feat_decoder.estimator.time_mlp.linear_2": "a.gen.dit.time_mlp.1",
|
||||
}
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
lm = self.global_config["lm_config"]
|
||||
dit = self.global_config["dit_config"]
|
||||
vae = self.global_config["audio_vae_config"]
|
||||
self.gguf_writer.add_file_type(self.ftype)
|
||||
|
||||
# LocEnc + LocDiT + residual LM + AudioVAE decoder, LocEnc has the hparams of LocDiT
|
||||
self.gguf_writer.add_clip_has_gen_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.VOXCPM2_GEN)
|
||||
self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_gen_audio_embedding_length(dit["hidden_dim"])
|
||||
self.gguf_writer.add_gen_audio_feed_forward_length(dit["ffn_dim"])
|
||||
self.gguf_writer.add_gen_audio_block_count(dit["num_layers"])
|
||||
self.gguf_writer.add_gen_audio_head_count(dit["num_heads"])
|
||||
self.gguf_writer.add_gen_audio_head_count_kv(lm["num_key_value_heads"])
|
||||
self.gguf_writer.add_gen_audio_attention_layernorm_eps(lm["rms_norm_eps"])
|
||||
|
||||
# AudioVAE encoder: raw waveform -> latents, LocEnc in the gen model then makes the backbone rows
|
||||
self.gguf_writer.add_clip_has_audio_encoder(True)
|
||||
self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.VOXCPM2_SPKENC)
|
||||
self.gguf_writer.add_audio_projection_dim(self.n_embd_text)
|
||||
self.gguf_writer.add_audio_num_mel_bins(1)
|
||||
self.gguf_writer.add_audio_block_count(0)
|
||||
self.gguf_writer.add_audio_embedding_length(vae["encoder_dim"])
|
||||
self.gguf_writer.add_audio_feed_forward_length(vae["encoder_dim"])
|
||||
self.gguf_writer.add_audio_head_count(1)
|
||||
self.gguf_writer.add_audio_attention_layernorm_eps(lm["rms_norm_eps"])
|
||||
# the ODE solver, the RoPE factors and the VAE layout are hard-coded in tools/mtmd
|
||||
|
||||
def tensor_force_quant(self, name, new_name, bid, n_dims):
|
||||
# the AudioVAE convs and the 1D tensors stay F32
|
||||
if new_name.startswith(("a.vae.", "a.gen.vae.")) or n_dims < 2:
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
if item[0].startswith("base_lm."):
|
||||
return None
|
||||
return item
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
# already named by generate_extra_tensors()
|
||||
if name.startswith("a."):
|
||||
yield (name, data_torch)
|
||||
return
|
||||
|
||||
# dt is always 0 (mean_mode = false), delta_time_mlp gives a constant that is folded into the time_mlp bias
|
||||
if name.startswith("feat_decoder.estimator.delta_time_mlp."):
|
||||
return
|
||||
if name == "feat_decoder.estimator.time_mlp.linear_2.bias":
|
||||
data_torch = LazyTorchTensor.to_eager(data_torch).float() + self._dtime_const()
|
||||
|
||||
if name == "feat_encoder.special_token":
|
||||
yield ("a.gen.enc.special_token", data_torch.reshape(-1))
|
||||
return
|
||||
|
||||
for src, dst in (("residual_lm.", "a.gen.res"), ("feat_encoder.encoder.", "a.gen.enc"),
|
||||
("feat_decoder.estimator.decoder.", "a.gen.code")):
|
||||
if not name.startswith(src):
|
||||
continue
|
||||
key = name.removeprefix(src)
|
||||
if key == "norm.weight":
|
||||
yield (f"{dst}.output_norm.weight" if dst == "a.gen.res" else f"{dst}.post_ln.weight", data_torch)
|
||||
return
|
||||
assert bid is not None, name
|
||||
key = key.removeprefix(f"layers.{bid}.")
|
||||
layer_map = self._RES_LAYER_MAP if dst == "a.gen.res" else self._LAYER_MAP
|
||||
yield (f"{dst}.blk.{bid}.{layer_map[key]}.weight", data_torch)
|
||||
return
|
||||
|
||||
base, _, suffix = name.rpartition(".")
|
||||
yield (f"{self._PROJ_MAP[base]}.{suffix}", data_torch)
|
||||
|
||||
# delta_time_mlp(SinusoidalPosEmb(0)), see VoxCPMLocDiTV2.forward()
|
||||
def _dtime_const(self) -> Tensor:
|
||||
d = {k: LazyTorchTensor.to_eager(self.model_tensors[f"feat_decoder.estimator.delta_time_mlp.{k}"]()).float()
|
||||
for k in ("linear_1.weight", "linear_1.bias", "linear_2.weight", "linear_2.bias")}
|
||||
dim = d["linear_1.weight"].shape[1]
|
||||
emb = torch.cat([torch.zeros(dim // 2), torch.ones(dim // 2)]) # sin(0), cos(0)
|
||||
h = torch.nn.functional.silu(d["linear_1.weight"] @ emb + d["linear_1.bias"])
|
||||
return d["linear_2.weight"] @ h + d["linear_2.bias"]
|
||||
|
||||
@staticmethod
|
||||
def _wn(sd: dict[str, Tensor], prefix: str) -> Tensor:
|
||||
# torch weight norm, over every dim but 0
|
||||
g, v = sd[prefix + ".weight_g"].float(), sd[prefix + ".weight_v"].float()
|
||||
return g * v / v.flatten(1).norm(dim=1).view(-1, *[1] * (v.dim() - 1))
|
||||
|
||||
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
|
||||
# LongRoPE factors of LocEnc and LocDiT, picked as in MiniCPMLongRoPE._set_cos_sin_cache()
|
||||
lm = self.global_config["lm_config"]
|
||||
rope = lm["rope_scaling"]
|
||||
if rope.get("type") != "longrope":
|
||||
raise NotImplementedError(f"VoxCPM2: unexpected rope_scaling {rope.get('type')}")
|
||||
if lm["max_position_embeddings"] > rope["original_max_position_embeddings"]:
|
||||
raise NotImplementedError("VoxCPM2: the long RoPE factors also scale cos/sin, not supported")
|
||||
yield ("a.gen.rope_factors", torch.tensor(rope["short_factor"], dtype=torch.float32))
|
||||
|
||||
path = self.dir_model / "audiovae.pth"
|
||||
logger.info(f"gguf: loading AudioVAE from '{path}'")
|
||||
sd = torch.load(str(path), map_location="cpu", mmap=True, weights_only=True)
|
||||
sd = sd.get("state_dict", sd)
|
||||
vae = self.global_config["audio_vae_config"]
|
||||
|
||||
def conv(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
yield (f"{dst}.weight", self._wn(sd, src))
|
||||
yield (f"{dst}.bias", sd[src + ".bias"].float())
|
||||
|
||||
def snake(dst: str, src: str) -> Iterable[tuple[str, Tensor]]:
|
||||
alpha, beta = fold_snake(sd[src + ".alpha"])
|
||||
yield (f"{dst}.alpha", alpha)
|
||||
yield (f"{dst}.beta", beta)
|
||||
|
||||
def res_units(dst: str, src: str, first: int) -> Iterable[tuple[str, Tensor]]:
|
||||
# dilation 1, 3, 9: Snake, depthwise conv k = 7, Snake, conv k = 1
|
||||
for r in range(3):
|
||||
s = f"{src}.{first + r}.block"
|
||||
d = f"{dst}.res.{r}"
|
||||
yield from snake(f"{d}.snake1", f"{s}.0")
|
||||
yield from conv(f"{d}.conv1", f"{s}.1")
|
||||
yield from snake(f"{d}.snake2", f"{s}.2")
|
||||
yield from conv(f"{d}.conv2", f"{s}.3")
|
||||
|
||||
# encoder: conv, then per stride (3 residual units, Snake, strided conv), then fc_mu
|
||||
p = "a.vae.enc"
|
||||
yield from conv(f"{p}.conv_in", "encoder.block.0")
|
||||
for b in range(len(vae["encoder_rates"])):
|
||||
s = f"encoder.block.{b + 1}.block"
|
||||
yield from res_units(f"{p}.blk.{b}", s, 0)
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.3")
|
||||
yield from conv(f"{p}.blk.{b}.down", f"{s}.4")
|
||||
yield from conv(f"{p}.fc_mu", "encoder.fc_mu")
|
||||
|
||||
# decoder: depthwise conv, conv k = 1, then per stride (sample-rate scale and bias, Snake, transposed conv, 3 residual units), then Snake, conv
|
||||
bounds = vae["sr_bin_boundaries"]
|
||||
sr_idx = sum(1 for b in bounds if b < VAE_OUT_SAMPLE_RATE) # torch.bucketize
|
||||
if vae["out_sample_rate"] != VAE_OUT_SAMPLE_RATE:
|
||||
raise NotImplementedError(f"VoxCPM2: unexpected out_sample_rate {vae['out_sample_rate']}")
|
||||
p = "a.gen.vae"
|
||||
yield from conv(f"{p}.conv_in_dw", "decoder.model.0")
|
||||
yield from conv(f"{p}.conv_in", "decoder.model.1")
|
||||
n_up = len(vae["decoder_rates"])
|
||||
for b in range(n_up):
|
||||
c = f"decoder.sr_cond_model.{b + 2}"
|
||||
yield (f"{p}.blk.{b}.sr_scale", sd[c + ".scale_embed.weight"][sr_idx].float())
|
||||
yield (f"{p}.blk.{b}.sr_bias", sd[c + ".bias_embed.weight"][sr_idx].float())
|
||||
s = f"decoder.model.{b + 2}.block"
|
||||
yield from snake(f"{p}.blk.{b}.snake", f"{s}.0")
|
||||
yield from conv(f"{p}.blk.{b}.up", f"{s}.1")
|
||||
yield from res_units(f"{p}.blk.{b}", s, 2)
|
||||
yield from snake(f"{p}.post_snake", f"decoder.model.{n_up + 2}")
|
||||
yield from conv(f"{p}.conv_out", f"decoder.model.{n_up + 3}")
|
||||
|
||||
assert math.prod(vae["encoder_rates"]) * VAE_OUT_SAMPLE_RATE == math.prod(vae["decoder_rates"]) * vae["sample_rate"]
|
||||
@@ -0,0 +1,144 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import torch
|
||||
|
||||
from typing import Callable, Iterable, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from torch import Tensor
|
||||
|
||||
from .base import ModelBase, TextModel, gguf
|
||||
|
||||
|
||||
@ModelBase.register("ZayaForCausalLM")
|
||||
@ModelBase.example("Zyphra/ZAYA1-8B")
|
||||
class ZayaModel(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.ZAYA1
|
||||
|
||||
# checkpoint names without a .weight/.bias suffix
|
||||
_RENAME = {
|
||||
"model.input_hidden_states_scale": "model.input_hidden_states.weight",
|
||||
"model.input_hidden_states_bias": "model.input_hidden_states.bias",
|
||||
}
|
||||
|
||||
# the two kv heads of attn_v: head 0 = current token, head 1 = previous token
|
||||
_V_PARTS = (
|
||||
"self_attn.qkv_proj.v_proj_current.weight",
|
||||
"self_attn.qkv_proj.v_proj_delayed.weight",
|
||||
)
|
||||
|
||||
# (hs + b_hs) * s_hs + (res + b_res) * s_res = hs * s_hs + res * s_res + (b_hs * s_hs + b_res * s_res)
|
||||
_RES_SCALES = {
|
||||
"post_attention_residual_scale.": (gguf.MODEL_TENSOR.ATTN_RES_SCALE_HS, gguf.MODEL_TENSOR.ATTN_RESID_SCALE),
|
||||
"post_mlp_residual_scale.": (gguf.MODEL_TENSOR.FFN_RES_SCALE_HS, gguf.MODEL_TENSOR.FFN_RESID_SCALE),
|
||||
}
|
||||
_RES_PARTS = ("hidden_states_scale", "hidden_states_bias", "residual_scale", "residual_bias")
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
# added tokens overlap the base vocab, so count the token list itself
|
||||
self._vocab = gguf.LlamaHfVocab(self.dir_model)
|
||||
self._n_vocab = self._vocab.vocab_size
|
||||
self._parts: dict[tuple[int, str], dict[str, Tensor]] = {}
|
||||
self.rope_parameters = self.hparams["rope_parameters"]["hybrid"]
|
||||
|
||||
def set_vocab(self):
|
||||
# added special tokens overlap the base vocab, so mark them as control here
|
||||
with open(self.dir_model / "tokenizer.json", "r", encoding="utf-8") as f:
|
||||
added = {t["id"]: t["special"] for t in json.load(f)["added_tokens"]}
|
||||
|
||||
tokens, scores, toktypes = [], [], []
|
||||
for i, (text, score, toktype) in enumerate(self._vocab.all_tokens()):
|
||||
tokens.append(text)
|
||||
scores.append(score)
|
||||
if added.get(i):
|
||||
toktype = gguf.TokenType.CONTROL
|
||||
toktypes.append(toktype)
|
||||
|
||||
self.gguf_writer.add_tokenizer_model("gemma4")
|
||||
self.gguf_writer.add_token_list(tokens)
|
||||
self.gguf_writer.add_token_scores(scores)
|
||||
self.gguf_writer.add_token_types(toktypes)
|
||||
|
||||
special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
|
||||
special_vocab.add_to_gguf(self.gguf_writer)
|
||||
self.gguf_writer.add_add_space_prefix(False)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
hparams = self.hparams
|
||||
super().set_gguf_parameters()
|
||||
|
||||
self.gguf_writer.add_rope_dimension_count(int(self.rope_parameters["partial_rotary_factor"] * hparams["head_dim"]))
|
||||
self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"])
|
||||
self.gguf_writer.add_ssm_conv_kernel(hparams["cca_time0"])
|
||||
self.gguf_writer.add_router_hidden_size(hparams["router_hidden_size"])
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, _ = item
|
||||
# skip the draft head tensors now, they will be added as part of the TiDAR changes
|
||||
if name.startswith(("diffusion_output_layer.", "diffusion_markov_head.")):
|
||||
return None
|
||||
|
||||
return super().filter_tensors(item)
|
||||
|
||||
def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
|
||||
# the router picks one expert, so its matmuls stay F32 like ffn_gate_inp
|
||||
if any(self.match_model_tensor_name(new_name, key, bid) for key in (
|
||||
gguf.MODEL_TENSOR.CCA_CONV_GRP,
|
||||
gguf.MODEL_TENSOR.ROUTER_FC1,
|
||||
gguf.MODEL_TENSOR.ROUTER_FC2,
|
||||
gguf.MODEL_TENSOR.ROUTER_OUT)):
|
||||
return gguf.GGMLQuantizationType.F32
|
||||
return super().tensor_force_quant(name, new_name, bid, n_dims)
|
||||
|
||||
def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
|
||||
if bid is not None:
|
||||
local = name.removeprefix(f"model.layers.{bid}.")
|
||||
if local in self._V_PARTS:
|
||||
p = self._parts.setdefault((bid, "v"), {})
|
||||
p[local] = data_torch
|
||||
if len(p) == len(self._V_PARTS):
|
||||
del self._parts[(bid, "v")]
|
||||
v = torch.cat([p[n] for n in self._V_PARTS], dim=0)
|
||||
yield from super().modify_tensors(v, f"model.layers.{bid}.{self._V_PARTS[0]}", bid)
|
||||
return
|
||||
for prefix, (key_hs, key_res) in self._RES_SCALES.items():
|
||||
if local.startswith(prefix) and local[len(prefix):] in self._RES_PARTS:
|
||||
p = self._parts.setdefault((bid, prefix), {})
|
||||
p[local[len(prefix):]] = data_torch
|
||||
if len(p) == len(self._RES_PARTS):
|
||||
del self._parts[(bid, prefix)]
|
||||
s_hs, b_hs, s_res, b_res = (p[n].float() for n in self._RES_PARTS)
|
||||
yield self.format_tensor_name(key_hs, bid, ".weight"), s_hs
|
||||
yield self.format_tensor_name(key_res, bid, ".weight"), s_res
|
||||
yield self.format_tensor_name(key_res, bid, ".bias"), b_hs * s_hs + b_res * s_res
|
||||
return
|
||||
|
||||
name = self._RENAME.get(name, name)
|
||||
if name.endswith("balancing_biases"):
|
||||
name += ".bias"
|
||||
elif not name.endswith((".weight", ".bias")):
|
||||
name += ".weight"
|
||||
new_name = self.map_tensor_name(name)
|
||||
|
||||
def is_tensor(key: gguf.MODEL_TENSOR) -> bool:
|
||||
return self.match_model_tensor_name(new_name, key, bid)
|
||||
|
||||
if any(is_tensor(key) for key in (gguf.MODEL_TENSOR.ATTN_Q, gguf.MODEL_TENSOR.ATTN_K)):
|
||||
# through the base class, which fuses q, k and v with --fuse-qkv
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
return
|
||||
if is_tensor(gguf.MODEL_TENSOR.TOKEN_EMBD):
|
||||
data_torch = data_torch[:self._n_vocab]
|
||||
elif is_tensor(gguf.MODEL_TENSOR.SSM_CONV1D):
|
||||
data_torch = data_torch.squeeze(1)
|
||||
elif is_tensor(gguf.MODEL_TENSOR.CCA_CONV_GRP):
|
||||
# [n_group*out, in, kernel] -> [kernel, n_group, out, in]
|
||||
n_group = self.hparams["num_attention_heads"] + self.hparams["num_key_value_heads"]
|
||||
n_out, n_in, n_k = data_torch.shape
|
||||
data_torch = data_torch.reshape(n_group, n_out // n_group, n_in, n_k).permute(3, 0, 1, 2).contiguous()
|
||||
|
||||
yield new_name, data_torch
|
||||
@@ -100,6 +100,12 @@ The OpenCL backend has the following CMake options that control the behavior of
|
||||
| `GGML_OPENCL_USE_ADRENO_KERNELS` | `ON` | Use kernels optimized for Adreno. |
|
||||
| `GGML_OPENCL_USE_ADRENO_BIN_KERNELS` | `OFF` | Allow using binary kernel lib for Adreno. |
|
||||
|
||||
## Decomposed Flash Attention
|
||||
|
||||
The P8 path rounds scratch row pitches for P and transposed V up to a multiple of 32 KiB before adding the existing 64-byte padding. This reduces cache conflicts when the KV length exceeds 16384 and the V head size is at most 256. Larger V heads keep the previous layout. The extra scratch space is less than 32 KiB per row at the default granule.
|
||||
|
||||
`GGML_OPENCL_FA_KV_PITCH_ROUND` sets the rounding granule in bytes to a positive power of two; `0` restores the previous layout. `GGML_OPENCL_FA_KV_PITCH_ROUND_MIN` sets the KV-length threshold (default `16384`), and `GGML_OPENCL_FA_KV_PITCH_ROUND_MAX_DV` sets the largest V head size to round (default `256`). These settings do not change the existing attention dispatch conditions.
|
||||
|
||||
## Program Binary Cache
|
||||
|
||||
Compiled `cl_program` binaries are cached on disk, so subsequent runs skip the expensive
|
||||
|
||||
@@ -195,6 +195,7 @@ The models below were tested with `llama-cli` and `Q4_K_M` files on an Intel® C
|
||||
| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ | ✓ | ✓ |
|
||||
| [LiquidAI/LFM2.5-2.6B-Q4_K_M](https://huggingface.co/LiquidAI/LFM2.5-2.6B-GGUF) | ✓ | ✓ | ✗ |
|
||||
| [bartowski/inclusionAI_Ling-mini-2.0-Q4_K_M](https://huggingface.co/bartowski/inclusionAI_Ling-mini-2.0-GGUF) | ✓ | ✓ | ✗ |
|
||||
| [tiiuae/Falcon-H1-0.5B-Instruct-Q4_K_M](https://huggingface.co/tiiuae/Falcon-H1-0.5B-Instruct-GGUF) | ✓ | ✓ | ✗ |
|
||||
| | | | |
|
||||
| [Qwen/Qwen3-Embedding-0.6B-Q8_0](https://huggingface.co/Qwen/Qwen3-Embedding-0.6B-GGUF) | ✓ | ✓ | ✗ |
|
||||
| [ggml-org/bge-m3-Q8_0](https://huggingface.co/ggml-org/bge-m3-Q8_0-GGUF) | ✓ | ✓ | ✗ |
|
||||
@@ -990,6 +991,7 @@ build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.
|
||||
- Multimodal features (audio/image/video) are a work in progress.
|
||||
- Embedding models: reranking models are not validated.
|
||||
- Llama.cpp tool coverage across CPU/GPU/NPU is not uniform.
|
||||
- Mamba-based models (e.g. Falcon-H1): `llama-perplexity` with several sequences per batch fails; set `-b` to the same value as `-c`. GPU accuracy is slightly lower than CPU.
|
||||
|
||||
**GPU-specific**
|
||||
|
||||
@@ -1012,6 +1014,7 @@ build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.
|
||||
- `llama-perplexity`: requires `-b 512` or smaller.
|
||||
- Flash attention must stay on, which is the default (`-fa auto`). With `-fa off`, the run fails on the NPU.
|
||||
- `llama-embedding` fails on the NPU.
|
||||
- Mamba-based models (e.g. Falcon-H1) fail on the NPU with `Failed to set tensor` (element type mismatch).
|
||||
- Lunar Lake NPU: decode slows down sharply when the context is large. The context size at which this starts depends on the model.
|
||||
- **Workaround:** Pass the smallest context that fits the use, e.g. `-c 512`, or `-c 256` for larger models.
|
||||
- Panther Lake NPU: the load of a Qwen3.5 model can end with a segmentation fault in the NPU compiler of OpenVINO 2026.4.1.
|
||||
|
||||
@@ -17,6 +17,7 @@ Supported models:
|
||||
- [MiniMax Music 3](#minimax-music-3) (songs)
|
||||
- [Kokoro](#kokoro)
|
||||
- [tiny-voca](#tiny-voca)
|
||||
- [VoxCPM2](#voxcpm2)
|
||||
|
||||
The sections below list the params supported by each model, with `llama-tts` examples. In `llama-server`, the same options are set with `--tts-lang` and `--tts-voice`, or per request (see the [`/tts` endpoint](../../tools/server/README.md#post-tts-generate-speech-audio-from-text)).
|
||||
|
||||
@@ -467,3 +468,45 @@ pip install git+https://github.com/n4ze3m/dizione
|
||||
python convert_hf_to_gguf.py path/to/tiny-voca --outfile tiny-voca.gguf
|
||||
python convert_hf_to_gguf.py path/to/tiny-voca --mmproj --outfile mmproj-tiny-voca.gguf
|
||||
```
|
||||
|
||||
## VoxCPM2
|
||||
|
||||
[VoxCPM2](https://huggingface.co/openbmb/VoxCPM2): a MiniCPM4 backbone (2B) makes one hidden state per frame, without an audio token. The mmproj turns it into a patch of 4 continuous latents: FSQ, an 8-layer residual LM (run with libllama, one context per sequence), then the LocDiT flow matching. The causal AudioVAE decodes the latents to 48 kHz audio.
|
||||
|
||||
Available params:
|
||||
- The language is the one of the text, `--tts-lang` is not used. A voice description in parentheses at the start of the text designs the voice, for example `(A young woman, gentle and warm)Hello!`
|
||||
- Voice cloning: `--tts-speaker-file` with the reference clip (resampled to 16 kHz, truncated to the first 30 s). `--tts-speaker-text` is not used, the transcript of the clip is not needed
|
||||
- `--temp` scales the noise of the flow matching (default 1.0), `--top-k` and `--top-p` are not used
|
||||
- The flow matching uses 10 Euler steps and classifier-free guidance with scale 2.0, like the reference. `llama-server` takes `flow_steps` (2 to 32) and `cfg_scale` to change them
|
||||
- Each frame is 4 latents, 7680 samples (160 ms). Generation stops on the stop head of the model, at least 4 frames, at most 6 frames per text token + 10 (the limit of the reference)
|
||||
- Not supported yet: the continuation mode of the reference (prompt audio with its transcript)
|
||||
- Not supported yet: the bad case retry of the reference (`retry_badcase`, a new seed when the audio reaches 6 frames per text token). Short prompts may then run long and repeat some words, try another seed
|
||||
|
||||
Example usage:
|
||||
|
||||
```sh
|
||||
llama-tts -m VoxCPM2.gguf \
|
||||
-mm mmproj-VoxCPM2.gguf \
|
||||
-p "(A calm male narrator)The quick brown fox jumps over the lazy dog." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
Voice cloning:
|
||||
|
||||
```sh
|
||||
llama-tts -m VoxCPM2.gguf \
|
||||
-mm mmproj-VoxCPM2.gguf \
|
||||
--tts-speaker-file reference.wav \
|
||||
-p "Hello, this is a cloned voice." \
|
||||
--output out.wav
|
||||
```
|
||||
|
||||
**Note for GGUF conversion:**
|
||||
|
||||
The mmproj holds LocEnc, LocDiT and the residual LM layers (loaded with libllama as a Granite text model with RoPE off) in the `--outtype` type, and the AudioVAE (`audiovae.pth`, encoder for voice cloning, decoder for the audio) always in F32:
|
||||
|
||||
```sh
|
||||
hf download openbmb/VoxCPM2 --local-dir VoxCPM2
|
||||
python convert_hf_to_gguf.py VoxCPM2 --outfile VoxCPM2.gguf
|
||||
python convert_hf_to_gguf.py VoxCPM2 --mmproj --outfile mmproj-VoxCPM2.gguf
|
||||
```
|
||||
|
||||
+1
-1
@@ -90,7 +90,7 @@ Legend:
|
||||
| PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ |
|
||||
| POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
|
||||
+128
-128
@@ -1155,134 +1155,134 @@
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[1,8,1,3],nr23=[1,1],r=2,v=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -8,10 +8,10 @@ extern "C" {
|
||||
|
||||
#define RPC_PROTO_MAJOR_VERSION 10
|
||||
#define RPC_PROTO_MINOR_VERSION 0
|
||||
#define RPC_PROTO_PATCH_VERSION 0
|
||||
#define RPC_PROTO_PATCH_VERSION 1
|
||||
|
||||
#ifdef __cplusplus
|
||||
static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
|
||||
static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
|
||||
#endif
|
||||
|
||||
#define GGML_RPC_MAX_SERVERS 16
|
||||
|
||||
+8
-6
@@ -429,6 +429,8 @@ extern "C" {
|
||||
|
||||
GGML_OP_GLU,
|
||||
|
||||
GGML_OP_SLEEP,
|
||||
|
||||
GGML_OP_COUNT,
|
||||
};
|
||||
|
||||
@@ -1314,12 +1316,6 @@ extern "C" {
|
||||
|
||||
// indirect matrix multiplication
|
||||
// an id of -1 skips the slot and zeroes the matching row of the result
|
||||
//
|
||||
// [TAG_MUL_MAT_ID_DUPLICATE_ID]
|
||||
// note: this op has undefined behaviour if a given `a` is referenced more than once for a given `b`
|
||||
// (i.e. duplicate experts per token are not guaranteed to be supported in all backends)
|
||||
// TODO: add backend support: https://github.com/ggml-org/llama.cpp/pull/29998#issuecomment-6153742533
|
||||
// the CPU backend skips negative ids, their rows of the result are zero
|
||||
GGML_API struct ggml_tensor * ggml_mul_mat_id(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * as,
|
||||
@@ -2599,6 +2595,12 @@ extern "C" {
|
||||
int n_tasks,
|
||||
void * userdata);
|
||||
|
||||
// Busy-wait operator for scheduling and backend testing.
|
||||
GGML_API struct ggml_tensor * ggml_sleep(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a, // wait after this tensor has executed, return value is simply a full view of this tensor
|
||||
int32_t us); // amount of time to wait in microseconds
|
||||
|
||||
// loss function
|
||||
|
||||
GGML_API struct ggml_tensor * ggml_cross_entropy_loss(
|
||||
|
||||
@@ -231,7 +231,7 @@ if (GGML_OPENMP_FETCH)
|
||||
message(FATAL_ERROR "GGML_OPENMP_FETCH currently requires Clang on Windows")
|
||||
endif()
|
||||
|
||||
set(GGML_OPENMP_LLVM_VERSION "20.1.8")
|
||||
set(GGML_OPENMP_LLVM_VERSION "22.1.8")
|
||||
string(REGEX MATCH "^[0-9]+" GGML_OPENMP_LLVM_VERSION_MAJOR "${GGML_OPENMP_LLVM_VERSION}")
|
||||
string(REGEX MATCH "^[0-9]+" GGML_OPENMP_COMPILER_VERSION_MAJOR "${CMAKE_C_COMPILER_VERSION}")
|
||||
if (NOT GGML_OPENMP_COMPILER_VERSION_MAJOR STREQUAL GGML_OPENMP_LLVM_VERSION_MAJOR)
|
||||
@@ -242,11 +242,11 @@ if (GGML_OPENMP_FETCH)
|
||||
if (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(amd64|x86_64)$")
|
||||
set(GGML_OPENMP_ARCH "x64")
|
||||
set(GGML_OPENMP_INSTALLER_SUFFIX "win64")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "3197846a2b19063687dd56e93e34cd941e3548d907f23a6131571321bdf9fe7b")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "16e5709785fef73c854646241c4a92c5cd574318d1b33c63330dd7721903e55c")
|
||||
elseif (GGML_OPENMP_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$")
|
||||
set(GGML_OPENMP_ARCH "arm64")
|
||||
set(GGML_OPENMP_INSTALLER_SUFFIX "woa64")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "7c4ac97eb2ae6b960ca5f9caf3ff6124c8d2a18cc07a7840a4d2ea15537bad8e")
|
||||
set(GGML_OPENMP_INSTALLER_SHA256 "76f44ef1ba6eeb5a65904e9500f042f588fade49952778ce48f0374daa934396")
|
||||
else()
|
||||
message(FATAL_ERROR "GGML_OPENMP_FETCH does not support ${CMAKE_SYSTEM_PROCESSOR}")
|
||||
endif()
|
||||
|
||||
@@ -2584,6 +2584,7 @@ bool ggml_op_alloc_size_may_expand(enum ggml_op op) {
|
||||
case GGML_OP_ARGSORT:
|
||||
case GGML_OP_TOP_K:
|
||||
case GGML_OP_ARGMAX:
|
||||
case GGML_OP_CONV_2D:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -2501,6 +2501,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
|
||||
break;
|
||||
case GGML_OP_SET_ROWS:
|
||||
{
|
||||
if (op->src[0]->type == GGML_TYPE_BF16) {
|
||||
return false;
|
||||
}
|
||||
switch (op->type) {
|
||||
case GGML_TYPE_F32:
|
||||
case GGML_TYPE_F16:
|
||||
@@ -2553,6 +2556,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
|
||||
if (((const int32_t *) op->op_params)[15] != 0) {
|
||||
return false; // FIXME: support ggml_rope_set_offset
|
||||
}
|
||||
if (op->src[0]->type != GGML_TYPE_F32 && op->src[0]->type != GGML_TYPE_F16) {
|
||||
return false;
|
||||
}
|
||||
if (op->src[0]->ne[0] > 896) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -262,13 +262,22 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
v128_t v0_0 = wasm_v128_load(x0->qs);
|
||||
v128_t v0_0l = wasm_v128_and(v0_0, m4b);
|
||||
v128_t v0_0h = wasm_u8x16_shr(v0_0, 4);
|
||||
v128_t v0_0ls = wasm_i8x16_sub(v0_0l, s8b);
|
||||
v128_t v0_0hs = wasm_i8x16_sub(v0_0h, s8b);
|
||||
|
||||
// Load y0 vectors
|
||||
v128_t y0_l = wasm_v128_load(y0->qs);
|
||||
v128_t y0_h = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// unsigned nibbles (0..15) in the 7-bit operand, then subtract 8*sum(y)
|
||||
v128_t dp0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_l, v0_0l, wasm_i32x4_splat(0));
|
||||
dp0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_h, v0_0h, dp0);
|
||||
v128_t ys0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_l, s8b, wasm_i32x4_splat(0));
|
||||
ys0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y0_h, s8b, ys0);
|
||||
dp0 = wasm_i32x4_sub(dp0, ys0);
|
||||
# else
|
||||
v128_t v0_0ls = wasm_i8x16_sub(v0_0l, s8b);
|
||||
v128_t v0_0hs = wasm_i8x16_sub(v0_0h, s8b);
|
||||
|
||||
// Extend to i16x8 and compute dot products
|
||||
v128_t dx0l = wasm_i16x8_extend_low_i8x16(v0_0ls);
|
||||
v128_t dx0h = wasm_i16x8_extend_high_i8x16(v0_0ls);
|
||||
@@ -290,18 +299,27 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_dot_i16x8(dx0hh, dy0hh)
|
||||
)
|
||||
);
|
||||
# endif
|
||||
|
||||
// Load and process x1
|
||||
v128_t v0_1 = wasm_v128_load(x1->qs);
|
||||
v128_t v0_1l = wasm_v128_and(v0_1, m4b);
|
||||
v128_t v0_1h = wasm_u8x16_shr(v0_1, 4);
|
||||
v128_t v0_1ls = wasm_i8x16_sub(v0_1l, s8b);
|
||||
v128_t v0_1hs = wasm_i8x16_sub(v0_1h, s8b);
|
||||
|
||||
// Load y1 vectors
|
||||
v128_t y1_l = wasm_v128_load(y1->qs);
|
||||
v128_t y1_h = wasm_v128_load(y1->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// unsigned nibbles (0..15) in the 7-bit operand, then subtract 8*sum(y)
|
||||
v128_t dp1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_l, v0_1l, wasm_i32x4_splat(0));
|
||||
dp1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_h, v0_1h, dp1);
|
||||
v128_t ys1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_l, s8b, wasm_i32x4_splat(0));
|
||||
ys1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(y1_h, s8b, ys1);
|
||||
dp1 = wasm_i32x4_sub(dp1, ys1);
|
||||
# else
|
||||
v128_t v0_1ls = wasm_i8x16_sub(v0_1l, s8b);
|
||||
v128_t v0_1hs = wasm_i8x16_sub(v0_1h, s8b);
|
||||
// Extend to i16x8 and compute dot products
|
||||
v128_t dx1l = wasm_i16x8_extend_low_i8x16(v0_1ls);
|
||||
v128_t dx1h = wasm_i16x8_extend_high_i8x16(v0_1ls);
|
||||
@@ -323,6 +341,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_dot_i16x8(dx1hh, dy1hh)
|
||||
)
|
||||
);
|
||||
# endif
|
||||
|
||||
// Accumulate results with scaling
|
||||
float scale0 = GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d);
|
||||
@@ -461,10 +480,18 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
// extract the 5th bit
|
||||
memcpy(&qh_, x0->qh, sizeof(qh_));
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// the high bit as is: unsigned 5-bit quants for the 7-bit operand
|
||||
tmp[0] = table_b2b_0[(qh_ >> 0) & 0xFF];
|
||||
tmp[1] = table_b2b_0[(qh_ >> 8) & 0xFF];
|
||||
tmp[2] = table_b2b_0[(qh_ >> 16) & 0xFF];
|
||||
tmp[3] = table_b2b_0[(qh_ >> 24) ];
|
||||
# else
|
||||
tmp[0] = table_b2b_1[(qh_ >> 0) & 0xFF];
|
||||
tmp[1] = table_b2b_1[(qh_ >> 8) & 0xFF];
|
||||
tmp[2] = table_b2b_1[(qh_ >> 16) & 0xFF];
|
||||
tmp[3] = table_b2b_1[(qh_ >> 24) ];
|
||||
# endif
|
||||
|
||||
const v128_t qhl = wasm_v128_load(tmp + 0);
|
||||
const v128_t qhh = wasm_v128_load(tmp + 2);
|
||||
@@ -475,14 +502,29 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
const v128_t v0l = wasm_v128_and (v0, m4b);
|
||||
const v128_t v0h = wasm_u8x16_shr(v0, 4);
|
||||
|
||||
// add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
|
||||
const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);
|
||||
const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);
|
||||
|
||||
// load y
|
||||
const v128_t v1l = wasm_v128_load(y0->qs);
|
||||
const v128_t v1h = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// unsigned 5-bit quants (0..31) in the 7-bit operand, then subtract 16*sum(y)
|
||||
const v128_t s16b = wasm_i8x16_splat(0x10);
|
||||
const v128_t v0lu = wasm_v128_or(v0l, qhl);
|
||||
const v128_t v0hu = wasm_v128_or(v0h, qhh);
|
||||
|
||||
v128_t dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1l, v0lu, wasm_i32x4_splat(0));
|
||||
dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1h, v0hu, dp);
|
||||
v128_t ys = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1l, s16b, wasm_i32x4_splat(0));
|
||||
ys = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1h, s16b, ys);
|
||||
dp = wasm_i32x4_sub(dp, ys);
|
||||
|
||||
sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(dp),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# else
|
||||
// add high bit and sub 16 (equivalent to sub 0x10 when bit is zero)
|
||||
const v128_t v0lf = wasm_i8x16_sub(v0l, qhl);
|
||||
const v128_t v0hf = wasm_i8x16_sub(v0h, qhh);
|
||||
|
||||
// int8x16 -> int16x8
|
||||
const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);
|
||||
const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);
|
||||
@@ -502,6 +544,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),
|
||||
wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# endif
|
||||
}
|
||||
|
||||
sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
|
||||
@@ -578,6 +621,13 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
const v128_t v1l = wasm_v128_load(y0->qs);
|
||||
const v128_t v1h = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1l, v0lf, wasm_i32x4_splat(0));
|
||||
dp = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(v1h, v0hf, dp);
|
||||
|
||||
sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(dp),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# else
|
||||
// int8x16 -> int16x8
|
||||
const v128_t v0lfl = wasm_i16x8_extend_low_i8x16 (v0lf);
|
||||
const v128_t v0lfh = wasm_i16x8_extend_high_i8x16(v0lf);
|
||||
@@ -597,6 +647,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i32x4_add(wasm_i32x4_dot_i16x8(v0hfl, v1hl),
|
||||
wasm_i32x4_dot_i16x8(v0hfh, v1hh)))),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
# endif
|
||||
}
|
||||
|
||||
sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
|
||||
@@ -642,6 +693,17 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
const v128_t y0_0 = wasm_v128_load(y0->qs);
|
||||
const v128_t y0_1 = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// |x| (0..127) in the 7-bit operand and y*sign(x) in the other; relies on
|
||||
// q8_0 values being in -127..127, as ggml's quantizers produce
|
||||
const v128_t zero = wasm_i8x16_splat(0);
|
||||
const v128_t sx0 = wasm_i8x16_lt(x0_0, zero);
|
||||
const v128_t sx1 = wasm_i8x16_lt(x0_1, zero);
|
||||
const v128_t ys0 = wasm_i8x16_sub(wasm_v128_xor(y0_0, sx0), sx0);
|
||||
const v128_t ys1 = wasm_i8x16_sub(wasm_v128_xor(y0_1, sx1), sx1);
|
||||
v128_t sum_dots = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(ys0, wasm_i8x16_abs(x0_0), wasm_i32x4_splat(0));
|
||||
sum_dots = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(ys1, wasm_i8x16_abs(x0_1), sum_dots);
|
||||
# else
|
||||
// Extend 8-bit to 16-bit
|
||||
const v128_t x0_0l = wasm_i16x8_extend_low_i8x16(x0_0);
|
||||
const v128_t x0_0h = wasm_i16x8_extend_high_i8x16(x0_0);
|
||||
@@ -661,7 +723,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
|
||||
// Sum all dot products
|
||||
const v128_t sum_dots = wasm_i32x4_add(wasm_i32x4_add(dx0_0, dx0_1), wasm_i32x4_add(dx1_0, dx1_1));
|
||||
|
||||
# endif
|
||||
// Convert to float and accumulate
|
||||
const float scale = GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d);
|
||||
sumv = wasm_f32x4_add(sumv, wasm_f32x4_mul(wasm_f32x4_convert_i32x4(sum_dots), wasm_f32x4_splat(scale)));
|
||||
@@ -750,6 +812,10 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
v128_t q2_shift_1 = wasm_u8x16_shr(q2_1, shift);
|
||||
v128_t q2_bits_1 = wasm_v128_and(q2_shift_1, wasm_i8x16_splat(0x03));
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t p_sum0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_0, q2_bits_0, wasm_i32x4_splat(0));
|
||||
v128_t p_sum1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_1, q2_bits_1, wasm_i32x4_splat(0));
|
||||
# else
|
||||
// Calculate dot products
|
||||
v128_t p0 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q8_0),
|
||||
@@ -768,10 +834,14 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q2_bits_1)
|
||||
);
|
||||
|
||||
v128_t p_sum0 = wasm_i32x4_add(p0, p1);
|
||||
v128_t p_sum1 = wasm_i32x4_add(p2, p3);
|
||||
# endif
|
||||
|
||||
// Accumulate scaled results
|
||||
v128_t scaled = wasm_i32x4_add(
|
||||
wasm_i32x4_mul(wasm_i32x4_add(p0, p1), wasm_i32x4_splat(d0)),
|
||||
wasm_i32x4_mul(wasm_i32x4_add(p2, p3), wasm_i32x4_splat(d1))
|
||||
wasm_i32x4_mul(p_sum0, wasm_i32x4_splat(d0)),
|
||||
wasm_i32x4_mul(p_sum1, wasm_i32x4_splat(d1))
|
||||
);
|
||||
|
||||
isum_vec = wasm_i32x4_add(isum_vec, scaled);
|
||||
@@ -817,7 +887,84 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
|
||||
const int nb = n / QK_K;
|
||||
|
||||
#if defined __wasm_simd128__
|
||||
#if defined(__wasm_relaxed_simd__)
|
||||
// Unsigned 3-bit quants (0..7) in the relaxed dot's 7-bit operand, q8 in
|
||||
// the other; the -4 offset is taken off with the q8 block sums.
|
||||
const v128_t m3 = wasm_i8x16_splat(0x03);
|
||||
const v128_t m1 = wasm_i8x16_splat(0x01);
|
||||
const v128_t zero = wasm_i32x4_splat(0);
|
||||
uint32_t auxs[4];
|
||||
|
||||
float sumf = 0;
|
||||
|
||||
for (int i = 0; i < nb; ++i) {
|
||||
const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
|
||||
|
||||
const uint8_t * GGML_RESTRICT q3 = x[i].qs;
|
||||
const int8_t * GGML_RESTRICT q8 = y[i].qs;
|
||||
|
||||
// Extract scales
|
||||
memcpy(auxs, x[i].scales, 12);
|
||||
uint32_t tmp = auxs[2];
|
||||
auxs[2] = ((auxs[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4);
|
||||
auxs[3] = ((auxs[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4);
|
||||
auxs[0] = (auxs[0] & kmask2) | (((tmp >> 0) & kmask1) << 4);
|
||||
auxs[1] = (auxs[1] & kmask2) | (((tmp >> 2) & kmask1) << 4);
|
||||
|
||||
const v128_t vsc = wasm_i8x16_sub(wasm_v128_load(auxs), wasm_i8x16_splat(32));
|
||||
const v128_t vsc_l = wasm_i16x8_extend_low_i8x16(vsc);
|
||||
const v128_t vsc_h = wasm_i16x8_extend_high_i8x16(vsc);
|
||||
|
||||
// sum(scale * bsum), for the -4 offset
|
||||
const v128_t vmins = wasm_i32x4_add(
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums), vsc_l),
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums + 8), vsc_h));
|
||||
|
||||
const v128_t hm0 = wasm_v128_load(x[i].hmask);
|
||||
const v128_t hm1 = wasm_v128_load(x[i].hmask + 16);
|
||||
|
||||
v128_t acc = zero;
|
||||
v128_t acc2 = zero;
|
||||
|
||||
for (int j = 0; j < QK_K/128; ++j) {
|
||||
const v128_t q3_0 = wasm_v128_load(q3);
|
||||
const v128_t q3_1 = wasm_v128_load(q3 + 16);
|
||||
q3 += 32;
|
||||
|
||||
const v128_t s32a = wasm_i32x4_extend_low_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
const v128_t s32b = wasm_i32x4_extend_high_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
const v128_t s32[2] = { s32a, s32b };
|
||||
|
||||
for (int g = 0; g < 4; ++g) {
|
||||
// 2 low bits at shift 2g, high bit 4j+g of hmask moved to bit 2
|
||||
// (16-bit shifts are safe on masked bytes)
|
||||
const int hb = 4*j + g;
|
||||
const v128_t u0 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(q3_0, 2*g), m3),
|
||||
wasm_i16x8_shl(wasm_v128_and(wasm_u16x8_shr(hm0, hb), m1), 2));
|
||||
const v128_t u1 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(q3_1, 2*g), m3),
|
||||
wasm_i16x8_shl(wasm_v128_and(wasm_u16x8_shr(hm1, hb), m1), 2));
|
||||
|
||||
const v128_t p0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8), u0, zero);
|
||||
const v128_t p1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 16), u1, zero);
|
||||
q8 += 32;
|
||||
|
||||
const v128_t sv = s32[g >> 1];
|
||||
const v128_t sc0 = (g & 1) ? wasm_i32x4_shuffle(sv, sv, 2, 2, 2, 2) : wasm_i32x4_shuffle(sv, sv, 0, 0, 0, 0);
|
||||
const v128_t sc1 = (g & 1) ? wasm_i32x4_shuffle(sv, sv, 3, 3, 3, 3) : wasm_i32x4_shuffle(sv, sv, 1, 1, 1, 1);
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p0, sc0));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p1, sc1));
|
||||
}
|
||||
}
|
||||
|
||||
acc = wasm_i32x4_sub(wasm_i32x4_add(acc, acc2), wasm_i32x4_shl(vmins, 2));
|
||||
const int32_t isum = wasm_i32x4_extract_lane(acc, 0) + wasm_i32x4_extract_lane(acc, 1) +
|
||||
wasm_i32x4_extract_lane(acc, 2) + wasm_i32x4_extract_lane(acc, 3);
|
||||
sumf += d * isum;
|
||||
}
|
||||
|
||||
*s = sumf;
|
||||
|
||||
#elif defined __wasm_simd128__
|
||||
int8_t aux8[QK_K];
|
||||
float sums[8] = {0};
|
||||
uint32_t auxs[4];
|
||||
@@ -986,6 +1133,14 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
q8 += 64;
|
||||
|
||||
// Low nibble products
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t vacc1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x0, q4l0, wasm_i32x4_splat(0)
|
||||
);
|
||||
vacc1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x1, q4l1, vacc1
|
||||
);
|
||||
# else
|
||||
v128_t vacc1 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q4l0),
|
||||
wasm_i16x8_extend_low_i8x16(q8x0)
|
||||
@@ -1002,8 +1157,17 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q4l1),
|
||||
wasm_i16x8_extend_high_i8x16(q8x1)
|
||||
));
|
||||
# endif
|
||||
|
||||
// High nibble products
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
v128_t vacc2 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x2, q4h0, wasm_i32x4_splat(0)
|
||||
);
|
||||
vacc2 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(
|
||||
q8x3, q4h1, vacc2
|
||||
);
|
||||
# else
|
||||
v128_t vacc2 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q4h0),
|
||||
wasm_i16x8_extend_low_i8x16(q8x2)
|
||||
@@ -1020,6 +1184,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q4h1),
|
||||
wasm_i16x8_extend_high_i8x16(q8x3)
|
||||
));
|
||||
# endif
|
||||
|
||||
// Accumulate scaled results
|
||||
int32_t vacc1_sum = wasm_i32x4_extract_lane(vacc1, 0) + wasm_i32x4_extract_lane(vacc1, 1) +
|
||||
@@ -1127,6 +1292,11 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
v128_t q8_3 = wasm_v128_load(q8 + 48);
|
||||
q8 += 64;
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// Process low quants using relaxed SIMD
|
||||
v128_t sum_low = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_0, q5l_0, wasm_i32x4_splat(0));
|
||||
sum_low = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_1, q5l_1, sum_low);
|
||||
# else
|
||||
// Process low quants
|
||||
v128_t pl0 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q5l_0),
|
||||
@@ -1145,7 +1315,13 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q8_1)
|
||||
));
|
||||
v128_t sum_low = wasm_i32x4_add(pl0, pl1);
|
||||
# endif
|
||||
|
||||
# if defined(__wasm_relaxed_simd__)
|
||||
// Process high quants using relaxed SIMD
|
||||
v128_t sum_high = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_2, q5h_0, wasm_i32x4_splat(0));
|
||||
sum_high = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(q8_3, q5h_1, sum_high);
|
||||
# else
|
||||
// Process high quants
|
||||
v128_t ph0 = wasm_i32x4_dot_i16x8(
|
||||
wasm_i16x8_extend_low_i8x16(q5h_0),
|
||||
@@ -1164,6 +1340,7 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
wasm_i16x8_extend_high_i8x16(q8_3)
|
||||
));
|
||||
v128_t sum_high = wasm_i32x4_add(ph0, ph1);
|
||||
# endif
|
||||
|
||||
// Accumulate with scale factors
|
||||
int32_t sl = wasm_i32x4_extract_lane(sum_low, 0) + wasm_i32x4_extract_lane(sum_low, 1) +
|
||||
@@ -1204,7 +1381,78 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
|
||||
const int nb = n / QK_K;
|
||||
|
||||
#if defined __wasm_simd128__
|
||||
#if defined(__wasm_relaxed_simd__)
|
||||
const v128_t m4 = wasm_i8x16_splat(0x0F);
|
||||
const v128_t m2 = wasm_i8x16_splat(0x03);
|
||||
const v128_t zero = wasm_i32x4_splat(0);
|
||||
|
||||
float sumf = 0;
|
||||
|
||||
for (int i = 0; i < nb; ++i) {
|
||||
const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d;
|
||||
const uint8_t * GGML_RESTRICT ql = x[i].ql;
|
||||
const uint8_t * GGML_RESTRICT qh = x[i].qh;
|
||||
const int8_t * GGML_RESTRICT q8 = y[i].qs;
|
||||
|
||||
const v128_t vsc = wasm_v128_load(x[i].scales);
|
||||
const v128_t vsc_l = wasm_i16x8_extend_low_i8x16(vsc);
|
||||
const v128_t vsc_h = wasm_i16x8_extend_high_i8x16(vsc);
|
||||
const v128_t vmins = wasm_i32x4_add(
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums), vsc_l),
|
||||
wasm_i32x4_dot_i16x8(wasm_v128_load(y[i].bsums + 8), vsc_h));
|
||||
|
||||
v128_t acc = zero;
|
||||
v128_t acc2 = zero;
|
||||
for (int j = 0; j < QK_K/128; ++j) {
|
||||
const v128_t ql0 = wasm_v128_load(ql);
|
||||
const v128_t ql1 = wasm_v128_load(ql + 16);
|
||||
const v128_t ql2 = wasm_v128_load(ql + 32);
|
||||
const v128_t ql3 = wasm_v128_load(ql + 48);
|
||||
const v128_t qh0 = wasm_v128_load(qh);
|
||||
const v128_t qh1 = wasm_v128_load(qh + 16);
|
||||
ql += 64;
|
||||
qh += 32;
|
||||
|
||||
const v128_t q6_0 = wasm_v128_or(wasm_v128_and(ql0, m4), wasm_i16x8_shl(wasm_v128_and(qh0, m2), 4));
|
||||
const v128_t q6_1 = wasm_v128_or(wasm_v128_and(ql1, m4), wasm_i16x8_shl(wasm_v128_and(qh1, m2), 4));
|
||||
const v128_t q6_2 = wasm_v128_or(wasm_v128_and(ql2, m4), wasm_i16x8_shl(wasm_v128_and(qh0, wasm_i8x16_splat(0x0C)), 2));
|
||||
const v128_t q6_3 = wasm_v128_or(wasm_v128_and(ql3, m4), wasm_i16x8_shl(wasm_v128_and(qh1, wasm_i8x16_splat(0x0C)), 2));
|
||||
const v128_t q6_4 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql0, 4), m4), wasm_v128_and(qh0, wasm_i8x16_splat(0x30)));
|
||||
const v128_t q6_5 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql1, 4), m4), wasm_v128_and(qh1, wasm_i8x16_splat(0x30)));
|
||||
const v128_t q6_6 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql2, 4), m4), wasm_u16x8_shr(wasm_v128_and(qh0, wasm_i8x16_splat((int8_t)0xC0)), 2));
|
||||
const v128_t q6_7 = wasm_v128_or(wasm_v128_and(wasm_u16x8_shr(ql3, 4), m4), wasm_u16x8_shr(wasm_v128_and(qh1, wasm_i8x16_splat((int8_t)0xC0)), 2));
|
||||
|
||||
const v128_t p0 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 0), q6_0, zero);
|
||||
const v128_t p1 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 16), q6_1, zero);
|
||||
const v128_t p2 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 32), q6_2, zero);
|
||||
const v128_t p3 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 48), q6_3, zero);
|
||||
const v128_t p4 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 64), q6_4, zero);
|
||||
const v128_t p5 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 80), q6_5, zero);
|
||||
const v128_t p6 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 96), q6_6, zero);
|
||||
const v128_t p7 = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_v128_load(q8 + 112), q6_7, zero);
|
||||
q8 += 128;
|
||||
|
||||
const v128_t s32a = wasm_i32x4_extend_low_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
const v128_t s32b = wasm_i32x4_extend_high_i16x8(j == 0 ? vsc_l : vsc_h);
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p0, wasm_i32x4_shuffle(s32a, s32a, 0, 0, 0, 0)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p1, wasm_i32x4_shuffle(s32a, s32a, 1, 1, 1, 1)));
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p2, wasm_i32x4_shuffle(s32a, s32a, 2, 2, 2, 2)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p3, wasm_i32x4_shuffle(s32a, s32a, 3, 3, 3, 3)));
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p4, wasm_i32x4_shuffle(s32b, s32b, 0, 0, 0, 0)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p5, wasm_i32x4_shuffle(s32b, s32b, 1, 1, 1, 1)));
|
||||
acc = wasm_i32x4_add(acc, wasm_i32x4_mul(p6, wasm_i32x4_shuffle(s32b, s32b, 2, 2, 2, 2)));
|
||||
acc2 = wasm_i32x4_add(acc2, wasm_i32x4_mul(p7, wasm_i32x4_shuffle(s32b, s32b, 3, 3, 3, 3)));
|
||||
}
|
||||
|
||||
acc = wasm_i32x4_sub(wasm_i32x4_add(acc, acc2), wasm_i32x4_shl(vmins, 5));
|
||||
const int32_t isum = wasm_i32x4_extract_lane(acc, 0) + wasm_i32x4_extract_lane(acc, 1) +
|
||||
wasm_i32x4_extract_lane(acc, 2) + wasm_i32x4_extract_lane(acc, 3);
|
||||
sumf += d * isum;
|
||||
}
|
||||
|
||||
*s = sumf;
|
||||
|
||||
#elif defined __wasm_simd128__
|
||||
int8_t aux8[QK_K] __attribute__((aligned(16)));
|
||||
int32_t aux32[8] __attribute__((aligned(16))) = {0};
|
||||
float sums[8] __attribute__((aligned(16))) = {0};
|
||||
|
||||
@@ -861,8 +861,8 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
__m128 acc_3 = _mm_setzero_ps();
|
||||
|
||||
for (; ib + 1 < nb; ib += 2) {
|
||||
_mm_prefetch(&x[ib] + sizeof(block_q4_0), _MM_HINT_T0);
|
||||
_mm_prefetch(&y[ib] + sizeof(block_q8_0), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&x[ib] + sizeof(block_q4_0)), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&y[ib] + sizeof(block_q8_0)), _MM_HINT_T0);
|
||||
|
||||
// Compute combined scale for the block 0 and 1
|
||||
const __m128 d_0_1 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib].d) * GGML_CPU_FP16_TO_FP32(y[ib].d) );
|
||||
@@ -879,8 +879,8 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
||||
bx_1 = _mm_sub_epi8(bx_1, off);
|
||||
const __m128i i32_1 = mul_sum_i8_pairs(bx_1, by_1);
|
||||
|
||||
_mm_prefetch(&x[ib] + 2 * sizeof(block_q4_0), _MM_HINT_T0);
|
||||
_mm_prefetch(&y[ib] + 2 * sizeof(block_q8_0), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&x[ib] + 2 * sizeof(block_q4_0)), _MM_HINT_T0);
|
||||
_mm_prefetch((const char *) (&y[ib] + 2 * sizeof(block_q8_0)), _MM_HINT_T0);
|
||||
|
||||
// Compute combined scale for the block 2 and 3
|
||||
const __m128 d_2_3 = _mm_set1_ps( GGML_CPU_FP16_TO_FP32(x[ib + 1].d) * GGML_CPU_FP16_TO_FP32(y[ib + 1].d) );
|
||||
|
||||
@@ -2034,6 +2034,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
|
||||
{
|
||||
ggml_compute_forward_fill(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_SLEEP:
|
||||
{
|
||||
ggml_compute_forward_sleep(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_FLASH_ATTN_EXT:
|
||||
{
|
||||
ggml_compute_forward_flash_attn_ext(params, tensor);
|
||||
@@ -2306,6 +2310,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
|
||||
case GGML_OP_REPEAT:
|
||||
case GGML_OP_REPEAT_BACK:
|
||||
case GGML_OP_LEAKY_RELU:
|
||||
case GGML_OP_SLEEP:
|
||||
{
|
||||
n_tasks = 1;
|
||||
} break;
|
||||
@@ -2378,10 +2383,8 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
|
||||
} break;
|
||||
case GGML_OP_GET_ROWS:
|
||||
{
|
||||
// FIXME: get_rows can use additional threads, but the cost of launching additional threads
|
||||
// decreases performance with GPU offloading
|
||||
//n_tasks = n_threads;
|
||||
n_tasks = 1;
|
||||
// waking the other threads costs more than a small lookup, e.g. when this is the only CPU node of an offloaded model
|
||||
n_tasks = ggml_nelements(node) >= 262144 ? n_threads : 1;
|
||||
} break;
|
||||
case GGML_OP_SCALE:
|
||||
case GGML_OP_SET:
|
||||
@@ -2930,7 +2933,8 @@ struct ggml_cplan ggml_graph_plan(
|
||||
} break;
|
||||
case GGML_OP_SET_ROWS:
|
||||
{
|
||||
if (node->src[0]->type == GGML_TYPE_F16 && node->type != GGML_TYPE_F16) {
|
||||
if ((node->src[0]->type == GGML_TYPE_F16 && node->type != GGML_TYPE_F16) ||
|
||||
(node->src[0]->type == GGML_TYPE_BF16 && node->type != GGML_TYPE_BF16)) {
|
||||
cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
|
||||
}
|
||||
} break;
|
||||
|
||||
@@ -2238,6 +2238,19 @@ void ggml_compute_forward_fill(const ggml_compute_params * params, ggml_tensor *
|
||||
}
|
||||
}
|
||||
|
||||
// ggml_compute_forward_sleep
|
||||
|
||||
void ggml_compute_forward_sleep(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
if (params->ith != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// same clock as ggml_time_us() callers; on macOS steady_clock is CLOCK_MONOTONIC_RAW, which can run faster
|
||||
const int64_t t_end = ggml_time_us() + ggml_get_op_params_i32(dst, 0);
|
||||
|
||||
while (ggml_time_us() < t_end) {}
|
||||
}
|
||||
|
||||
// ggml_compute_tri
|
||||
|
||||
template <typename T>
|
||||
@@ -5518,7 +5531,7 @@ static void ggml_compute_forward_set_rows_impl(
|
||||
assert(ne0 == nc);
|
||||
assert(ne2 == ne02);
|
||||
assert(ne3 == ne03);
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);
|
||||
assert(ne02 % ne11 == 0);
|
||||
assert(ne03 % ne12 == 0);
|
||||
|
||||
@@ -5565,6 +5578,20 @@ static void ggml_compute_forward_set_rows_impl(
|
||||
from_float(wdata,
|
||||
((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc);
|
||||
}
|
||||
} else if constexpr (std::is_same_v<src_t, ggml_bf16_t>) {
|
||||
if (dst->type == GGML_TYPE_BF16) {
|
||||
memcpy(
|
||||
((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3),
|
||||
((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03),
|
||||
rs);
|
||||
} else {
|
||||
float * wdata = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith;
|
||||
ggml_bf16_to_fp32_row(
|
||||
(const ggml_bf16_t *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03),
|
||||
wdata, nc);
|
||||
from_float(wdata,
|
||||
((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc);
|
||||
}
|
||||
} else {
|
||||
GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type));
|
||||
}
|
||||
@@ -5601,6 +5628,16 @@ void ggml_compute_forward_set_rows(
|
||||
GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type));
|
||||
}
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
if (src1->type == GGML_TYPE_I64) {
|
||||
ggml_compute_forward_set_rows_impl<ggml_bf16_t, int64_t>(params, dst);
|
||||
} else if (src1->type == GGML_TYPE_I32) {
|
||||
ggml_compute_forward_set_rows_impl<ggml_bf16_t, int32_t>(params, dst);
|
||||
} else {
|
||||
GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type));
|
||||
}
|
||||
} break;
|
||||
default:
|
||||
{
|
||||
GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type));
|
||||
@@ -6289,7 +6326,7 @@ static void rotate_pairs(const int64_t n, const int64_t n_offset, const float *
|
||||
}
|
||||
}
|
||||
|
||||
template<typename T> //float or ggml_fp16_t
|
||||
template<typename T> //float, ggml_fp16_t or ggml_bf16_t
|
||||
static void ggml_compute_forward_rope_flt(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst,
|
||||
@@ -6299,7 +6336,7 @@ static void ggml_compute_forward_rope_flt(
|
||||
const ggml_tensor * src1 = dst->src[1];
|
||||
const ggml_tensor * src2 = dst->src[2];
|
||||
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);
|
||||
GGML_ASSERT(src1->type == GGML_TYPE_I32);
|
||||
|
||||
float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
|
||||
@@ -6459,6 +6496,10 @@ void ggml_compute_forward_rope(
|
||||
{
|
||||
ggml_compute_forward_rope_flt<ggml_fp16_t>(params, dst, true);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_rope_flt<ggml_bf16_t>(params, dst, true);
|
||||
} break;
|
||||
case GGML_TYPE_F32:
|
||||
{
|
||||
ggml_compute_forward_rope_flt<float>(params, dst, true);
|
||||
@@ -6483,6 +6524,10 @@ void ggml_compute_forward_rope_back(
|
||||
{
|
||||
ggml_compute_forward_rope_flt<ggml_fp16_t>(params, dst, false);
|
||||
} break;
|
||||
case GGML_TYPE_BF16:
|
||||
{
|
||||
ggml_compute_forward_rope_flt<ggml_bf16_t>(params, dst, false);
|
||||
} break;
|
||||
case GGML_TYPE_F32:
|
||||
{
|
||||
ggml_compute_forward_rope_flt<float>(params, dst, false);
|
||||
|
||||
@@ -84,6 +84,7 @@ void ggml_compute_forward_top_k(const struct ggml_compute_params * params, struc
|
||||
void ggml_compute_forward_leaky_relu(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_fill(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_sleep(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_flash_attn_ext(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_flash_attn_back(
|
||||
const struct ggml_compute_params * params,
|
||||
|
||||
@@ -133,6 +133,8 @@ if (CUDAToolkit_FOUND)
|
||||
${GGML_SOURCES_CUDA}
|
||||
)
|
||||
|
||||
target_link_libraries(ggml-cuda PRIVATE ${CMAKE_DL_LIBS})
|
||||
|
||||
if (GGML_CUDA_GRAPHS)
|
||||
add_compile_definitions(GGML_CUDA_USE_GRAPHS)
|
||||
endif()
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
#include "batch-get.cuh"
|
||||
#include "batch-copy.cuh"
|
||||
#include "common.cuh"
|
||||
#include "ggml-cpp.h"
|
||||
|
||||
#include <climits>
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
struct ggml_cuda_batch_get_descriptor {
|
||||
struct ggml_cuda_batch_copy_descriptor {
|
||||
const unsigned char * src;
|
||||
unsigned char * dst;
|
||||
size_t size;
|
||||
};
|
||||
|
||||
static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_descriptor * copies, size_t count) {
|
||||
static __global__ void ggml_cuda_batch_copy_kernel(const ggml_cuda_batch_copy_descriptor * copies, size_t count) {
|
||||
// Blocks along y cooperate on each descriptor.
|
||||
const size_t index = blockIdx.x;
|
||||
if (index >= count) {
|
||||
@@ -50,11 +50,11 @@ static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_desc
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_cuda_batch_get_mapping {
|
||||
struct ggml_cuda_batch_copy_mapping {
|
||||
void * host = nullptr;
|
||||
unsigned char * device = nullptr;
|
||||
|
||||
ggml_cuda_batch_get_mapping(void * host, size_t size) {
|
||||
ggml_cuda_batch_copy_mapping(void * host, size_t size) {
|
||||
const cudaError_t err = cudaHostRegister(host, size, cudaHostRegisterMapped | cudaHostRegisterPortable);
|
||||
if (err != cudaSuccess) {
|
||||
(void) cudaGetLastError();
|
||||
@@ -70,10 +70,10 @@ struct ggml_cuda_batch_get_mapping {
|
||||
}
|
||||
}
|
||||
|
||||
ggml_cuda_batch_get_mapping(const ggml_cuda_batch_get_mapping &) = delete;
|
||||
ggml_cuda_batch_get_mapping & operator=(const ggml_cuda_batch_get_mapping &) = delete;
|
||||
ggml_cuda_batch_copy_mapping(const ggml_cuda_batch_copy_mapping &) = delete;
|
||||
ggml_cuda_batch_copy_mapping & operator=(const ggml_cuda_batch_copy_mapping &) = delete;
|
||||
|
||||
~ggml_cuda_batch_get_mapping() {
|
||||
~ggml_cuda_batch_copy_mapping() {
|
||||
if (host) {
|
||||
// Release only the registration acquired by this object.
|
||||
CUDA_CHECK(cudaHostUnregister(host));
|
||||
@@ -82,11 +82,12 @@ struct ggml_cuda_batch_get_mapping {
|
||||
};
|
||||
#endif
|
||||
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
static bool ggml_cuda_copy_tensor_n(int device, cudaMemcpyKind direction,
|
||||
const ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
GGML_UNUSED(device);
|
||||
GGML_UNUSED(direction);
|
||||
GGML_UNUSED(tensors);
|
||||
GGML_UNUSED(data);
|
||||
GGML_UNUSED(offsets);
|
||||
@@ -94,14 +95,17 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
GGML_UNUSED(n_tensors);
|
||||
return false;
|
||||
#else
|
||||
const bool to_host = direction == cudaMemcpyDeviceToHost;
|
||||
// Provisional dispatch values based on experiments.
|
||||
constexpr size_t min_batch_reads = 128;
|
||||
constexpr size_t max_gather_read_bytes = 128 * 1024;
|
||||
// Collect non-empty CPU destination ranges.
|
||||
constexpr size_t max_kernel_read_bytes = 128 * 1024;
|
||||
// Collect destination ranges; repeated sources are allowed.
|
||||
std::vector<std::pair<uintptr_t, uintptr_t>> ranges;
|
||||
ranges.reserve(n_tensors);
|
||||
size_t payload_bytes = 0;
|
||||
// Number of non-empty reads whose size is at most max_gather_read_bytes.
|
||||
uintptr_t begin = UINTPTR_MAX;
|
||||
uintptr_t end = 0;
|
||||
// Number of non-empty reads whose size is at most max_kernel_read_bytes.
|
||||
size_t n_reads_small = 0;
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
if (sizes[i] == 0) {
|
||||
@@ -111,15 +115,19 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
GGML_ASSERT(address <= UINTPTR_MAX - sizes[i]);
|
||||
GGML_ASSERT(payload_bytes <= SIZE_MAX - sizes[i]);
|
||||
payload_bytes += sizes[i];
|
||||
n_reads_small += sizes[i] <= max_gather_read_bytes;
|
||||
ranges.emplace_back(address, address + sizes[i]);
|
||||
n_reads_small += sizes[i] <= max_kernel_read_bytes;
|
||||
begin = std::min(begin, address);
|
||||
end = std::max(end, address + sizes[i]);
|
||||
const uintptr_t destination = to_host ? address : (uintptr_t) tensors[i]->data + offsets[i];
|
||||
GGML_ASSERT(destination <= UINTPTR_MAX - sizes[i]);
|
||||
ranges.emplace_back(destination, destination + sizes[i]);
|
||||
}
|
||||
if (ranges.empty()) {
|
||||
return true;
|
||||
}
|
||||
// Total number of non-empty reads.
|
||||
const size_t n_reads = ranges.size();
|
||||
if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_get_descriptor) || n_reads > INT_MAX) {
|
||||
if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_copy_descriptor) || n_reads > INT_MAX) {
|
||||
return false;
|
||||
}
|
||||
if (n_reads < 2 || (n_reads < min_batch_reads && n_reads_small == n_reads)) {
|
||||
@@ -127,72 +135,85 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
}
|
||||
std::sort(ranges.begin(), ranges.end());
|
||||
for (size_t i = 1; i < ranges.size(); ++i) {
|
||||
// Preserve the ordered copy path when CPU destinations overlap.
|
||||
// Preserve the ordered copy path when destinations overlap.
|
||||
if (ranges[i].first < ranges[i - 1].second) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Find the CPU span to register, including gaps between destinations.
|
||||
const uintptr_t begin = ranges.front().first;
|
||||
const uintptr_t end = ranges.back().second;
|
||||
if (end - begin - payload_bytes > payload_bytes) {
|
||||
// Limit gaps in the CPU span without underflow when sources overlap.
|
||||
if (end - begin > payload_bytes && end - begin - payload_bytes > payload_bytes) {
|
||||
return false;
|
||||
}
|
||||
// Register and map the existing CPU destination so the kernel can write to it directly.
|
||||
ggml_cuda_batch_get_mapping mapping((void *) begin, end - begin);
|
||||
// Register and map the CPU span for direct kernel access.
|
||||
ggml_cuda_batch_copy_mapping mapping((void *) begin, end - begin);
|
||||
if (!mapping.host) {
|
||||
return false;
|
||||
}
|
||||
// Reads selected for gather; zero means all reads use DMA.
|
||||
size_t n_reads_gather = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0;
|
||||
// Reads selected for the kernel; zero means all reads use DMA.
|
||||
size_t n_reads_kernel = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0;
|
||||
// Keep descriptor storage alive until the final stream synchronization.
|
||||
constexpr size_t max_batch_descriptors = 128 * 1024;
|
||||
const size_t bytes = std::min(n_reads_gather, max_batch_descriptors) * sizeof(ggml_cuda_batch_get_descriptor);
|
||||
ggml_backend_buffer_ptr descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr);
|
||||
const size_t bytes = std::min(n_reads_kernel, max_batch_descriptors) * sizeof(ggml_cuda_batch_copy_descriptor);
|
||||
ggml_backend_buffer_ptr descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr);
|
||||
if (!descriptors) {
|
||||
n_reads_gather = 0;
|
||||
n_reads_kernel = 0;
|
||||
}
|
||||
const auto host_type = ggml_backend_cuda_host_buffer_type();
|
||||
ggml_backend_buffer_ptr host_descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr);
|
||||
ggml_backend_buffer_ptr host_descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr);
|
||||
if (!host_descriptors || ggml_backend_buffer_get_type(host_descriptors.get()) != host_type) {
|
||||
n_reads_gather = 0;
|
||||
n_reads_kernel = 0;
|
||||
}
|
||||
if (n_reads_gather) {
|
||||
auto * copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get()));
|
||||
auto * device_copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(descriptors.get()));
|
||||
if (n_reads_kernel) {
|
||||
auto * copies = static_cast<ggml_cuda_batch_copy_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get()));
|
||||
auto * device_copies = static_cast<ggml_cuda_batch_copy_descriptor *>(ggml_backend_buffer_get_base(descriptors.get()));
|
||||
const int threads = 256;
|
||||
const size_t nsm = ggml_cuda_info().devices[device].nsm;
|
||||
size_t i = 0;
|
||||
for (size_t first = 0; first < n_reads_gather; first += max_batch_descriptors) {
|
||||
const size_t n_batch = std::min(n_reads_gather - first, max_batch_descriptors);
|
||||
for (size_t first = 0; first < n_reads_kernel; first += max_batch_descriptors) {
|
||||
const size_t n_batch = std::min(n_reads_kernel - first, max_batch_descriptors);
|
||||
size_t count = 0;
|
||||
for (; i < n_tensors && count < n_batch; ++i) {
|
||||
if (sizes[i] == 0 || sizes[i] > max_gather_read_bytes) {
|
||||
if (sizes[i] == 0 || sizes[i] > max_kernel_read_bytes) {
|
||||
continue;
|
||||
}
|
||||
copies[count++] = {(const unsigned char *) tensors[i]->data + offsets[i],
|
||||
mapping.device + ((uintptr_t) data[i] - begin), sizes[i]};
|
||||
auto * host = mapping.device + ((uintptr_t) data[i] - begin);
|
||||
auto * tensor = (unsigned char *) tensors[i]->data + offsets[i];
|
||||
copies[count++] = {to_host ? tensor : host, to_host ? host : tensor, sizes[i]};
|
||||
}
|
||||
const size_t blocks_per_copy = std::min<size_t>(UINT16_MAX, std::max<size_t>(1, (nsm + n_batch - 1) / n_batch));
|
||||
const dim3 blocks((unsigned int) n_batch, (unsigned int) blocks_per_copy);
|
||||
// Stream ordering keeps each kernel ahead of the next descriptor overwrite.
|
||||
CUDA_CHECK(cudaMemcpyAsync(device_copies, copies, n_batch * sizeof(copies[0]), cudaMemcpyHostToDevice, cudaStreamPerThread));
|
||||
ggml_cuda_batch_get_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch);
|
||||
ggml_cuda_batch_copy_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch);
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
if (first + n_batch < n_reads_gather) {
|
||||
if (first + n_batch < n_reads_kernel) {
|
||||
// Wait before refilling the pinned host descriptor buffer.
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Keep the destination pinned for DMA too; overlapping destinations already declined above.
|
||||
// Keep the CPU span pinned for DMA too; overlapping destinations already declined above.
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
if (sizes[i] && (!n_reads_gather || sizes[i] > max_gather_read_bytes)) {
|
||||
CUDA_CHECK(cudaMemcpyAsync(data[i], (const char *) tensors[i]->data + offsets[i], sizes[i], cudaMemcpyDeviceToHost, cudaStreamPerThread));
|
||||
if (sizes[i] && (!n_reads_kernel || sizes[i] > max_kernel_read_bytes)) {
|
||||
void * host = const_cast<void *>(data[i]);
|
||||
void * tensor = (char *) tensors[i]->data + offsets[i];
|
||||
CUDA_CHECK(cudaMemcpyAsync(to_host ? host : tensor, to_host ? tensor : host, sizes[i], direction, cudaStreamPerThread));
|
||||
}
|
||||
}
|
||||
// Wait for GPU writes before releasing the CPU mapping and descriptor buffers.
|
||||
// Wait for transfers before releasing the CPU mapping and descriptor buffers.
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
return ggml_cuda_copy_tensor_n(device, cudaMemcpyDeviceToHost, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
|
||||
bool ggml_cuda_set_tensor_n(int device,
|
||||
ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
return ggml_cuda_copy_tensor_n(device, cudaMemcpyHostToDevice, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
@@ -5,3 +5,7 @@
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors);
|
||||
|
||||
bool ggml_cuda_set_tensor_n(int device,
|
||||
ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors);
|
||||
@@ -59,7 +59,7 @@
|
||||
// While BW spans CC 1000, 1100 & 1200, we are integrating Tensor Core instructions available to 1200 family, see
|
||||
// https://docs.nvidia.com/cutlass/media/docs/cpp/blackwell_functionality.html#blackwell-sm120-gemms
|
||||
#define GGML_CUDA_CC_BLACKWELL 1200
|
||||
#define GGML_CUDA_CC_DGX_SPARK 1210
|
||||
#define GGML_CUDA_CC_GB10 1210
|
||||
#define GGML_CUDA_CC_RUBIN 1300
|
||||
#define GGML_CUDA_CC_OFFSET_AMD 0x1000000
|
||||
#define GGML_CUDA_CC_OFFSET_MTHREADS 0x0100000
|
||||
@@ -452,6 +452,16 @@ static __device__ __forceinline__ ggml_cuda_thread_mask_t ggml_cuda_ballot(int p
|
||||
return __ballot_sync(mask, predicate);
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ int ggml_cuda_popc(ggml_cuda_thread_mask_t x) {
|
||||
static_assert(sizeof(ggml_cuda_thread_mask_t) == 4 || sizeof(ggml_cuda_thread_mask_t) == 8, "unsupported thread mask type");
|
||||
if constexpr (sizeof(ggml_cuda_thread_mask_t) == 4) {
|
||||
return __popc(x);
|
||||
} else {
|
||||
return __popcll(x);
|
||||
}
|
||||
}
|
||||
template<typename T> static __device__ __forceinline__ int ggml_cuda_popc(T x) = delete; // Makes implicit type casts result in an error.
|
||||
|
||||
static __device__ __forceinline__ int ggml_cuda_all(int predicate, ggml_cuda_thread_mask_t mask = GGML_CUDA_FULL_THREAD_MASK) {
|
||||
return __all_sync(mask, predicate);
|
||||
}
|
||||
@@ -1293,6 +1303,7 @@ struct ggml_cuda_device_info {
|
||||
size_t total_vram;
|
||||
int warp_size; // Number of threads in a dispatch
|
||||
bool supports_cooperative_launch; // whether cooperative launch is supported
|
||||
bool prefer_fp32_acc; // prefer BF16 inputs with FP32 accumulation for quantized cuBLAS
|
||||
int physical_device; // backing physical CUDA device for this (virtual) device
|
||||
int physical_share_count; // number of (virtual) devices sharing this device's physical GPU
|
||||
int virtual_index; // index of this (virtual) device among those sharing its physical GPU
|
||||
|
||||
@@ -1206,7 +1206,7 @@ void launch_fattn(
|
||||
dim3 blocks_num;
|
||||
if (stream_k) {
|
||||
// Stream-K splits the work before the mask scan is applied, so skipped KV tiles make the blocks uneven.
|
||||
const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK && async_kv_preload && scan_mask;
|
||||
const bool prefer_whole_tiles = GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10 && async_kv_preload && scan_mask;
|
||||
|
||||
auto should_use_stream_k = [prefer_whole_tiles](const int cc, const int ntiles_dst, const int max_blocks, const int DKQ) {
|
||||
const int tiles_nwaves = (ntiles_dst + max_blocks - 1) / max_blocks;
|
||||
|
||||
@@ -17,15 +17,6 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
constexpr int values_per_lane = 8;
|
||||
using lane_mask_t = std::conditional_t<warp_size == 64, uint64_t, uint32_t>;
|
||||
constexpr uint64_t ballot_mask = ~(lane_mask_t)0;
|
||||
const auto popc = [](const lane_mask_t mask) {
|
||||
if constexpr (warp_size == 64) {
|
||||
return __popcll(mask);
|
||||
} else {
|
||||
return __popc(mask);
|
||||
}
|
||||
};
|
||||
const int tid = threadIdx.x;
|
||||
const int warp = tid / warp_size;
|
||||
const int lane = tid % warp_size;
|
||||
@@ -48,7 +39,7 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
__syncthreads();
|
||||
|
||||
for (int i0 = 0; i0 < ne30; i0 += blockDim.x*values_per_lane) {
|
||||
lane_mask_t selected_warp[values_per_lane];
|
||||
ggml_cuda_thread_mask_t selected_warp[values_per_lane];
|
||||
int warp_count = 0;
|
||||
#pragma unroll
|
||||
for (int item = 0; item < values_per_lane; ++item) {
|
||||
@@ -60,8 +51,8 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
selected |= (!oob || q < q1 - q0) && isfinite(__half2float(mask[q*s31 + i]));
|
||||
}
|
||||
}
|
||||
selected_warp[item] = (lane_mask_t)ggml_cuda_ballot((int) selected, ballot_mask);
|
||||
warp_count += popc(selected_warp[item]);
|
||||
selected_warp[item] = ggml_cuda_ballot((int) selected);
|
||||
warp_count += ggml_cuda_popc(selected_warp[item]);
|
||||
}
|
||||
|
||||
if (lane == 0) {
|
||||
@@ -81,16 +72,16 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const lane_mask_t lane_mask = lane == 0 ? 0 : ((lane_mask_t)1 << lane) - 1;
|
||||
const ggml_cuda_thread_mask_t lane_mask = lane == 0 ? 0 : ((ggml_cuda_thread_mask_t)1 << lane) - 1;
|
||||
int warp_item_offset = 0;
|
||||
#pragma unroll
|
||||
for (int item = 0; item < values_per_lane; ++item) {
|
||||
const int i = i0 + (warp*values_per_lane + item)*warp_size + lane;
|
||||
const int dst = row_count + warp_offsets[warp] + warp_item_offset + popc(selected_warp[item] & lane_mask);
|
||||
if ((selected_warp[item] & ((lane_mask_t)1 << lane)) && dst < n_kv_max) {
|
||||
const int dst = row_count + warp_offsets[warp] + warp_item_offset + ggml_cuda_popc(selected_warp[item] & lane_mask);
|
||||
if ((selected_warp[item] & ((ggml_cuda_thread_mask_t)1 << lane)) && dst < n_kv_max) {
|
||||
indices[dst] = i;
|
||||
}
|
||||
warp_item_offset += popc(selected_warp[item]);
|
||||
warp_item_offset += ggml_cuda_popc(selected_warp[item]);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
@@ -397,7 +388,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
|
||||
if (amd_mfma_available(cc) && ggml_cuda_flash_attn_ext_mma_f16_shall_use_sparse(cc, dst, 1, 16)) {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
|
||||
} else if (gqa_ratio == 20 && GGML_CUDA_CC_IS_NVIDIA(cc)) { // GLM 4.7 Flash
|
||||
if (cc >= GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (cc >= GGML_CUDA_CC_GB10) {
|
||||
if (Q->ne[1] <= 8) {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 16>(ctx, dst);
|
||||
break;
|
||||
|
||||
@@ -468,7 +468,7 @@ static gdn_path select_gdn_mma_path(int device, const ggml_cuda_gdn_mma_args & a
|
||||
// Enable only architectures benchmarked for this schedule. MMA support alone does not imply a speedup.
|
||||
if (!ampere_mma_available(info.cc) ||
|
||||
(info.cc != GGML_CUDA_CC_AMPERE_GA10X && info.cc != GGML_CUDA_CC_ADA_LOVELACE &&
|
||||
info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_DGX_SPARK)) {
|
||||
info.cc != GGML_CUDA_CC_BLACKWELL && info.cc != GGML_CUDA_CC_GB10)) {
|
||||
return gdn_path::ar;
|
||||
}
|
||||
// Value splitting pays off when whole heads leave at least half the SMs idle.
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
#include "ggml-cuda/arange.cuh"
|
||||
#include "ggml-cuda/argmax.cuh"
|
||||
#include "ggml-cuda/argsort.cuh"
|
||||
#include "ggml-cuda/batch-get.cuh"
|
||||
#include "ggml-cuda/batch-copy.cuh"
|
||||
#include "ggml-cuda/binbcast.cuh"
|
||||
#include "ggml-cuda/clamp.cuh"
|
||||
#include "ggml-cuda/col2im-1d.cuh"
|
||||
@@ -48,6 +48,7 @@
|
||||
#include "ggml-cuda/rope.cuh"
|
||||
#include "ggml-cuda/roll.cuh"
|
||||
#include "ggml-cuda/scale.cuh"
|
||||
#include "ggml-cuda/sleep.cuh"
|
||||
#include "ggml-cuda/snake.cuh"
|
||||
#include "ggml-cuda/softcap.cuh"
|
||||
#include "ggml-cuda/softmax.cuh"
|
||||
@@ -96,6 +97,11 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
#include "ggml-backend-dl.h"
|
||||
#include <nvml.h>
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
static_assert(sizeof(half) == sizeof(ggml_fp16_t), "wrong fp16 size");
|
||||
|
||||
#define GGML_LOG_WARN_ONCE(str) \
|
||||
@@ -309,6 +315,64 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
||||
}
|
||||
total_vram = 0;
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
#if defined(_WIN32)
|
||||
// NVML is installed by the driver; its NVSMI directory is not added to PATH on Windows.
|
||||
dl_handle_ptr library(LoadLibraryExW(L"nvml.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32));
|
||||
if (!library) {
|
||||
const DWORD size = GetEnvironmentVariableW(L"ProgramW6432", nullptr, 0);
|
||||
if (size > 0) {
|
||||
std::wstring path(size, L'\0');
|
||||
const DWORD length = GetEnvironmentVariableW(L"ProgramW6432", &path[0], size);
|
||||
if (length > 0 && length < size) {
|
||||
path.resize(length);
|
||||
path += L"\\NVIDIA Corporation\\NVSMI\\nvml.dll";
|
||||
if (fs::path(path).is_absolute()) {
|
||||
library.reset(LoadLibraryExW(path.c_str(), nullptr, LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
dl_handle_ptr library(dlopen("libnvidia-ml.so.1", RTLD_NOW | RTLD_LOCAL));
|
||||
#endif // defined(_WIN32)
|
||||
const auto get_symbol = [&](const char * name) {
|
||||
#if defined(_WIN32)
|
||||
return library ? GetProcAddress(library.get(), name) : nullptr;
|
||||
#else
|
||||
return library ? dlsym(library.get(), name) : nullptr;
|
||||
#endif // defined(_WIN32)
|
||||
};
|
||||
const auto init = reinterpret_cast<decltype(&nvmlInit_v2)>(get_symbol("nvmlInit_v2"));
|
||||
const auto shutdown = reinterpret_cast<decltype(&nvmlShutdown)>(get_symbol("nvmlShutdown"));
|
||||
const auto get_device = reinterpret_cast<decltype(&nvmlDeviceGetHandleByPciBusId_v2)>(get_symbol("nvmlDeviceGetHandleByPciBusId_v2"));
|
||||
const auto get_brand = reinterpret_cast<decltype(&nvmlDeviceGetBrand)>(get_symbol("nvmlDeviceGetBrand"));
|
||||
const bool nvml_initialized = init && shutdown && get_device && get_brand && init() == NVML_SUCCESS;
|
||||
|
||||
const auto should_use_fp32_acc = [&](const cudaDeviceProp & prop, const int cc) {
|
||||
if (cc == GGML_CUDA_CC_GB10) {
|
||||
return true;
|
||||
}
|
||||
if (!fast_bf16_hardware_available(cc) || !nvml_initialized) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// CUDA and NVML device indices can differ, e.g. with CUDA_VISIBLE_DEVICES.
|
||||
char pci_bus_id[32];
|
||||
snprintf(pci_bus_id, sizeof(pci_bus_id), "%04x:%02x:%02x.0",
|
||||
(unsigned int) prop.pciDomainID, (unsigned int) prop.pciBusID, (unsigned int) prop.pciDeviceID);
|
||||
nvmlDevice_t device = nullptr;
|
||||
nvmlBrandType_t brand = NVML_BRAND_UNKNOWN;
|
||||
bool prefer_fp32_acc = false;
|
||||
if (get_device(pci_bus_id, &device) == NVML_SUCCESS && get_brand(device, &brand) == NVML_SUCCESS) {
|
||||
prefer_fp32_acc = brand == NVML_BRAND_QUADRO ||
|
||||
brand == NVML_BRAND_QUADRO_RTX ||
|
||||
brand == NVML_BRAND_NVIDIA_RTX;
|
||||
}
|
||||
return prefer_fp32_acc;
|
||||
};
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
std::vector<std::pair<int, std::string>> turing_devices_without_mma;
|
||||
for (int id = 0; id < info.device_count; ++id) {
|
||||
const int physical_id = info.devices[id].physical_device;
|
||||
@@ -379,6 +443,11 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
||||
#else
|
||||
info.devices[id].smpbo = prop.sharedMemPerBlockOptin;
|
||||
info.devices[id].cc = 100*prop.major + 10*prop.minor;
|
||||
if (info.devices[id].virtual_index == 0) {
|
||||
info.devices[id].prefer_fp32_acc = should_use_fp32_acc(prop, info.devices[id].cc);
|
||||
} else {
|
||||
info.devices[id].prefer_fp32_acc = info.devices[physical_id].prefer_fp32_acc;
|
||||
}
|
||||
GGML_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s, VRAM: %zu MiB\n",
|
||||
id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no",
|
||||
device_vram_mib);
|
||||
@@ -403,6 +472,13 @@ static ggml_cuda_device_info ggml_cuda_init() {
|
||||
#endif // defined(GGML_USE_HIP)
|
||||
}
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
if (nvml_initialized) {
|
||||
(void) shutdown();
|
||||
}
|
||||
library.reset();
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
if (ggml_cuda_highest_compiled_arch(GGML_CUDA_CC_TURING) >= GGML_CUDA_CC_TURING && !turing_devices_without_mma.empty()) {
|
||||
GGML_LOG_INFO("The following devices will have suboptimal performance due to a lack of tensor cores:\n");
|
||||
for (size_t device_pos = 0; device_pos < turing_devices_without_mma.size(); device_pos++) {
|
||||
@@ -1041,6 +1117,11 @@ static void ggml_backend_cuda_buffer_set_tensor_n(ggml_backend_buffer_t buffer,
|
||||
|
||||
ggml_cuda_set_device(ctx->device);
|
||||
|
||||
if (!ggml_cuda_pinned_memory_disabled() &&
|
||||
ggml_cuda_set_tensor_n(ctx->device, tensors, data, offsets, sizes, n_tensors)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
@@ -1641,7 +1722,7 @@ static void * ggml_backend_cuda_comm_init_rank(ggml_backend_t backend, int rank,
|
||||
// NCCL_MAX_CTAS still overrides this.
|
||||
ncclConfig_t config = NCCL_CONFIG_INITIALIZER;
|
||||
const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc;
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10) {
|
||||
config.maxCTAs = 8;
|
||||
}
|
||||
|
||||
@@ -2047,6 +2128,9 @@ static void ggml_cuda_mul_mat_cublas(ggml_backend_cuda_context & ctx, const ggml
|
||||
ggml_type compute_type = src0->type;
|
||||
if (ggml_is_quantized(compute_type) || compute_type == GGML_TYPE_F8_E4M3) {
|
||||
compute_type = fast_fp16_hardware_available(cc) ? GGML_TYPE_F16 : GGML_TYPE_F32;
|
||||
if (ggml_cuda_info().devices[ctx.device].prefer_fp32_acc) {
|
||||
compute_type = GGML_TYPE_BF16;
|
||||
}
|
||||
} else if (compute_type == GGML_TYPE_F16 && !fast_fp16_hardware_available(cc)) {
|
||||
compute_type = GGML_TYPE_F32;
|
||||
} else if (compute_type == GGML_TYPE_BF16 && !fast_bf16_hardware_available(cc)) {
|
||||
@@ -2646,7 +2730,8 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
|
||||
ids_from_sorted_host[i12*n_expert_used + iex] = ids_to_sorted_host.size();
|
||||
ids_to_sorted_host.push_back(i12*ne11 + iex % ne11);
|
||||
tokens_per_expert[i02]++;
|
||||
break;
|
||||
// TODO: removing the break here was done to make duplicate IDs work but this caused a ~10% performance regression, better solution needed.
|
||||
// break;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3067,6 +3152,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
|
||||
case GGML_OP_FILL:
|
||||
ggml_cuda_op_fill(ctx, dst);
|
||||
break;
|
||||
case GGML_OP_SLEEP:
|
||||
ggml_cuda_op_sleep(ctx, dst);
|
||||
break;
|
||||
case GGML_OP_LIGHTNING_INDEXER:
|
||||
ggml_cuda_lightning_indexer(ctx, dst);
|
||||
break;
|
||||
@@ -3417,7 +3505,12 @@ static bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope,
|
||||
return false;
|
||||
}
|
||||
|
||||
if (set_rows->type != GGML_TYPE_F32 && set_rows->type != GGML_TYPE_F16) {
|
||||
if (set_rows->type != GGML_TYPE_F32 && set_rows->type != GGML_TYPE_F16 && set_rows->type != GGML_TYPE_BF16) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// BF16 rope fuses into an F16 or BF16 cache, other ropes not into a BF16 cache
|
||||
if (rope->type == GGML_TYPE_BF16 ? set_rows->type == GGML_TYPE_F32 : set_rows->type == GGML_TYPE_BF16) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -6423,7 +6516,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || op->type == GGML_TYPE_F8_E4M3 ||
|
||||
op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 ||
|
||||
op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) &&
|
||||
op->src[0]->type == GGML_TYPE_F32
|
||||
(op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16)
|
||||
) || (
|
||||
op->type == GGML_TYPE_F16 && op->src[0]->type == GGML_TYPE_F16
|
||||
)
|
||||
@@ -6644,7 +6737,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
return false;
|
||||
case GGML_OP_ROPE:
|
||||
case GGML_OP_ROPE_BACK: {
|
||||
return op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && ggml_is_contiguous_2(op->src[0]);
|
||||
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) &&
|
||||
op->type == op->src[0]->type &&
|
||||
op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && ggml_is_contiguous_2(op->src[0]);
|
||||
}
|
||||
case GGML_OP_IM2COL:
|
||||
case GGML_OP_IM2COL_3D:
|
||||
@@ -6731,6 +6826,12 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
return ggml_cuda_lightning_indexer_supported(dev_ctx->device, op);
|
||||
case GGML_OP_TOP_K_MASK:
|
||||
return op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32;
|
||||
case GGML_OP_SLEEP:
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
return false;
|
||||
#else
|
||||
return true;
|
||||
#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -43,7 +43,6 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
const int64_t s13 = src1->nb[3] / ts_src1;
|
||||
const int64_t s3 = dst->nb[3] / ts_dst;
|
||||
|
||||
const int64_t ids_s0 = ids ? ids->nb[0] / ggml_type_size(ids->type) : 0;
|
||||
const int64_t ids_s1 = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;
|
||||
|
||||
mmf_ids_data ids_info{};
|
||||
@@ -70,7 +69,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
nchannels_y = ids->ne[0];
|
||||
}
|
||||
|
||||
if (ids && ncols_dst > 16) {
|
||||
if (ids) {
|
||||
const int64_t n_expert_used = ids->ne[0];
|
||||
const int64_t n_experts = ne02;
|
||||
const int64_t n_tokens = ne12;
|
||||
@@ -110,7 +109,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
constexpr int vals_per_T = 1;
|
||||
mul_mat_f_switch_rows_per_block<float>(
|
||||
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
|
||||
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
|
||||
} break;
|
||||
case GGML_TYPE_F16: {
|
||||
@@ -118,7 +117,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
constexpr int vals_per_T = 2;
|
||||
mul_mat_f_switch_rows_per_block<half2>(
|
||||
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
|
||||
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
|
||||
} break;
|
||||
case GGML_TYPE_BF16: {
|
||||
@@ -126,7 +125,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
constexpr int vals_per_T = 2;
|
||||
mul_mat_f_switch_rows_per_block<nv_bfloat162>(
|
||||
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
|
||||
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
|
||||
} break;
|
||||
default:
|
||||
|
||||
+69
-176
@@ -47,12 +47,11 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
|
||||
bool ggml_cuda_should_use_mmf(enum ggml_type type, int cc, int warp_size, const int64_t * scr0_ne, const size_t * src0_nb, const int src1_ncols, bool mul_mat_id);
|
||||
|
||||
template <typename T, int rows_per_block, int cols_per_block, int nwarps, bool has_ids>
|
||||
template <typename T, int rows_per_block, int cols_per_block, int nwarps>
|
||||
__launch_bounds__(ggml_cuda_get_physical_warp_size()*nwarps, 1)
|
||||
static __global__ void mul_mat_f(
|
||||
const T * __restrict__ x, const float * __restrict__ y, const int32_t * __restrict__ ids, float * __restrict__ dst,
|
||||
const int ncols, const int ncols_dst_total, const int nchannels_dst, const int stride_row, const int stride_col_y, const int stride_col_dst,
|
||||
const int stride_col_id, const int stride_row_id,
|
||||
const T * __restrict__ x, const float * __restrict__ y, float * __restrict__ dst,
|
||||
const int ncols, const int stride_row, const int stride_col_y, const int stride_col_dst,
|
||||
const int channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
|
||||
const int sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst) {
|
||||
// TODO: handle this in a consistent and simpler way after AMD MFMA support has been added
|
||||
@@ -90,85 +89,26 @@ static __global__ void mul_mat_f(
|
||||
constexpr int ntA = rows_per_block / tile_A::I;
|
||||
constexpr int ntB = (cols_per_block + tile_B::I - 1) / tile_B::I;
|
||||
|
||||
const int row0 = blockIdx.x * rows_per_block;
|
||||
const int row0 = blockIdx.x * rows_per_block;
|
||||
|
||||
int expert_idx = 0;
|
||||
[[maybe_unused]] int col_base = 0;
|
||||
|
||||
const int channel_dst = has_ids ? 0 : blockIdx.y;
|
||||
|
||||
if constexpr (has_ids) {
|
||||
// experts + tiles of ncols_dst are packed in the y dimension
|
||||
int col_tiles = (ncols_dst_total + cols_per_block - 1) / cols_per_block;
|
||||
const int nchannels_x = gridDim.y / col_tiles;
|
||||
const int tile_idx = blockIdx.y / nchannels_x;
|
||||
expert_idx = blockIdx.y - tile_idx * nchannels_x;
|
||||
col_base = tile_idx * cols_per_block;
|
||||
}
|
||||
|
||||
const int channel_x = has_ids ? expert_idx : (channel_dst / channel_ratio);
|
||||
const int channel_dst = blockIdx.y;
|
||||
const int channel_x = channel_dst / channel_ratio;
|
||||
const int channel_y = channel_dst;
|
||||
const int sample_dst = blockIdx.z;
|
||||
const int sample_x = sample_dst / sample_ratio;
|
||||
const int sample_y = sample_dst;
|
||||
|
||||
x += int64_t(sample_x) *stride_sample_x + channel_x *stride_channel_x + row0*stride_row ;
|
||||
y += int64_t(sample_y) *stride_sample_y + (has_ids ? 0 : channel_y *stride_channel_y);
|
||||
dst += int64_t(sample_dst)*stride_sample_dst + (has_ids ? 0 : channel_dst*stride_channel_dst);
|
||||
|
||||
if constexpr (has_ids) {
|
||||
constexpr int y_stride_scale = std::is_same_v<T, float> ? 1 : 2;
|
||||
const int64_t col_offset = col_base;
|
||||
y += col_offset * stride_col_y * y_stride_scale;
|
||||
dst += col_offset * stride_col_dst;
|
||||
ids += col_offset * stride_row_id;
|
||||
}
|
||||
y += int64_t(sample_y) *stride_sample_y + channel_y *stride_channel_y;
|
||||
dst += int64_t(sample_dst)*stride_sample_dst + channel_dst*stride_channel_dst;
|
||||
|
||||
[[maybe_unused]] const float2 * y2 = (const float2 *) y;
|
||||
|
||||
extern __shared__ char data_mmv[];
|
||||
|
||||
char * shmem_base = data_mmv;
|
||||
[[maybe_unused]] int * slot_map = (int *) shmem_base;
|
||||
char * compute_base = has_ids ? (shmem_base + GGML_PAD(cols_per_block, 16) * sizeof(int)) : shmem_base;
|
||||
|
||||
tile_C C[ntA][ntB];
|
||||
|
||||
T * tile_xy = (T *) compute_base + threadIdx.y*(tile_A::I * tile_k_padded);
|
||||
|
||||
if constexpr (has_ids) {
|
||||
int found = 0;
|
||||
|
||||
for (int j0 = 0; j0 < cols_per_block; j0 += nwarps) {
|
||||
const int j = j0 + threadIdx.y;
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
slot_map[j] = -1;
|
||||
}
|
||||
ggml_cuda_syncwarp();
|
||||
|
||||
if (col_base + j >= ncols_dst_total) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const int32_t * __restrict__ id_row = ids + j*stride_row_id;
|
||||
|
||||
for (int k = threadIdx.x; k < nchannels_dst; k += warp_size) {
|
||||
int match = id_row[k*stride_col_id] == expert_idx;
|
||||
|
||||
if (match) {
|
||||
slot_map[j] = k;
|
||||
found = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!__syncthreads_or(found)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
T * tile_xy = (T *) data_mmv + threadIdx.y*(tile_A::I * tile_k_padded);
|
||||
|
||||
for (int col = threadIdx.y*warp_size + threadIdx.x; col < ncols; col += nwarps*warp_size) {
|
||||
tile_A A[ntA][warp_size / tile_A::J];
|
||||
@@ -194,26 +134,15 @@ static __global__ void mul_mat_f(
|
||||
for (int j0 = 0; j0 < tile_B::I; ++j0) {
|
||||
const int j = j0 + itB*tile_B::I;
|
||||
|
||||
if constexpr (!has_ids) {
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = j < cols_per_block ? y[j*stride_col_y + col] : 0.0f;
|
||||
} else {
|
||||
const bool valid = j < cols_per_block && (col_base + j) < ncols_dst_total && slot_map[j] >= 0;
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = valid ? y[slot_map[j]*stride_channel_y + j*stride_col_y + col] : 0.0f;
|
||||
}
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = j < cols_per_block ? y[j*stride_col_y + col] : 0.0f;
|
||||
}
|
||||
} else if constexpr (std::is_same_v<T, half2> || std::is_same_v<T, nv_bfloat162>) {
|
||||
#pragma unroll
|
||||
for (int j0 = 0; j0 < tile_B::I; ++j0) {
|
||||
const int j = j0 + itB*tile_B::I;
|
||||
|
||||
if constexpr (!has_ids) {
|
||||
const float2 tmp = j < cols_per_block ? y2[j*stride_col_y + col] : make_float2(0.0f, 0.0f);
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
|
||||
} else {
|
||||
const bool valid = j < cols_per_block && (col_base + j) < ncols_dst_total && slot_map[j] >= 0;
|
||||
float2 tmp = valid ? *(const float2*) &y[slot_map[j]*stride_channel_y + 2*(j*stride_col_y + col)] : make_float2(0.0f, 0.0f);
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
|
||||
}
|
||||
const float2 tmp = j < cols_per_block ? y2[j*stride_col_y + col] : make_float2(0.0f, 0.0f);
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
|
||||
}
|
||||
} else {
|
||||
static_assert(std::is_same_v<T, void>, "unsupported type");
|
||||
@@ -231,7 +160,7 @@ static __global__ void mul_mat_f(
|
||||
}
|
||||
}
|
||||
|
||||
float * buf_iw = (float *) compute_base;
|
||||
float * buf_iw = (float *) data_mmv;
|
||||
constexpr int kiw = nwarps*rows_per_block + mmf_get_padding();
|
||||
|
||||
if (nwarps > 1) {
|
||||
@@ -278,26 +207,15 @@ static __global__ void mul_mat_f(
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (!has_ids) {
|
||||
#pragma unroll
|
||||
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
|
||||
dst[j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
|
||||
}
|
||||
} else {
|
||||
const int slot = (j < cols_per_block) ? slot_map[j] : -1;
|
||||
if (slot >= 0 && (col_base + j) < ncols_dst_total) {
|
||||
#pragma unroll
|
||||
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
|
||||
dst[slot*stride_channel_dst + j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
|
||||
}
|
||||
}
|
||||
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
|
||||
dst[j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
GGML_UNUSED_VARS(x, y, ids, dst,
|
||||
ncols, ncols_dst_total, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id,
|
||||
GGML_UNUSED_VARS(x, y, dst,
|
||||
ncols, stride_row, stride_col_y, stride_col_dst,
|
||||
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
|
||||
NO_DEVICE_CODE;
|
||||
@@ -588,22 +506,21 @@ static __global__ void mul_mat_f_ids(
|
||||
|
||||
template<typename T, int rows_per_block, int cols_per_block, int nwarps>
|
||||
static inline void mul_mat_f_switch_ids(
|
||||
const T * x, const float * y, const int32_t * ids, float * dst,
|
||||
const T * x, const float * y, float * dst,
|
||||
const int64_t ncols_x, const int64_t ncols_dst, const int64_t nchannels_dst,
|
||||
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
|
||||
const int64_t stride_col_id, const int64_t stride_row_id,
|
||||
const int64_t channel_ratio, const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst,
|
||||
const int64_t sample_ratio, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
|
||||
const dim3 & block_nums, const dim3 & block_dims, const int nbytes_shared_total, cudaStream_t stream,
|
||||
const mmf_ids_data * ids_data) {
|
||||
|
||||
const bool has_ids_data = ids_data && ids_data->ids_src_compact;
|
||||
|
||||
// Use the compact-ids kernel only for larger tiles; for small ncols_dst (< 16)
|
||||
// we prefer the normal mul_mat_f path with has_ids=true.
|
||||
if (has_ids_data && ncols_dst > 16) {
|
||||
if (has_ids_data) {
|
||||
if (ids_data->max_tiles == 0) {
|
||||
return;
|
||||
}
|
||||
GGML_ASSERT(block_nums.z == 1);
|
||||
dim3 block_nums_ids(block_nums.x, ids_data->max_tiles, 1);
|
||||
|
||||
const uint3 sis1_fd = ids_data->sis1 > 0 ? init_fastdiv_values((uint32_t) ids_data->sis1) : make_uint3(0, 0, 1);
|
||||
@@ -615,19 +532,10 @@ static inline void mul_mat_f_switch_ids(
|
||||
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
sis1_fd, nch_fd);
|
||||
} else if (ids) {
|
||||
const int64_t col_tiles = (ncols_dst + cols_per_block - 1) / cols_per_block;
|
||||
dim3 block_nums_ids = block_nums;
|
||||
block_nums_ids.y *= col_tiles;
|
||||
|
||||
mul_mat_f<T, rows_per_block, cols_per_block, nwarps, true><<<block_nums_ids, block_dims, nbytes_shared_total, stream>>>
|
||||
(x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
|
||||
} else {
|
||||
mul_mat_f<T, rows_per_block, cols_per_block, nwarps, false><<<block_nums, block_dims, nbytes_shared_total, stream>>>
|
||||
(x, y, ids, dst, ncols_x, cols_per_block, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
mul_mat_f<T, rows_per_block, cols_per_block, nwarps><<<block_nums, block_dims, nbytes_shared_total, stream>>>
|
||||
(x, y, dst, ncols_x, stride_row, stride_col_y, stride_col_dst,
|
||||
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
|
||||
}
|
||||
}
|
||||
@@ -637,7 +545,6 @@ void mul_mat_f_cuda(
|
||||
const T * x, const float * y, const int32_t * ids, float * dst,
|
||||
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
|
||||
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
|
||||
const int64_t stride_col_id, const int64_t stride_row_id,
|
||||
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
|
||||
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
|
||||
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
|
||||
@@ -673,70 +580,59 @@ void mul_mat_f_cuda(
|
||||
const int nbytes_shared_iter = nwarps_best * (volta_mma_available(cc) ? tile_A_32::I : tile_A_16::I) * (warp_size + mmf_get_padding(cc)) * 4;
|
||||
const int nbytes_cols_per_block_pad = (amd_wmma_available(cc) || amd_mfma_available(cc)) ? tile_B_16::I : tile_B_8::I;
|
||||
const int nbytes_shared_combine = GGML_PAD(cols_per_block, nbytes_cols_per_block_pad) * (nwarps_best*rows_per_block + mmf_get_padding(cc)) * 4;
|
||||
const int nbytes_shared = std::max(nbytes_shared_iter, nbytes_shared_combine);
|
||||
const int nbytes_slotmap = ids ? GGML_PAD(cols_per_block, 16) * sizeof(int) : 0;
|
||||
const int nbytes_shared_total = nbytes_shared + nbytes_slotmap;
|
||||
const int64_t grid_y = ids ? nchannels_x : nchannels_dst;
|
||||
const int nbytes_shared_total = std::max(nbytes_shared_iter, nbytes_shared_combine);
|
||||
|
||||
const dim3 block_nums(nrows_x/rows_per_block, grid_y, nsamples_dst);
|
||||
const dim3 block_nums(nrows_x/rows_per_block, nchannels_dst, nsamples_dst);
|
||||
const dim3 block_dims(warp_size, nwarps_best, 1);
|
||||
|
||||
switch (nwarps_best) {
|
||||
case 1: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 1>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 2: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 2>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 3: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 3>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 4: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 4>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 5: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 5>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 6: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 6>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 7: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 7>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 8: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 8>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
default:
|
||||
GGML_ABORT("fatal error");
|
||||
@@ -750,95 +646,94 @@ static void mul_mat_f_switch_cols_per_block(
|
||||
const T * x, const float * y, const int32_t * ids, float * dst,
|
||||
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
|
||||
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
|
||||
const int64_t stride_col_id, const int stride_row_id,
|
||||
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
|
||||
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
|
||||
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
|
||||
cudaStream_t stream, const mmf_ids_data * ids_data) {
|
||||
|
||||
const int ncols_case = (ids && ncols_dst > 16) ? 16 : ncols_dst;
|
||||
const int ncols_case = ids ? 16 : ncols_dst;
|
||||
|
||||
GGML_ASSERT(ids || ncols_dst <= 16);
|
||||
|
||||
switch (ncols_case) {
|
||||
case 1: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 1>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 2: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 2>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 3: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 3>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 4: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 4>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 5: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 5>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 6: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 6>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 7: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 7>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 8: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 8>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 9: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 9>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 10: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 10>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 11: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 11>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 12: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 12>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 13: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 13>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 14: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 14>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 15: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 15>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 16: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 16>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
default:
|
||||
@@ -851,7 +746,6 @@ static void mul_mat_f_switch_rows_per_block(
|
||||
const int rows_per_block, const T * x, const float * y, const int32_t * ids, float * dst,
|
||||
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
|
||||
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
|
||||
const int64_t stride_col_id, const int stride_row_id,
|
||||
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
|
||||
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
|
||||
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
|
||||
@@ -860,13 +754,13 @@ static void mul_mat_f_switch_rows_per_block(
|
||||
case MMF_ROWS_PER_BLOCK: {
|
||||
mul_mat_f_switch_cols_per_block<T, MMF_ROWS_PER_BLOCK>(
|
||||
x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case MMF_ROWS_PER_BLOCK_CDNA: {
|
||||
mul_mat_f_switch_cols_per_block<T, MMF_ROWS_PER_BLOCK_CDNA>(
|
||||
x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
default:
|
||||
@@ -878,7 +772,6 @@ static void mul_mat_f_switch_rows_per_block(
|
||||
template void mul_mat_f_cuda<T, nrows_dst, ncols_dst>( \
|
||||
const T * x, const float * y, const int32_t * ids, float * dst, \
|
||||
const int64_t ncols_x, const int64_t nrows_x, int64_t ncols_dst_total, const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst, \
|
||||
const int64_t stride_col_id, const int64_t stride_row_id, \
|
||||
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst, \
|
||||
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,\
|
||||
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst, \
|
||||
|
||||
+159
-60
@@ -24,6 +24,50 @@ template <int n> struct mm_ids_pow2 { static constexpr int value = 2*mm_ids_pow2
|
||||
template <> struct mm_ids_pow2<1> { static constexpr int value = 1; };
|
||||
template <> struct mm_ids_pow2<0> { static constexpr int value = 1; };
|
||||
|
||||
template <bool increment_nex_prev, bool write_to_store>
|
||||
static __device__ __forceinline__ void check_experts(
|
||||
mm_ids_helper_store * store, int & nex_prev, int & it_compact,
|
||||
const int expert, const int it, const int iex, const int expert_found) {
|
||||
if (increment_nex_prev) {
|
||||
nex_prev += expert_found < expert;
|
||||
}
|
||||
|
||||
if (write_to_store) {
|
||||
const ggml_cuda_thread_mask_t iex_used = ggml_cuda_ballot(expert_found == expert);
|
||||
const ggml_cuda_thread_mask_t mask = (ggml_cuda_thread_mask_t(1) << threadIdx.x) - 1;
|
||||
const int it_compact_offset = ggml_cuda_popc(iex_used & mask);
|
||||
|
||||
if (expert_found == expert) {
|
||||
store[it_compact + it_compact_offset] = mm_ids_helper_store(it, iex);
|
||||
}
|
||||
|
||||
it_compact += ggml_cuda_popc(iex_used);
|
||||
}
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ void dump_store(
|
||||
int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst,
|
||||
const int n_expert_used, const int nchannels_y, const int sis1, const bool write_inverse,
|
||||
const mm_ids_helper_store * __restrict__ store, const int nex_prev, int & n_dumped, const int it_compact) {
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
|
||||
for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {
|
||||
const mm_ids_helper_store store_it = store[itc];
|
||||
const int it = store_it.it();
|
||||
const int iex_used = store_it.iex_used();
|
||||
ids_dst[nex_prev + n_dumped + itc] = it*n_expert_used + iex_used;
|
||||
// ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup
|
||||
if (write_inverse) {
|
||||
ids_src1[it*n_expert_used + iex_used] = nex_prev + n_dumped + itc;
|
||||
} else {
|
||||
ids_src1[nex_prev + n_dumped + itc] = it*sis1 + iex_used % nchannels_y;
|
||||
}
|
||||
}
|
||||
n_dumped += it_compact;
|
||||
}
|
||||
|
||||
constexpr int n_unroll = 4; // Manually unroll this many loop iterations for better performance.
|
||||
|
||||
// Helper function for mul_mat_id, converts ids to a more convenient format.
|
||||
// ids_src1 describes how to permute the flattened column indices of src1 in order to get a compact src1 tensor sorted by expert.
|
||||
// ids_dst describes the same mapping but for the dst tensor.
|
||||
@@ -36,85 +80,138 @@ static __global__ void mm_ids_helper(
|
||||
const int32_t * __restrict__ ids,
|
||||
int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec,
|
||||
const int n_tokens, const int n_expert_used_var, const int nchannels_y, const uint3 tile_width,
|
||||
const int si1, const int sis1, const bool write_inverse) {
|
||||
const int si1, const int sis1, const bool write_inverse, const int ne_store) {
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
const int n_expert_used = n_expert_used_template == 0 ? n_expert_used_var : n_expert_used_template;
|
||||
const int expert = blockIdx.x;
|
||||
|
||||
const int expert = blockIdx.x; // The expert this CUDA block handles.
|
||||
|
||||
extern __shared__ char data_mm_ids_helper[];
|
||||
mm_ids_helper_store * store = (mm_ids_helper_store *) data_mm_ids_helper;
|
||||
|
||||
int nex_prev = 0; // Number of columns for experts with a lower index.
|
||||
int n_dumped = 0; // Number of data values that have been dumped to output buffers.
|
||||
int it_compact = 0; // Running index for the compact slice of this expert.
|
||||
|
||||
// Step 1: Iterate over n_tokens, increment nex_prev to determine the output indices, fill up store.
|
||||
// Step 1.5: If the store does not have enough space to hold all values, iterate over the rest of the tokens just to increment nex_prev.
|
||||
// Step 2: Dump the stored values using nex_prev to get the correct output indices.
|
||||
// Step 3: If we were not able to process all values, resume at the index up to which we were able to dump the output and re-do.
|
||||
|
||||
if constexpr (n_expert_used_template == 0) {
|
||||
// Generic implementation:
|
||||
for (int it = 0; it < n_tokens; ++it) {
|
||||
int iex_used = -1; // The index at which the expert is used, if any.
|
||||
for (int iex = threadIdx.x; iex < n_expert_used; iex += warp_size) {
|
||||
const int expert_used = ids[it*si1 + iex];
|
||||
nex_prev += expert_used < expert;
|
||||
if (expert_used == expert) {
|
||||
iex_used = iex;
|
||||
int it = 0;
|
||||
for (; it < n_tokens && it_compact + max(warp_size, n_expert_used) <= ne_store; ++it) {
|
||||
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
|
||||
const int iex = iex0 + threadIdx.x;
|
||||
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = true;
|
||||
constexpr bool write_to_store = true;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
}
|
||||
}
|
||||
int it_dumped = it;
|
||||
for (; it < n_tokens; ++it) {
|
||||
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
|
||||
const int iex = iex0 + threadIdx.x;
|
||||
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = true;
|
||||
constexpr bool write_to_store = false;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
}
|
||||
}
|
||||
|
||||
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
it_compact = 0;
|
||||
|
||||
for (int it = it_dumped; it < n_tokens; ++it) {
|
||||
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
|
||||
const int iex = iex0 + threadIdx.x;
|
||||
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = false;
|
||||
constexpr bool write_to_store = true;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
if (it_compact + max(warp_size, n_expert_used) >= ne_store) {
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
it_compact = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (iex_used != -1) {
|
||||
store[it_compact] = mm_ids_helper_store(it, iex_used);
|
||||
}
|
||||
|
||||
if (warp_reduce_any<warp_size>(iex_used != -1)) {
|
||||
it_compact++;
|
||||
}
|
||||
}
|
||||
if (it_compact > 0) {
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
}
|
||||
} else {
|
||||
// Implementation optimized for specific numbers of experts used:
|
||||
// a warp holds a whole number of token slots, so the slot count is padded to a power of 2
|
||||
constexpr int neu_padded = mm_ids_pow2<n_expert_used_template>::value;
|
||||
static_assert(neu_padded <= warp_size && warp_size % neu_padded == 0, "bad n_expert_used");
|
||||
for (int it0 = 0; it0 < n_tokens; it0 += warp_size/neu_padded) {
|
||||
const int it = it0 + threadIdx.x / neu_padded;
|
||||
const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any.
|
||||
|
||||
const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any.
|
||||
const int expert_used = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
|
||||
ids[it*si1 + iex] : INT_MAX;
|
||||
const int iex_used = expert_used == expert ? iex : -1;
|
||||
nex_prev += expert_used < expert;
|
||||
|
||||
// Whether the threads at this token position have used the expert:
|
||||
const int it_compact_add_self = warp_reduce_any<neu_padded>(iex_used != -1);
|
||||
|
||||
// Do a scan over threads at lower token positions in warp to get the correct index for writing data:
|
||||
int it_compact_add_lower = 0;
|
||||
int it0 = 0;
|
||||
for (; it0 < n_tokens && it_compact + n_unroll*warp_size <= ne_store; it0 += n_unroll*warp_size/neu_padded) {
|
||||
#pragma unroll
|
||||
for (int offset = neu_padded; offset < warp_size; offset += neu_padded) {
|
||||
const int tmp = ggml_cuda_shfl_up(it_compact_add_self, offset, warp_size);
|
||||
if (threadIdx.x >= static_cast<unsigned int>(offset)) {
|
||||
it_compact_add_lower += tmp;
|
||||
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
|
||||
const int it = it0 + it1 + threadIdx.x / neu_padded;
|
||||
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
|
||||
ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = true;
|
||||
constexpr bool write_to_store = true;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
}
|
||||
}
|
||||
int it0_dumped = it0;
|
||||
for (; it0 < n_tokens; it0 += n_unroll*warp_size/neu_padded) {
|
||||
#pragma unroll
|
||||
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
|
||||
const int it = it0 + it1 + threadIdx.x / neu_padded;
|
||||
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
|
||||
ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = true;
|
||||
constexpr bool write_to_store = false;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
}
|
||||
}
|
||||
|
||||
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
it_compact = 0;
|
||||
|
||||
for (it0 = it0_dumped; it0 < n_tokens; it0 += n_unroll*warp_size/neu_padded) {
|
||||
#pragma unroll
|
||||
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
|
||||
const int it = it0 + it1 + threadIdx.x / neu_padded;
|
||||
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
|
||||
ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = false;
|
||||
constexpr bool write_to_store = true;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
if (it_compact + warp_size >= ne_store) {
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
it_compact = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (iex_used != -1) {
|
||||
store[it_compact + it_compact_add_lower] = mm_ids_helper_store(it, iex_used);
|
||||
}
|
||||
|
||||
// The thread with the highest index in the warp always has the sum over the whole warp, use it to increment all threads:
|
||||
it_compact += ggml_cuda_shfl(it_compact_add_lower + it_compact_add_self, warp_size - 1, warp_size);
|
||||
}
|
||||
}
|
||||
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
|
||||
ggml_cuda_syncwarp();
|
||||
|
||||
for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {
|
||||
const mm_ids_helper_store store_it = store[itc];
|
||||
const int it = store_it.it();
|
||||
const int iex_used = store_it.iex_used();
|
||||
ids_dst[nex_prev + itc] = it*n_expert_used + iex_used;
|
||||
// ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup
|
||||
if (write_inverse) {
|
||||
ids_src1[it*n_expert_used + iex_used] = nex_prev + itc;
|
||||
} else {
|
||||
ids_src1[nex_prev + itc] = it*sis1 + iex_used % nchannels_y;
|
||||
if (it_compact > 0) {
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,7 +220,7 @@ static __global__ void mm_ids_helper(
|
||||
}
|
||||
|
||||
int col_low = nex_prev;
|
||||
int col_high = nex_prev + min(int(tile_width.z), it_compact);
|
||||
int col_high = nex_prev + min(int(tile_width.z), n_dumped);
|
||||
|
||||
// The while loop covers all tiles at the seams containing the column with % tile_width == 0.
|
||||
while (col_high > fastdiv(col_low + tile_width.z - 1, tile_width) * tile_width.z) {
|
||||
@@ -131,7 +228,7 @@ static __global__ void mm_ids_helper(
|
||||
= {expert, col_low, col_high, /*padding =*/ 0};
|
||||
|
||||
col_low = col_high;
|
||||
col_high = min(col_high + int(tile_width.z), nex_prev + it_compact);
|
||||
col_high = min(col_high + int(tile_width.z), nex_prev + n_dumped);
|
||||
}
|
||||
|
||||
// Finally, at most one extra tile_spec between the seams per expert.
|
||||
@@ -157,10 +254,12 @@ static void launch_mm_ids_helper(
|
||||
|
||||
const dim3 num_blocks(n_experts, 1, 1);
|
||||
const dim3 block_size(warp_size, 1, 1);
|
||||
const size_t nbytes_shared = n_tokens*sizeof(mm_ids_helper_store);
|
||||
GGML_ASSERT(nbytes_shared <= smpbo);
|
||||
const size_t ne_shared_pad = n_expert_used_template == 0 ? std::max(n_expert_used_var, warp_size) : n_unroll*warp_size;
|
||||
const size_t nbytes_store_core = std::min(smpbo - ne_shared_pad * sizeof(mm_ids_helper_store), n_tokens * sizeof(mm_ids_helper_store));
|
||||
const size_t nbytes_shared = nbytes_store_core + ne_shared_pad * sizeof(mm_ids_helper_store);
|
||||
const size_t ne_store = nbytes_shared / sizeof(mm_ids_helper_store);
|
||||
mm_ids_helper<n_expert_used_template><<<num_blocks, block_size, nbytes_shared, stream>>>
|
||||
(ids, ids_src1, ids_dst, tile_spec, n_tokens, n_expert_used_var, nchannels_y, tile_width_fd, si1, sis1, write_inverse);
|
||||
(ids, ids_src1, ids_dst, tile_spec, n_tokens, n_expert_used_var, nchannels_y, tile_width_fd, si1, sis1, write_inverse, ne_store);
|
||||
}
|
||||
|
||||
void ggml_cuda_launch_mm_ids_helper(
|
||||
|
||||
@@ -584,7 +584,7 @@ static __global__ void mul_mat_q_few(
|
||||
ggml_cuda_pdl_sync();
|
||||
}
|
||||
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
// DGX Spark DRAM serves the short row pieces of the stage copies slowly, so the rows of the block are first prefetched into L2 in address order.
|
||||
{
|
||||
const int tid = threadIdx.y*warp_size + threadIdx.x;
|
||||
@@ -599,7 +599,7 @@ static __global__ void mul_mat_q_few(
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#endif // __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
|
||||
#pragma unroll
|
||||
for (int stage = 0; stage < nstages - 1; ++stage) {
|
||||
@@ -880,14 +880,14 @@ static int ggml_cuda_mmq_few_min_ncols(const ggml_type type, const int cc, const
|
||||
return 0;
|
||||
}
|
||||
// on DGX Spark the kernel needs its L2 prefetch, which is only in the DGX Spark device code
|
||||
if (cc == GGML_CUDA_CC_DGX_SPARK && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (cc == GGML_CUDA_CC_GB10 && ggml_cuda_highest_compiled_arch(cc) != GGML_CUDA_CC_GB10) {
|
||||
return 0;
|
||||
}
|
||||
// with fewer than 8 units of 256 values per row the warps idle, MMVQ and MMQ are better there (MUL_MAT_ID splits the rows instead)
|
||||
if (mmq_few_unit_size(type) == 0 || (!has_ids && ne00 < 8*QK_K)) {
|
||||
return 0;
|
||||
}
|
||||
if (cc == GGML_CUDA_CC_DGX_SPARK) { // tuned on DGX Spark GB10, also for MUL_MAT_ID
|
||||
if (cc == GGML_CUDA_CC_GB10) { // tuned on DGX Spark GB10, also for MUL_MAT_ID
|
||||
return type == GGML_TYPE_IQ4_XS || type == GGML_TYPE_IQ4_NL ? 8 : 2;
|
||||
}
|
||||
if (has_ids) { // after MMVQ, which is faster up to its max. batch size
|
||||
|
||||
@@ -1094,7 +1094,7 @@ static __global__ void mul_mat_q(
|
||||
int col_high = min(col_low + J, ncols_dst);
|
||||
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
||||
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
||||
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
||||
int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
|
||||
|
||||
if (ids_dst) {
|
||||
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
|
||||
@@ -1176,7 +1176,7 @@ static __global__ void mul_mat_q(
|
||||
int col_high = min(col_low + J, ncols_dst);
|
||||
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
||||
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
||||
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
||||
int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
|
||||
|
||||
if (ids_dst) {
|
||||
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
|
||||
@@ -1257,7 +1257,7 @@ static __global__ void mul_mat_q(
|
||||
int col_high = min(col_low + J, ncols_dst);
|
||||
int offset_y = wt*stride_sample_y + zt*stride_channel_y;
|
||||
int offset_y_scale = wt*nchannels_y.z*ncols_y + zt*ncols_y;
|
||||
int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst;
|
||||
int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst;
|
||||
|
||||
if (ids_dst) {
|
||||
const ggml_cuda_mm_id_tile_spec tsj = tile_spec[jt];
|
||||
@@ -1403,7 +1403,7 @@ static __global__ void mul_mat_q_stream_k_fixup(
|
||||
const int it = tmp2.x;
|
||||
|
||||
if (!ids_dst) {
|
||||
const int offset_dst = wt*stride_sample_dst + zt*stride_channel_dst + jt*J*stride_col_dst + it*I;
|
||||
const int64_t offset_dst = (int64_t) wt*stride_sample_dst + (int64_t) zt*stride_channel_dst + (int64_t) jt*J*stride_col_dst + it*I;
|
||||
dst += offset_dst;
|
||||
|
||||
const int i_max = nrows_x - it*I - 1;
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
// only enabled on DGX Spark, where it is a gain on every type below. On the higher-bandwidth parts the kernel
|
||||
// has little exposed latency left to hide and the extra requests cost more than they save.
|
||||
// For perf data, see https://github.com/ggml-org/llama.cpp/pull/26705#issuecomment-5569335031
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
// returns true only for those quants that benefit from prefetch and false otherwise
|
||||
static constexpr __host__ __device__ bool mmvq_should_prefetch(ggml_type type) {
|
||||
switch (type) {
|
||||
@@ -110,7 +110,7 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() {
|
||||
return MMVQ_PARAMETERS_GCN;
|
||||
#elif __CUDA_ARCH__ >= GGML_CUDA_CC_VOLTA && __CUDA_ARCH__ < GGML_CUDA_CC_AMPERE
|
||||
return MMVQ_PARAMETERS_TURING;
|
||||
#elif __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#elif __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
return MMVQ_PARAMETERS_GB10;
|
||||
#else
|
||||
return MMVQ_PARAMETERS_GENERIC;
|
||||
@@ -133,7 +133,7 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) >= GGML_CUDA_CC_VOLTA && ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_AMPERE) {
|
||||
return MMVQ_PARAMETERS_TURING;
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_GB10) {
|
||||
return MMVQ_PARAMETERS_GB10;
|
||||
}
|
||||
return MMVQ_PARAMETERS_GENERIC;
|
||||
@@ -342,7 +342,7 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
|
||||
return ne11 <= MMVQ_MAX_BATCH_SIZE;
|
||||
}
|
||||
}
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) {
|
||||
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_GB10) {
|
||||
switch (type) { // tuned on DGX Spark GB10
|
||||
case GGML_TYPE_Q2_K:
|
||||
return ne11 <= 6;
|
||||
@@ -751,7 +751,7 @@ static __global__ void mul_mat_vec_q(
|
||||
// x block quant index when casting the quants to int
|
||||
const int kqs = vdr * (tid % (qi/vdr));
|
||||
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_DGX_SPARK
|
||||
#if __CUDA_ARCH__ == GGML_CUDA_CC_GB10
|
||||
// start the next iterations' weight loads early
|
||||
if constexpr (mmvq_should_prefetch(type)) {
|
||||
// loop iterations, not blocks; q3_K and q6_K keep the distance in blocks they had with vdr 1
|
||||
|
||||
+50
-13
@@ -92,13 +92,16 @@ static __global__ void rope_norm(const T * x,
|
||||
} else if constexpr (std::is_same_v<half, D>) {
|
||||
half2 v = make_half2(x0, x1);
|
||||
ggml_cuda_memcpy_1<4>(dst + idst, &v);
|
||||
} else if constexpr (std::is_same_v<nv_bfloat16, D>) {
|
||||
const __align__(4) nv_bfloat16 v[2] = { ggml_cuda_cast<nv_bfloat16>(x0), ggml_cuda_cast<nv_bfloat16>(x1) };
|
||||
ggml_cuda_memcpy_1<sizeof(v)>(dst + idst, v);
|
||||
}
|
||||
};
|
||||
if (i0 < n_offs || i0 >= n_offs + n_dims) {
|
||||
if (inplace) {
|
||||
return;
|
||||
}
|
||||
store_coaelsced(x[ix + 0], x[ix + 1]);
|
||||
store_coaelsced(ggml_cuda_cast<float>(x[ix + 0]), ggml_cuda_cast<float>(x[ix + 1]));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -113,8 +116,8 @@ static __global__ void rope_norm(const T * x,
|
||||
|
||||
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta);
|
||||
|
||||
const float x0 = x[ix + 0];
|
||||
const float x1 = x[ix + 1];
|
||||
const float x0 = ggml_cuda_cast<float>(x[ix + 0]);
|
||||
const float x1 = ggml_cuda_cast<float>(x[ix + 1]);
|
||||
|
||||
store_coaelsced(x0 * cos_theta - x1 * sin_theta, x0 * sin_theta + x1 * cos_theta);
|
||||
}
|
||||
@@ -189,8 +192,8 @@ static __global__ void rope_neox(const T * x,
|
||||
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta);
|
||||
|
||||
// idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
|
||||
const float x0 = x[ix + n_offs/2 + 0];
|
||||
const float x1 = x[ix + n_offs/2 + n_dims/2];
|
||||
const float x0 = ggml_cuda_cast<float>(x[ix + n_offs/2 + 0]);
|
||||
const float x1 = ggml_cuda_cast<float>(x[ix + n_offs/2 + n_dims/2]);
|
||||
|
||||
dst[idst + n_offs/2 + 0] = ggml_cuda_cast<D>(x0 * cos_theta - x1 * sin_theta);
|
||||
dst[idst + n_offs/2 + n_dims / 2] = ggml_cuda_cast<D>(x0 * sin_theta + x1 * cos_theta);
|
||||
@@ -293,8 +296,8 @@ static __global__ void rope_multi(const T * x,
|
||||
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta);
|
||||
|
||||
// idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
|
||||
const float x0 = x[ix + n_offs/2 + 0];
|
||||
const float x1 = x[ix + n_offs/2 + n_dims/2];
|
||||
const float x0 = ggml_cuda_cast<float>(x[ix + n_offs/2 + 0]);
|
||||
const float x1 = ggml_cuda_cast<float>(x[ix + n_offs/2 + n_dims/2]);
|
||||
|
||||
dst[idst + n_offs/2 + 0] = ggml_cuda_cast<D>(x0*cos_theta - x1*sin_theta);
|
||||
dst[idst + n_offs/2 + n_dims/2] = ggml_cuda_cast<D>(x0*sin_theta + x1*cos_theta);
|
||||
@@ -357,11 +360,11 @@ static __global__ void rope_vision(const T * x,
|
||||
|
||||
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta);
|
||||
|
||||
const float x0 = x[ix + 0];
|
||||
const float x1 = x[ix + n_dims];
|
||||
const float x0 = ggml_cuda_cast<float>(x[ix + 0]);
|
||||
const float x1 = ggml_cuda_cast<float>(x[ix + n_dims]);
|
||||
|
||||
dst[idst + 0] = x0*cos_theta - x1*sin_theta;
|
||||
dst[idst + n_dims] = x0*sin_theta + x1*cos_theta;
|
||||
dst[idst + 0] = ggml_cuda_cast<T>(x0*cos_theta - x1*sin_theta);
|
||||
dst[idst + n_dims] = ggml_cuda_cast<T>(x0*sin_theta + x1*cos_theta);
|
||||
}
|
||||
|
||||
template <bool forward, typename T, typename D>
|
||||
@@ -569,8 +572,8 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
|
||||
}
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
|
||||
GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);
|
||||
GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_BF16);
|
||||
// When not fused, src0 and dst types must match
|
||||
// When fused (ROPE+VIEW+SET_ROWS), src0 may be F32 and dst may be F16
|
||||
GGML_ASSERT(src0->type == dst->type || (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16));
|
||||
@@ -655,6 +658,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
||||
set_rows_stride, inplace, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) {
|
||||
rope_neox_cuda<forward, nv_bfloat16, nv_bfloat16>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
||||
set_rows_stride, inplace, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) {
|
||||
rope_neox_cuda<forward, nv_bfloat16, half>((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
||||
set_rows_stride, inplace, stream);
|
||||
} else {
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
@@ -674,6 +687,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
|
||||
row_indices, set_rows_stride, inplace, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) {
|
||||
rope_multi_cuda<forward, nv_bfloat16, nv_bfloat16>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
|
||||
row_indices, set_rows_stride, inplace, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) {
|
||||
rope_multi_cuda<forward, nv_bfloat16, half>((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
|
||||
row_indices, set_rows_stride, inplace, stream);
|
||||
} else {
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
@@ -686,6 +709,10 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
|
||||
rope_vision_cuda<forward>((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,
|
||||
s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,
|
||||
corr_dims, freq_factors, sections, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16) {
|
||||
rope_vision_cuda<forward>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,
|
||||
s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,
|
||||
corr_dims, freq_factors, sections, stream);
|
||||
} else {
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
@@ -705,6 +732,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
||||
set_rows_stride, inplace, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) {
|
||||
rope_norm_cuda<forward, nv_bfloat16, nv_bfloat16>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
||||
set_rows_stride, inplace, stream);
|
||||
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) {
|
||||
rope_norm_cuda<forward, nv_bfloat16, half>((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
|
||||
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
||||
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
||||
set_rows_stride, inplace, stream);
|
||||
} else {
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
|
||||
@@ -4,8 +4,8 @@
|
||||
typedef void (*set_rows_kernel_t)(const char * src, char * dst);
|
||||
|
||||
// Generic quantized set_rows kernel template
|
||||
template <typename idx_t, typename block_type, int qk, void (*quantize_func)(const float *, block_type *)>
|
||||
static __global__ void k_set_rows_quant(const float * __restrict__ src0,
|
||||
template <typename src_t, typename idx_t, typename block_type, int qk, void (*quantize_func)(const float *, block_type *)>
|
||||
static __global__ void k_set_rows_quant(const src_t * __restrict__ src0,
|
||||
const idx_t * __restrict__ src1,
|
||||
block_type * __restrict__ dst,
|
||||
const int64_t ne_total,
|
||||
@@ -56,13 +56,23 @@ static __global__ void k_set_rows_quant(const float * __restrict__ src0,
|
||||
ggml_cuda_pdl_sync();
|
||||
const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12);
|
||||
|
||||
const float * src0_row = src0 + i01*s01 + i02*s02 + i03*s03;
|
||||
const src_t * src0_row = src0 + i01*s01 + i02*s02 + i03*s03;
|
||||
block_type * dst_row_ptr = dst + (dst_row*s1 + i02*s2 + i03*s3) / sizeof(block_type);
|
||||
|
||||
const float * src_block = src0_row + i00;
|
||||
const src_t * src_block = src0_row + i00;
|
||||
block_type * dst_block = dst_row_ptr + i00 / qk;
|
||||
|
||||
quantize_func(src_block, dst_block);
|
||||
if constexpr (std::is_same_v<src_t, float>) {
|
||||
quantize_func(src_block, dst_block);
|
||||
} else {
|
||||
// the quantize functions take F32 input: widen the block first
|
||||
float src_block_f32[qk];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < qk; ++j) {
|
||||
src_block_f32[j] = ggml_cuda_cast<float>(src_block[j]);
|
||||
}
|
||||
quantize_func(src_block_f32, dst_block);
|
||||
}
|
||||
|
||||
GGML_UNUSED(ne10);
|
||||
GGML_UNUSED(ne11);
|
||||
@@ -71,9 +81,9 @@ static __global__ void k_set_rows_quant(const float * __restrict__ src0,
|
||||
}
|
||||
|
||||
// Template dispatch function for quantized set_rows
|
||||
template<typename idx_t, typename block_type, int qk, void (*quantize_func)(const float*, block_type*)>
|
||||
template<typename src_t, typename idx_t, typename block_type, int qk, void (*quantize_func)(const float*, block_type*)>
|
||||
static void set_rows_cuda_quant(
|
||||
const float * src0_d, const idx_t * src1_d, block_type * dst_d,
|
||||
const src_t * src0_d, const idx_t * src1_d, block_type * dst_d,
|
||||
const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,
|
||||
const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,
|
||||
const size_t nb01, const size_t nb02, const size_t nb03,
|
||||
@@ -87,9 +97,9 @@ static void set_rows_cuda_quant(
|
||||
const dim3 block_size(CUDA_SET_ROWS_BLOCK_SIZE);
|
||||
const dim3 grid_size(num_blocks);
|
||||
|
||||
const int64_t s01 = nb01/sizeof(float);
|
||||
const int64_t s02 = nb02/sizeof(float);
|
||||
const int64_t s03 = nb03/sizeof(float);
|
||||
const int64_t s01 = nb01/sizeof(src_t);
|
||||
const int64_t s02 = nb02/sizeof(src_t);
|
||||
const int64_t s03 = nb03/sizeof(src_t);
|
||||
const int64_t s10 = nb10/sizeof(idx_t);
|
||||
const int64_t s11 = nb11/sizeof(idx_t);
|
||||
const int64_t s12 = nb12/sizeof(idx_t);
|
||||
@@ -104,7 +114,7 @@ static void set_rows_cuda_quant(
|
||||
const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11);
|
||||
const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12);
|
||||
|
||||
k_set_rows_quant<idx_t, block_type, qk, quantize_func><<<grid_size, block_size, 0, stream>>>(
|
||||
k_set_rows_quant<src_t, idx_t, block_type, qk, quantize_func><<<grid_size, block_size, 0, stream>>>(
|
||||
src0_d, src1_d, dst_d, ne_total, ne10, ne11, ne12, ne13, s01, s02, s03, s10, s11, s12, s1, s2, s3, ne00_fd,
|
||||
ne01_fd, ne02_fd, ne11_fd, ne12_fd);
|
||||
}
|
||||
@@ -268,7 +278,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
|
||||
stream
|
||||
);
|
||||
} else if (dst->type == GGML_TYPE_Q4_0) {
|
||||
set_rows_cuda_quant<idx_t, block_q4_0, QK4_0, quantize_f32_q4_0_block>(
|
||||
set_rows_cuda_quant<src_t, idx_t, block_q4_0, QK4_0, quantize_f32_q4_0_block>(
|
||||
src0_d, src1_d, (block_q4_0*)dst->data,
|
||||
ne00, ne01, ne02, ne03,
|
||||
ne10, ne11, ne12, ne13,
|
||||
@@ -278,7 +288,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
|
||||
stream
|
||||
);
|
||||
} else if (dst->type == GGML_TYPE_Q4_1) {
|
||||
set_rows_cuda_quant<idx_t, block_q4_1, QK4_1, quantize_f32_q4_1_block>(
|
||||
set_rows_cuda_quant<src_t, idx_t, block_q4_1, QK4_1, quantize_f32_q4_1_block>(
|
||||
src0_d, src1_d, (block_q4_1*)dst->data,
|
||||
ne00, ne01, ne02, ne03,
|
||||
ne10, ne11, ne12, ne13,
|
||||
@@ -288,7 +298,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
|
||||
stream
|
||||
);
|
||||
} else if (dst->type == GGML_TYPE_Q5_0) {
|
||||
set_rows_cuda_quant<idx_t, block_q5_0, QK5_0, quantize_f32_q5_0_block>(
|
||||
set_rows_cuda_quant<src_t, idx_t, block_q5_0, QK5_0, quantize_f32_q5_0_block>(
|
||||
src0_d, src1_d, (block_q5_0*)dst->data,
|
||||
ne00, ne01, ne02, ne03,
|
||||
ne10, ne11, ne12, ne13,
|
||||
@@ -298,7 +308,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
|
||||
stream
|
||||
);
|
||||
} else if (dst->type == GGML_TYPE_Q5_1) {
|
||||
set_rows_cuda_quant<idx_t, block_q5_1, QK5_1, quantize_f32_q5_1_block>(
|
||||
set_rows_cuda_quant<src_t, idx_t, block_q5_1, QK5_1, quantize_f32_q5_1_block>(
|
||||
src0_d, src1_d, (block_q5_1*)dst->data,
|
||||
ne00, ne01, ne02, ne03,
|
||||
ne10, ne11, ne12, ne13,
|
||||
@@ -308,7 +318,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
|
||||
stream
|
||||
);
|
||||
} else if (dst->type == GGML_TYPE_Q8_0) {
|
||||
set_rows_cuda_quant<idx_t, block_q8_0, QK8_0, quantize_f32_q8_0_block>(
|
||||
set_rows_cuda_quant<src_t, idx_t, block_q8_0, QK8_0, quantize_f32_q8_0_block>(
|
||||
src0_d, src1_d, (block_q8_0*)dst->data,
|
||||
ne00, ne01, ne02, ne03,
|
||||
ne10, ne11, ne12, ne13,
|
||||
@@ -318,7 +328,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
|
||||
stream
|
||||
);
|
||||
} else if (dst->type == GGML_TYPE_IQ4_NL) {
|
||||
set_rows_cuda_quant<idx_t, block_iq4_nl, QK4_NL, quantize_f32_iq4_nl_block>(
|
||||
set_rows_cuda_quant<src_t, idx_t, block_iq4_nl, QK4_NL, quantize_f32_iq4_nl_block>(
|
||||
src0_d, src1_d, (block_iq4_nl*)dst->data,
|
||||
ne00, ne01, ne02, ne03,
|
||||
ne10, ne11, ne12, ne13,
|
||||
@@ -382,12 +392,11 @@ void set_rows_cuda<half, int64_t>(ggml_backend_cuda_context & ctx, const ggml_te
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const ggml_tensor * src0 = dst->src[0];
|
||||
const ggml_tensor * src1 = dst->src[1];
|
||||
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16));
|
||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16));
|
||||
GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32);
|
||||
|
||||
if (src0->type == GGML_TYPE_F32) {
|
||||
@@ -402,6 +411,12 @@ void ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
} else {
|
||||
set_rows_cuda<half, int32_t>(ctx, src0, src1, dst);
|
||||
}
|
||||
} else if (src0->type == GGML_TYPE_BF16) {
|
||||
if (src1->type == GGML_TYPE_I64) {
|
||||
set_rows_cuda<nv_bfloat16, int64_t>(ctx, src0, src1, dst);
|
||||
} else {
|
||||
set_rows_cuda<nv_bfloat16, int32_t>(ctx, src0, src1, dst);
|
||||
}
|
||||
} else {
|
||||
GGML_ABORT("unsupported type %s", ggml_type_name(src0->type));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#include "sleep.cuh"
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
// %globaltimer is a nanosecond wall clock, unlike clock64() it is unaffected by the SM clock and by frequency scaling
|
||||
static __device__ __forceinline__ uint64_t globaltimer_ns() {
|
||||
uint64_t t;
|
||||
asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
|
||||
return t;
|
||||
}
|
||||
|
||||
// a single thread is enough, the next kernel on the same stream cannot start before this one retires
|
||||
static __global__ void sleep_ns(const uint64_t ns) {
|
||||
const uint64_t t0 = globaltimer_ns();
|
||||
|
||||
while (globaltimer_ns() - t0 < ns) {}
|
||||
}
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
sleep_ns<<<1, 1, 0, ctx.stream()>>>(1000*(uint64_t) ggml_get_op_params_i32(dst, 0));
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
GGML_UNUSED(ctx);
|
||||
GGML_UNUSED(dst);
|
||||
GGML_ABORT("GGML_OP_SLEEP requires the %%globaltimer register, which is only available on CUDA");
|
||||
}
|
||||
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
@@ -0,0 +1,3 @@
|
||||
#include "common.cuh"
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
@@ -148,13 +148,14 @@ static void ssm_conv_f32_cuda(const float * src0, const float * src1, const floa
|
||||
};
|
||||
|
||||
switch (nc) {
|
||||
case 2: launch_kernel(std::integral_constant<int, 2 >{}); break;
|
||||
case 3: launch_kernel(std::integral_constant<int, 3 >{}); break;
|
||||
case 4: launch_kernel(std::integral_constant<int, 4 >{}); break;
|
||||
case 5: launch_kernel(std::integral_constant<int, 5 >{}); break;
|
||||
case 7: launch_kernel(std::integral_constant<int, 7 >{}); break;
|
||||
case 9: launch_kernel(std::integral_constant<int, 9 >{}); break;
|
||||
case 15: launch_kernel(std::integral_constant<int, 15>{}); break;
|
||||
default: GGML_ABORT("Only support kernel sizes 3, 4, 5, 7, 9, 15 right now.");
|
||||
default: GGML_ABORT("Only support kernel sizes 2, 3, 4, 5, 7, 9, 15 right now.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -184,13 +184,6 @@ static __global__ void top_k_radix_gather(
|
||||
constexpr int NBINS = 1 << RADIX_BITS;
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
constexpr int nwarps = BLOCK_SIZE / warp_size;
|
||||
const auto popc = [](const ggml_cuda_thread_mask_t mask) {
|
||||
if constexpr (warp_size == 64) {
|
||||
return __popcll(mask);
|
||||
} else {
|
||||
return __popc(mask);
|
||||
}
|
||||
};
|
||||
__shared__ int warp_ties[nwarps];
|
||||
|
||||
const int row = blockIdx.x / blocks_per_row;
|
||||
@@ -247,11 +240,11 @@ static __global__ void top_k_radix_gather(
|
||||
const bool is_tie = col < warp_end && top_k_float_to_ordered(row_src[col]) == threshold;
|
||||
|
||||
const ggml_cuda_thread_mask_t ties = ggml_cuda_ballot(is_tie);
|
||||
const int tie_pos = pos + popc(ties & (((ggml_cuda_thread_mask_t) 1 << lane) - 1));
|
||||
const int tie_pos = pos + ggml_cuda_popc(ties & (((ggml_cuda_thread_mask_t) 1 << lane) - 1));
|
||||
if (is_tie && tie_pos < pos_end) {
|
||||
row_dst[k - rank + tie_pos] = col;
|
||||
}
|
||||
pos += popc(ties);
|
||||
pos += ggml_cuda_popc(ties);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2551,6 +2551,26 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_
|
||||
char base[256];
|
||||
char name[256];
|
||||
|
||||
const bool has_tensor = ggml_metal_device_get_props(ggml_metal_library_get_device(lib))->has_tensor;
|
||||
|
||||
if (has_tensor) {
|
||||
// nr0 output positions per threadgroup, nr1 output channels, same tiles as kernel_mul_mm
|
||||
snprintf(base, 256, "kernel_conv_2d_mm_tensor_%s_%s", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type));
|
||||
snprintf(name, 256, "%s", base);
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
|
||||
}
|
||||
|
||||
res.nr0 = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y;
|
||||
res.nr1 = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X;
|
||||
res.nsg = N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y;
|
||||
res.smem = res.nr0 * N_MM_NK_TOTAL * sizeof(ggml_fp16_t);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
const int nr0 = op->src[0]->ne[3] <= 32 ? 32 : 64;
|
||||
const int nr1 = 2048/nr0;
|
||||
|
||||
@@ -2564,13 +2584,27 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_
|
||||
|
||||
res.nr0 = nr0;
|
||||
res.nr1 = nr1;
|
||||
res.nsg = 4;
|
||||
res.nsg = N_SG_CONV_2D;
|
||||
|
||||
res.smem = nr0*nr1*sizeof(float);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm_reduce(ggml_metal_library_t lib, const ggml_tensor * op) {
|
||||
assert(op->op == GGML_OP_CONV_2D);
|
||||
GGML_UNUSED(op);
|
||||
|
||||
const char * name = "kernel_conv_2d_mm_reduce";
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw(ggml_metal_library_t lib, const ggml_tensor * op, bool tiled) {
|
||||
assert(op->op == GGML_OP_CONV_2D_DW);
|
||||
|
||||
|
||||
@@ -170,6 +170,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake (ggml_metal_library_t lib, enum ggml_type type);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm_reduce (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tiled);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
|
||||
@@ -1713,7 +1713,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
||||
return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32 && (ggml_is_contiguous_rows(op->src[0]));
|
||||
case GGML_OP_ROPE:
|
||||
case GGML_OP_ROPE_BACK:
|
||||
return ggml_rope_get_truncate(op);
|
||||
return ggml_rope_get_truncate(op) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
|
||||
case GGML_OP_IM2COL:
|
||||
return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32);
|
||||
case GGML_OP_CONV_2D:
|
||||
@@ -2375,7 +2375,7 @@ void ggml_metal_buffer_memset_tensor(ggml_metal_buffer_t buf, struct ggml_tensor
|
||||
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
||||
|
||||
[encoder fillBuffer:bid_dst.metal
|
||||
range:NSMakeRange(bid_dst.offs, bid_dst.offs + size)
|
||||
range:NSMakeRange(bid_dst.offs, size)
|
||||
value:value];
|
||||
|
||||
[encoder endEncoding];
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
#define N_ARGMAX_NPART_MAX 256
|
||||
#define N_ARGMAX_CHUNK 4096
|
||||
|
||||
#define N_SG_CONV_2D 4
|
||||
|
||||
// kernel parameters for mat-vec threadgroups
|
||||
//
|
||||
// N_R0: number of src0 rows to process per simdgroup
|
||||
@@ -752,8 +754,14 @@ typedef struct {
|
||||
int32_t p1;
|
||||
int32_t d0;
|
||||
int32_t d1;
|
||||
int32_t nk; // reduction length per split
|
||||
} ggml_metal_kargs_conv_2d;
|
||||
|
||||
typedef struct {
|
||||
int32_t ne; // output elements
|
||||
int32_t nsplit;
|
||||
} ggml_metal_kargs_conv_2d_reduce;
|
||||
|
||||
typedef struct {
|
||||
uint64_t nb00; // kernel strides
|
||||
uint64_t nb01;
|
||||
|
||||
@@ -4959,6 +4959,33 @@ int ggml_metal_op_im2col(ggml_metal_op_t ctx, int idx) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// number of reduction splits of the conv_2d matmul kernels
|
||||
// A small output with a long reduction leaves the GPU idle, so the reduction is split until there are about 1M partial sums.
|
||||
static int ggml_metal_op_conv_2d_nsplit(const ggml_tensor * op, int32_t * nk) {
|
||||
const int64_t K = op->src[0]->ne[0]*op->src[0]->ne[1]*op->src[0]->ne[2];
|
||||
const int64_t nsteps = (K + 31)/32;
|
||||
const int64_t ne = ggml_nelements(op);
|
||||
|
||||
// at least 3 reduction steps of 32 per split, at most 32 splits
|
||||
const int64_t nsplit = std::max<int64_t>(1, std::min<int64_t>({ (1024*1024 + ne - 1)/ne, nsteps/3, 32 }));
|
||||
const int64_t steps = (nsteps + nsplit - 1)/nsplit;
|
||||
|
||||
if (nk) {
|
||||
*nk = (int32_t) (steps*32);
|
||||
}
|
||||
|
||||
return (int) ((nsteps + steps - 1)/steps);
|
||||
}
|
||||
|
||||
size_t ggml_metal_op_conv_2d_extra_tmp(const ggml_tensor * op) {
|
||||
assert(op->op == GGML_OP_CONV_2D);
|
||||
|
||||
const int nsplit = ggml_metal_op_conv_2d_nsplit(op, nullptr);
|
||||
|
||||
// partial sums of the reduction splits
|
||||
return nsplit > 1 ? nsplit*ggml_nelements(op)*sizeof(float) : 0;
|
||||
}
|
||||
|
||||
int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_tensor * op = ctx->node(idx);
|
||||
|
||||
@@ -5012,11 +5039,17 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
/*.p1 =*/ p1,
|
||||
/*.d0 =*/ d0,
|
||||
/*.d1 =*/ d1,
|
||||
/*.nk =*/ 0,
|
||||
};
|
||||
|
||||
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev);
|
||||
|
||||
if (props_dev->has_simdgroup_mm) {
|
||||
// the matmul kernels walk the patches with 32-bit byte offsets inside an image
|
||||
// larger images, possibly a strided view, take the generic kernel
|
||||
const uint64_t max_off = (ne12 - 1)*nb12 + (ne11 - 1)*nb11 + (ne10 - 1)*nb10 + sizeof(float);
|
||||
const bool fits_32 = max_off <= UINT32_MAX;
|
||||
|
||||
if (props_dev->has_simdgroup_mm && fits_32) {
|
||||
auto pipeline = ggml_metal_library_get_pipeline_conv_2d_mm(lib, op);
|
||||
|
||||
const int nr0 = pipeline.nr0;
|
||||
@@ -5026,15 +5059,48 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
const int64_t NP = ne3*ne1*ne0;
|
||||
const int64_t OC = ne03;
|
||||
|
||||
const int nsplit = ggml_metal_op_conv_2d_nsplit(op, &args.nk);
|
||||
|
||||
ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);
|
||||
ggml_metal_buffer_id bid_tmp = bid_dst;
|
||||
bid_tmp.offs += ggml_nbytes(op);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline);
|
||||
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
|
||||
ggml_metal_encoder_set_buffer (enc, nsplit > 1 ? bid_tmp : bid_dst, 3);
|
||||
|
||||
ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, 1, 32, nsg, 1);
|
||||
if (props_dev->has_tensor) {
|
||||
// per image: nr0 positions times nr1 channels per threadgroup
|
||||
const int64_t OHW = ne1*ne0;
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (OC + nr1 - 1)/nr1, (OHW + nr0 - 1)/nr0, ne3*nsplit, 32, nsg, 1);
|
||||
} else {
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, nsplit, 32, nsg, 1);
|
||||
}
|
||||
|
||||
if (nsplit > 1) {
|
||||
GGML_ASSERT(ggml_is_contiguous(op));
|
||||
|
||||
// sync the 2 kernels
|
||||
ggml_metal_op_concurrency_reset(ctx);
|
||||
|
||||
ggml_metal_kargs_conv_2d_reduce args0 = {
|
||||
/*.ne =*/ (int32_t) ggml_nelements(op),
|
||||
/*.nsplit =*/ nsplit,
|
||||
};
|
||||
|
||||
auto pipeline0 = ggml_metal_library_get_pipeline_conv_2d_mm_reduce(lib, op);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline0);
|
||||
ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_tmp, 1);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_dst, 2);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (args0.ne + 255)/256, 1, 1, 256, 1, 1);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -49,6 +49,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_rows(const struct ggml_tensor * op);
|
||||
|
||||
size_t ggml_metal_op_argmax_extra(const struct ggml_tensor * op);
|
||||
|
||||
size_t ggml_metal_op_conv_2d_extra_tmp(const struct ggml_tensor * op);
|
||||
|
||||
int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_acc (ggml_metal_op_t ctx, int idx);
|
||||
|
||||
@@ -311,6 +311,10 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
|
||||
{
|
||||
res = 2*sizeof(int32_t)*ggml_nelements(tensor->src[0]);
|
||||
} break;
|
||||
case GGML_OP_CONV_2D:
|
||||
{
|
||||
res += ggml_metal_op_conv_2d_extra_tmp(tensor);
|
||||
} break;
|
||||
case GGML_OP_ARGMAX:
|
||||
{
|
||||
res += ggml_metal_op_argmax_extra(tensor);
|
||||
|
||||
@@ -285,7 +285,7 @@ kernel void kernel_conv_2d_mm(
|
||||
ushort tiitg[[thread_index_in_threadgroup]],
|
||||
ushort tiisg[[thread_index_in_simdgroup]],
|
||||
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
|
||||
constexpr short NSG = 4;
|
||||
constexpr short NSG = N_SG_CONV_2D;
|
||||
constexpr short NTH = 32*NSG;
|
||||
constexpr short NK = 32;
|
||||
constexpr short NKB = NK/8;
|
||||
@@ -300,6 +300,10 @@ kernel void kernel_conv_2d_mm(
|
||||
const int K = args.IC*args.KH*args.KW;
|
||||
const int NP = args.N*args.OH*args.OW;
|
||||
|
||||
// this threadgroup reduces [k_begin, k_end), the splits are summed by kernel_conv_2d_mm_reduce
|
||||
const int k_begin = tgpig.z*args.nk;
|
||||
const int k_end = min(K, k_begin + args.nk);
|
||||
|
||||
const int r0 = tgpig.y*NR0;
|
||||
const int r1 = tgpig.x*NR1;
|
||||
|
||||
@@ -308,8 +312,14 @@ kernel void kernel_conv_2d_mm(
|
||||
const short lb_n = 8*sgitg + tiisg%8;
|
||||
const short lb_k = 8*(tiisg/8);
|
||||
|
||||
// the patches are walked with 32-bit byte offsets inside the image, the host checks that an image fits
|
||||
const uint nb10 = args.nb10;
|
||||
const uint nb11 = args.nb11;
|
||||
const uint nb12 = args.nb12;
|
||||
|
||||
int ix0[NPL];
|
||||
int iy0[NPL];
|
||||
uint off0[NPL];
|
||||
bool ip_ok[NPL];
|
||||
|
||||
device const char * srcb[NPL];
|
||||
@@ -324,6 +334,8 @@ kernel void kernel_conv_2d_mm(
|
||||
ix0[t] = iow*args.s0 - args.p0;
|
||||
iy0[t] = ioh*args.s1 - args.p1;
|
||||
|
||||
off0[t] = iy0[t]*nb11 + ix0[t]*nb10;
|
||||
|
||||
ip_ok[t] = ip < NP;
|
||||
|
||||
srcb[t] = src + in*args.nb13;
|
||||
@@ -337,7 +349,7 @@ kernel void kernel_conv_2d_mm(
|
||||
mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
|
||||
}
|
||||
|
||||
for (int k0 = 0; k0 < K; k0 += NK) {
|
||||
for (int k0 = k_begin; k0 < k_end; k0 += NK) {
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
{
|
||||
@@ -349,7 +361,7 @@ kernel void kernel_conv_2d_mm(
|
||||
const int oc = r0 + row;
|
||||
|
||||
half v = 0;
|
||||
if (kk < K && oc < args.OC) {
|
||||
if (kk < k_end && oc < args.OC) {
|
||||
v = *((device const TK *)(weights + oc*args.nb03) + kk);
|
||||
}
|
||||
|
||||
@@ -364,7 +376,11 @@ kernel void kernel_conv_2d_mm(
|
||||
|
||||
int kw = kk % args.KW;
|
||||
int kh = (kk / args.KW) % args.KH;
|
||||
int ic = kk / (args.KW*args.KH);
|
||||
int dx = kw*args.d0;
|
||||
int dy = kh*args.d1;
|
||||
|
||||
// byte offset of the reduction index kk, relative to the patch origin
|
||||
uint offk = (kk / (args.KW*args.KH))*nb12 + dy*nb11 + dx*nb10;
|
||||
|
||||
FOR_UNROLL (short i = 0; i < 8; i++) {
|
||||
const short k = lb_k + i;
|
||||
@@ -372,12 +388,12 @@ kernel void kernel_conv_2d_mm(
|
||||
FOR_UNROLL (short t = 0; t < NPL; t++) {
|
||||
const short n = 32*t + lb_n;
|
||||
|
||||
const int ix = ix0[t] + kw*args.d0;
|
||||
const int iy = iy0[t] + kh*args.d1;
|
||||
const int ix = ix0[t] + dx;
|
||||
const int iy = iy0[t] + dy;
|
||||
|
||||
half v = 0;
|
||||
if (ip_ok[t] && kk < K && ix >= 0 && ix < args.IW && iy >= 0 && iy < args.IH) {
|
||||
v = *(device const float *)(srcb[t] + ic*args.nb12 + iy*args.nb11 + ix*args.nb10);
|
||||
if (ip_ok[t] && kk < k_end && (uint) ix < (uint) args.IW && (uint) iy < (uint) args.IH) {
|
||||
v = *(device const float *)(srcb[t] + (off0[t] + offk));
|
||||
}
|
||||
|
||||
const short ib = NKB*(n/8) + k/8;
|
||||
@@ -386,11 +402,17 @@ kernel void kernel_conv_2d_mm(
|
||||
}
|
||||
|
||||
kk++;
|
||||
dx += args.d0;
|
||||
offk += args.d0*nb10;
|
||||
if (++kw == args.KW) {
|
||||
kw = 0;
|
||||
kw = 0;
|
||||
dx = 0;
|
||||
dy += args.d1;
|
||||
offk += args.d1*nb11 - args.KW*args.d0*nb10;
|
||||
if (++kh == args.KH) {
|
||||
kh = 0;
|
||||
ic++;
|
||||
kh = 0;
|
||||
dy = 0;
|
||||
offk += nb12 - args.KH*args.d1*nb11;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -398,7 +420,7 @@ kernel void kernel_conv_2d_mm(
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
const short nkb = min((K - k0 + 7)/8, (int) NKB);
|
||||
const short nkb = min((k_end - k0 + 7)/8, (int) NKB);
|
||||
|
||||
for (short ik = 0; ik < nkb; ik++) {
|
||||
FOR_UNROLL (short i = 0; i < 4; i++) {
|
||||
@@ -436,7 +458,8 @@ kernel void kernel_conv_2d_mm(
|
||||
const int oh = (p / args.OW) % args.OH;
|
||||
const int n = p / (args.OW*args.OH);
|
||||
|
||||
device char * dstp = dst + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
|
||||
// with several splits, dst is the partial sum buffer, one contiguous output per split
|
||||
device char * dstp = dst + (uint64_t) tgpig.z*args.N*args.nb3 + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
|
||||
|
||||
for (short j = 0; j < NR0*NR1/NTH; j++) {
|
||||
const short row = tiitg/NR1 + (NTH/NR1)*j;
|
||||
@@ -448,6 +471,158 @@ kernel void kernel_conv_2d_mm(
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef GGML_METAL_HAS_TENSOR
|
||||
// conv_2d as a matmul on the tensor cores, like kernel_mul_mm
|
||||
// The patches of NRA output positions are gathered into threadgroup memory, the kernel rows are read from device memory.
|
||||
// One threadgroup computes NRA positions of one image times NRB output channels, over the reduction range of its split.
|
||||
template <typename TK>
|
||||
kernel void kernel_conv_2d_mm_tensor(
|
||||
constant ggml_metal_kargs_conv_2d & args,
|
||||
device const char * weights,
|
||||
device const char * src,
|
||||
device char * dst,
|
||||
threadgroup char * shmem [[threadgroup(0)]],
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
uint3 tgpg[[threadgroups_per_grid]],
|
||||
ushort tiitg[[thread_index_in_threadgroup]]) {
|
||||
constexpr int NRB = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X; // output channels
|
||||
constexpr int NRA = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y; // output positions
|
||||
constexpr int NK = N_MM_NK_TOTAL;
|
||||
constexpr int NTH = N_SIMDWIDTH * N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y;
|
||||
|
||||
// each thread gathers NK/(NTH/NRA) consecutive reduction indices of one position
|
||||
constexpr int KPT = NK/(NTH/NRA);
|
||||
|
||||
const int K = args.IC*args.KH*args.KW;
|
||||
const int OHW = args.OH*args.OW;
|
||||
|
||||
const int nsplit = tgpg.z/args.N;
|
||||
const int img = tgpig.z/nsplit;
|
||||
const int split = tgpig.z%nsplit;
|
||||
|
||||
const int k_begin = split*args.nk;
|
||||
const int k_end = min(K, k_begin + args.nk);
|
||||
|
||||
const int ra = tgpig.y*NRA;
|
||||
const int rb = tgpig.x*NRB;
|
||||
|
||||
threadgroup half * sa = (threadgroup half *) shmem;
|
||||
|
||||
// the position this thread gathers
|
||||
const int row = tiitg%NRA;
|
||||
const int kc = (tiitg/NRA)*KPT;
|
||||
const int p = ra + row;
|
||||
const bool pok = p < OHW;
|
||||
const int ix0 = (p%args.OW)*args.s0 - args.p0;
|
||||
const int iy0 = (p/args.OW)*args.s1 - args.p1;
|
||||
|
||||
device const char * srcb = src + img*args.nb13;
|
||||
|
||||
// the patch is walked with 32-bit byte offsets inside the image, the host checks that an image fits
|
||||
const uint nb10 = args.nb10;
|
||||
const uint nb11 = args.nb11;
|
||||
const uint nb12 = args.nb12;
|
||||
|
||||
const uint off0 = iy0*nb11 + ix0*nb10;
|
||||
|
||||
device TK * ptrB = (device TK *) weights;
|
||||
const int strideB = args.nb03/sizeof(TK);
|
||||
|
||||
mpp::tensor_ops::matmul2d<
|
||||
mpp::tensor_ops::matmul2d_descriptor(
|
||||
NRB, NRA, static_cast<int>(dynamic_extent), false, true, true,
|
||||
mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
|
||||
execution_simdgroups<N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y>> mm;
|
||||
|
||||
auto tB0 = tensor(ptrB, dextents<int32_t, 2>(K, args.OC), array<int, 2>({1, strideB}));
|
||||
auto tA0 = tensor(sa, dextents<int32_t, 2>(NK, NRA));
|
||||
auto cT = mm.get_destination_cooperative_tensor<decltype(tB0), decltype(tA0), float>();
|
||||
|
||||
for (int k0 = k_begin; k0 < k_end; k0 += NK) {
|
||||
// the reduction indices past k_end are beyond the extent given to the matmul, they are not gathered
|
||||
if (k0 + kc < k_end) {
|
||||
// gather KPT patch values of this position, walking the reduction index incrementally
|
||||
int kk = k0 + kc;
|
||||
int kw = kk % args.KW;
|
||||
int kh = (kk / args.KW) % args.KH;
|
||||
int dx = kw*args.d0;
|
||||
int dy = kh*args.d1;
|
||||
|
||||
// byte offset of the reduction index kk, relative to the patch origin
|
||||
uint offk = (kk / (args.KW*args.KH))*nb12 + dy*nb11 + dx*nb10;
|
||||
|
||||
FOR_UNROLL (short i = 0; i < KPT; i++) {
|
||||
const int ix = ix0 + dx;
|
||||
const int iy = iy0 + dy;
|
||||
|
||||
half v = 0;
|
||||
if (pok && kk < k_end && (uint) ix < (uint) args.IW && (uint) iy < (uint) args.IH) {
|
||||
v = *(device const float *)(srcb + (off0 + offk));
|
||||
}
|
||||
sa[row*NK + kc + i] = v;
|
||||
|
||||
kk++;
|
||||
dx += args.d0;
|
||||
offk += args.d0*nb10;
|
||||
if (++kw == args.KW) {
|
||||
kw = 0;
|
||||
dx = 0;
|
||||
dy += args.d1;
|
||||
offk += args.d1*nb11 - args.KW*args.d0*nb10;
|
||||
if (++kh == args.KH) {
|
||||
kh = 0;
|
||||
dy = 0;
|
||||
offk += nb12 - args.KH*args.d1*nb11;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
// clamp the reduction extent so neither operand is read past the end of the split
|
||||
const int kExt = min(NK, k_end - k0);
|
||||
|
||||
auto tAv = tensor(sa, dextents<int32_t, 2>(kExt, NRA), array<int, 2>({1, NK}));
|
||||
auto tBv = tensor(ptrB + k0 + rb*strideB, dextents<int32_t, 2>(kExt, args.OC - rb), array<int, 2>({1, strideB}));
|
||||
|
||||
mm.run(tBv, tAv, cT);
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
}
|
||||
|
||||
// dst holds [OHW, OC] per image, with several splits it is the partial sum buffer, one contiguous output per split
|
||||
const int tileM = min(NRA, OHW - ra);
|
||||
const int tileN = min(NRB, args.OC - rb);
|
||||
|
||||
device float * dstTile = (device float *) (dst + (uint64_t) split*args.N*args.nb3 + img*args.nb3) + (uint64_t) rb*OHW + ra;
|
||||
|
||||
auto tD = tensor(dstTile, dextents<int32_t, 2>(tileM, tileN), array<int, 2>({1, OHW}));
|
||||
cT.store(tD);
|
||||
}
|
||||
|
||||
typedef decltype(kernel_conv_2d_mm_tensor<float>) kernel_conv_2d_mm_tensor_t;
|
||||
|
||||
template [[host_name("kernel_conv_2d_mm_tensor_f32_f32")]] kernel kernel_conv_2d_mm_tensor_t kernel_conv_2d_mm_tensor<float>;
|
||||
template [[host_name("kernel_conv_2d_mm_tensor_f16_f32")]] kernel kernel_conv_2d_mm_tensor_t kernel_conv_2d_mm_tensor<half>;
|
||||
#endif
|
||||
|
||||
kernel void kernel_conv_2d_mm_reduce(
|
||||
constant ggml_metal_kargs_conv_2d_reduce & args,
|
||||
device const float * tmp,
|
||||
device float * dst,
|
||||
uint tpig[[thread_position_in_grid]]) {
|
||||
if ((int) tpig >= args.ne) {
|
||||
return;
|
||||
}
|
||||
|
||||
float sum = 0.0f;
|
||||
for (int s = 0; s < args.nsplit; ++s) {
|
||||
sum += tmp[(uint64_t) s*args.ne + tpig];
|
||||
}
|
||||
dst[tpig] = sum;
|
||||
}
|
||||
|
||||
typedef decltype(kernel_conv_2d_mm<float, 64, 32>) kernel_conv_2d_mm_t;
|
||||
|
||||
template [[host_name("kernel_conv_2d_mm_f32_f32_64x32")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<float, 64, 32>;
|
||||
|
||||
@@ -323,6 +323,9 @@ set(GGML_OPENCL_KERNELS
|
||||
mul_mm_f16_f32_kq_kqv
|
||||
mul_mm_f16_f32_kq_p8
|
||||
mul_mm_q8_f32_kqv
|
||||
mul_mm_q8_kq_p8
|
||||
mul_mm_q8_kq_p8_dk512
|
||||
flash_attn_q8_rows
|
||||
conv2d
|
||||
conv2d_f16_f32
|
||||
conv2_dw
|
||||
|
||||
@@ -781,6 +781,10 @@ struct ggml_backend_opencl_context {
|
||||
ggml_cl_buffer prealloc_fa_pd;
|
||||
ggml_cl_buffer prealloc_fa_bmax;
|
||||
ggml_cl_buffer prealloc_fa_bsum;
|
||||
ggml_cl_buffer prealloc_fa_k8;
|
||||
ggml_cl_buffer prealloc_fa_k8_scale;
|
||||
ggml_cl_buffer prealloc_fa_q8;
|
||||
ggml_cl_buffer prealloc_fa_q8_scale;
|
||||
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
ggml_cl_buffer prealloc_adreno_xmem_const;
|
||||
@@ -899,6 +903,10 @@ struct ggml_backend_opencl_context {
|
||||
cl_kernel kernel_fa_kqv_direct_f16 = nullptr;
|
||||
cl_kernel kernel_fa_kqv_direct_reduce = nullptr;
|
||||
cl_kernel kernel_fa_v_transpose_q8 = nullptr;
|
||||
cl_kernel kernel_fa_q8_rows_f16 = nullptr;
|
||||
cl_kernel kernel_fa_q8_rows_f32 = nullptr;
|
||||
cl_kernel kernel_mul_mm_q8_kq_p8 = nullptr;
|
||||
cl_kernel kernel_mul_mm_q8_kq_p8_dk512 = nullptr;
|
||||
ggml_opencl_fa_kernels fa;
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
ggml_cl_adreno_xmem_attn_state adreno_xmem_attn;
|
||||
@@ -1506,7 +1514,18 @@ struct ggml_backend_opencl_context {
|
||||
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
|
||||
~ggml_backend_opencl_context() {
|
||||
for (cl_kernel kernel : {kernel_mul_mm_f16_f32_kq_p8, kernel_mul_mm_q8_f32_kqv, kernel_fa_p8_fixup, kernel_fa_v_transpose_q8, kernel_fa_kqv_direct_f16, kernel_fa_kqv_direct_reduce}) {
|
||||
for (cl_kernel kernel : {
|
||||
kernel_mul_mm_f16_f32_kq_p8,
|
||||
kernel_mul_mm_q8_f32_kqv,
|
||||
kernel_fa_p8_fixup,
|
||||
kernel_fa_v_transpose_q8,
|
||||
kernel_fa_kqv_direct_f16,
|
||||
kernel_fa_kqv_direct_reduce,
|
||||
kernel_fa_q8_rows_f16,
|
||||
kernel_fa_q8_rows_f32,
|
||||
kernel_mul_mm_q8_kq_p8,
|
||||
kernel_mul_mm_q8_kq_p8_dk512
|
||||
}) {
|
||||
if (kernel != nullptr) {
|
||||
CL_CHECK(clReleaseKernel(kernel));
|
||||
}
|
||||
@@ -1648,7 +1667,7 @@ static cl_program build_program_from_source_ex(cl_context ctx, cl_device_id dev,
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts) {
|
||||
static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts, bool fatal = true) {
|
||||
cl_context ctx = backend_ctx->context;
|
||||
cl_device_id dev = backend_ctx->device;
|
||||
|
||||
@@ -1660,7 +1679,7 @@ static cl_program build_program_from_source(ggml_backend_opencl_context * backen
|
||||
return p_cached;
|
||||
}
|
||||
|
||||
cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, /*fatal=*/true);
|
||||
cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, fatal);
|
||||
|
||||
// Best-effort save of the freshly-built binary (no-op if cache disabled).
|
||||
if (p != nullptr) {
|
||||
@@ -3995,6 +4014,52 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
}
|
||||
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
|
||||
#ifdef GGML_OPENCL_EMBED_KERNELS
|
||||
const std::string rows_src {
|
||||
#include "flash_attn_q8_rows.cl.h"
|
||||
};
|
||||
const std::string kq_src {
|
||||
#include "mul_mm_q8_kq_p8.cl.h"
|
||||
};
|
||||
#else
|
||||
const std::string rows_src = read_file("flash_attn_q8_rows.cl");
|
||||
const std::string kq_src = read_file("mul_mm_q8_kq_p8.cl");
|
||||
#endif
|
||||
cl_program prog = build_program_from_source(backend_ctx, rows_src.c_str(), compile_opts, /*fatal=*/false);
|
||||
if (prog != nullptr) {
|
||||
CL_CHECK((backend_ctx->kernel_fa_q8_rows_f16 = clCreateKernel(prog, "kernel_fa_q8_rows_f16", &err), err));
|
||||
CL_CHECK((backend_ctx->kernel_fa_q8_rows_f32 = clCreateKernel(prog, "kernel_fa_q8_rows_f32", &err), err));
|
||||
CL_CHECK(clReleaseProgram(prog));
|
||||
}
|
||||
prog = build_program_from_source(backend_ctx, kq_src.c_str(), compile_opts, /*fatal=*/false);
|
||||
if (prog != nullptr) {
|
||||
CL_CHECK((backend_ctx->kernel_mul_mm_q8_kq_p8 = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8", &err), err));
|
||||
CL_CHECK(clReleaseProgram(prog));
|
||||
}
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
// mul_mm_q8_kq_p8_dk512
|
||||
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
|
||||
#ifdef GGML_OPENCL_EMBED_KERNELS
|
||||
const std::string kernel_src {
|
||||
#include "mul_mm_q8_kq_p8_dk512.cl.h"
|
||||
};
|
||||
#else
|
||||
const std::string kernel_src = read_file("mul_mm_q8_kq_p8_dk512.cl");
|
||||
#endif
|
||||
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts, /*fatal=*/false);
|
||||
if (prog != nullptr) {
|
||||
cl_kernel kernel = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8_dk512", &err);
|
||||
CL_CHECK(clReleaseProgram(prog));
|
||||
if (err == CL_SUCCESS) {
|
||||
backend_ctx->kernel_mul_mm_q8_kq_p8_dk512 = kernel;
|
||||
}
|
||||
}
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
// mul_mm_f16_f32_kq_p8
|
||||
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
|
||||
#ifdef GGML_OPENCL_EMBED_KERNELS
|
||||
@@ -12305,7 +12370,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
|
||||
}
|
||||
case GGML_OP_SOLVE_TRI:
|
||||
return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]);
|
||||
@@ -22922,12 +22987,58 @@ static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_te
|
||||
}
|
||||
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
|
||||
static void ggml_cl_fa_q8_rows(
|
||||
ggml_backend_t backend, const ggml_tensor * src, const ggml_tensor * dst,
|
||||
cl_mem values, cl_mem scales, int64_t row0, int nrow
|
||||
) {
|
||||
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
|
||||
auto * extra = (ggml_tensor_extra_cl *) (src->view_src ? src->view_src->extra : src->extra);
|
||||
const cl_ulong offset = extra->offset + src->view_offs + row0 * src->nb[1];
|
||||
const cl_ulong nb1 = src->nb[1];
|
||||
const cl_ulong nb2 = src->nb[2];
|
||||
const int dk = src->ne[0];
|
||||
const int nhead = src->ne[2];
|
||||
cl_kernel kernel;
|
||||
switch (src->type) {
|
||||
case GGML_TYPE_F16: kernel = backend_ctx->kernel_fa_q8_rows_f16; break;
|
||||
case GGML_TYPE_F32: kernel = backend_ctx->kernel_fa_q8_rows_f32; break;
|
||||
default: GGML_ABORT("unsupported int8 KQ input type");
|
||||
}
|
||||
cl_uint a = 0;
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &offset));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb2));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &values));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &scales));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dk));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nrow));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nhead));
|
||||
const size_t blocks = (size_t) (dk / 32) * nrow * nhead;
|
||||
size_t gws[3] = {(blocks + 63) / 64 * 64, 1, 1};
|
||||
size_t lws[3] = {64, 1, 1};
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
}
|
||||
|
||||
static void ggml_cl_fa_kq_int8_prepare(ggml_backend_t backend, const ggml_tensor * dst, int64_t n_q_chunk) {
|
||||
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
|
||||
const ggml_tensor * q = dst->src[0];
|
||||
const ggml_tensor * k = dst->src[1];
|
||||
const size_t k_bytes = (size_t) k->ne[0] * k->ne[1] * k->ne[2];
|
||||
const size_t q_bytes = (size_t) q->ne[0] * n_q_chunk * q->ne[2];
|
||||
backend_ctx->prealloc_fa_k8.allocate(backend_ctx->context, k_bytes);
|
||||
backend_ctx->prealloc_fa_k8_scale.allocate(backend_ctx->context, k_bytes / 32 * sizeof(cl_half));
|
||||
backend_ctx->prealloc_fa_q8.allocate(backend_ctx->context, q_bytes);
|
||||
backend_ctx->prealloc_fa_q8_scale.allocate(backend_ctx->context, q_bytes / 32 * sizeof(cl_half));
|
||||
ggml_cl_fa_q8_rows(backend, k, dst, backend_ctx->prealloc_fa_k8.buffer, backend_ctx->prealloc_fa_k8_scale.buffer, 0, k->ne[1]);
|
||||
}
|
||||
|
||||
static void ggml_cl_fa_kq_p8_dispatch(
|
||||
ggml_backend_t backend,
|
||||
const ggml_tensor * q, const ggml_tensor * k,
|
||||
const ggml_tensor * mask, const ggml_tensor * sinks,
|
||||
const ggml_tensor * dst,
|
||||
int64_t q0, int nqc, float scale, float max_bias, int p_pitch
|
||||
int64_t q0, int nqc, float scale, float max_bias, int p_pitch, bool kq_int8
|
||||
) {
|
||||
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
|
||||
auto * extra_k = (ggml_tensor_extra_cl *) (k->view_src ? k->view_src->extra : k->extra);
|
||||
@@ -22940,64 +23051,94 @@ static void ggml_cl_fa_kq_p8_dispatch(
|
||||
const int D_A = k->ne[2];
|
||||
const int D_B = q->ne[2];
|
||||
|
||||
cl_int err;
|
||||
|
||||
cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
|
||||
cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
cl_image_format fmt = {CL_RGBA, CL_FLOAT};
|
||||
cl_image_desc desc = {};
|
||||
desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
|
||||
desc.image_width = region.size / 16;
|
||||
desc.buffer = k_sub;
|
||||
cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
|
||||
region.size = (size_t) K * N * D_B * sizeof(float);
|
||||
cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
const cl_ulong off_mask = extra_m->offset + mask->view_offs + q0 * mask->nb[1];
|
||||
const cl_ulong mask_nb1 = mask->nb[1];
|
||||
const int n_head_log2 = 1u << (unsigned) floorf(log2f((float) D_B));
|
||||
const float m0 = powf(2.0f, -max_bias / n_head_log2);
|
||||
const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
|
||||
|
||||
cl_kernel kernel;
|
||||
cl_uint a = 0;
|
||||
|
||||
cl_kernel kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
|
||||
if (kq_int8) {
|
||||
ggml_cl_fa_q8_rows(backend, q, dst, backend_ctx->prealloc_fa_q8.buffer, backend_ctx->prealloc_fa_q8_scale.buffer, q0, nqc);
|
||||
kernel = K <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8_scale.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8_scale.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
|
||||
size_t gws[3] = {64 * (((size_t) N + 31) / 32) * (D_B / D_A), ((size_t) M + 511) / 512, (size_t) D_A};
|
||||
size_t lws[3] = {64, 1, 1};
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
} else {
|
||||
cl_int err;
|
||||
|
||||
const size_t n_tiles = (N + 31) / 32;
|
||||
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
|
||||
cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
|
||||
cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
|
||||
size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
|
||||
cl_image_format fmt = {CL_RGBA, CL_FLOAT};
|
||||
cl_image_desc desc = {};
|
||||
desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
|
||||
desc.image_width = region.size / 16;
|
||||
desc.buffer = k_sub;
|
||||
cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
CL_CHECK(clReleaseMemObject(k_img));
|
||||
CL_CHECK(clReleaseMemObject(k_sub));
|
||||
CL_CHECK(clReleaseMemObject(q_sub));
|
||||
region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
|
||||
region.size = (size_t) K * N * D_B * sizeof(float);
|
||||
cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
|
||||
|
||||
const size_t n_tiles = (N + 31) / 32;
|
||||
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
|
||||
|
||||
const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
|
||||
size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
|
||||
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
CL_CHECK(clReleaseMemObject(k_img));
|
||||
CL_CHECK(clReleaseMemObject(k_sub));
|
||||
CL_CHECK(clReleaseMemObject(q_sub));
|
||||
}
|
||||
|
||||
cl_mem sinks_buf = extra_q->data_device;
|
||||
cl_ulong sinks_off = 0;
|
||||
@@ -23261,7 +23402,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
|
||||
ggml_backend_t backend,
|
||||
const ggml_tensor * q, const ggml_tensor * k,
|
||||
ggml_tensor * dst,
|
||||
int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct
|
||||
int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct, bool kq_int8
|
||||
) {
|
||||
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
|
||||
const ggml_tensor * v = dst->src[2];
|
||||
@@ -23277,8 +23418,31 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
|
||||
const int64_t kqv_m = pad_up(dv, 64);
|
||||
const int64_t tile_n = pad_up(n_q_chunk, 32);
|
||||
|
||||
// Pad the int8 row pitch by one cache line, as in the P8 KQV kernel.
|
||||
const int kv_pitch = (int) n_kv + 64;
|
||||
static const int64_t kv_pitch_round = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND");
|
||||
const int64_t v = (e && e[0]) ? strtoll(e, nullptr, 10) : 32768;
|
||||
return v > 0 && v <= INT_MAX && (v & (v - 1)) == 0 ? v : (int64_t) 0;
|
||||
}();
|
||||
static const int64_t kv_pitch_round_min = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND_MIN");
|
||||
return (e && e[0]) ? strtoll(e, nullptr, 10) : (int64_t) 16384;
|
||||
}();
|
||||
static const int64_t kv_pitch_round_max_dv = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KV_PITCH_ROUND_MAX_DV");
|
||||
return (e && e[0]) ? strtoll(e, nullptr, 10) : (int64_t) 256;
|
||||
}();
|
||||
|
||||
// Round long P and V^T rows to reduce cache conflicts, then add one cache line.
|
||||
int64_t kv_pitch_value = n_kv + 64;
|
||||
if (kv_pitch_round > 0 && n_kv > kv_pitch_round_min && dv <= kv_pitch_round_max_dv) {
|
||||
const int64_t rounded_pitch = pad_up(n_kv, kv_pitch_round) + 64;
|
||||
if (rounded_pitch <= INT_MAX &&
|
||||
(size_t) rounded_pitch <= SIZE_MAX / n_q_chunk / n_head &&
|
||||
(size_t) rounded_pitch <= SIZE_MAX / dv / n_head_kv) {
|
||||
kv_pitch_value = rounded_pitch;
|
||||
}
|
||||
}
|
||||
const int kv_pitch = (int) kv_pitch_value;
|
||||
const size_t nblk = (size_t) kv_pitch / 32;
|
||||
|
||||
backend_ctx->prealloc_fa_pq.allocate(backend_ctx->context, (size_t) kv_pitch * n_q_chunk * n_head);
|
||||
@@ -23322,13 +23486,17 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
}
|
||||
|
||||
if (kq_int8) {
|
||||
ggml_cl_fa_kq_int8_prepare(backend, dst, n_q_chunk);
|
||||
}
|
||||
|
||||
ggml_tensor_extra_cl extra_kqv;
|
||||
ggml_tensor kqv;
|
||||
|
||||
for (int64_t q0 = 0; q0 < n_q; q0 += n_q_chunk) {
|
||||
const int64_t nqc = MIN(n_q_chunk, n_q - q0);
|
||||
|
||||
ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch);
|
||||
ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch, kq_int8);
|
||||
|
||||
ggml_cl_fa_decompose_scratch_tensor(kqv, extra_kqv, backend_ctx->prealloc_fa_kqv.buffer, GGML_TYPE_F32, dv, nqc, n_head);
|
||||
|
||||
@@ -23385,6 +23553,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
|
||||
// head size. GGML_OPENCL_FA_DECOMPOSE_MB sets the KQ scratch budget (default 256).
|
||||
// GGML_OPENCL_FA_KQ_P8=0 disables U8 probabilities and int8 KQV.
|
||||
// GGML_OPENCL_FA_KQV_DIRECT=0 disables direct PV for small query batches.
|
||||
// GGML_OPENCL_FA_KQ_INT8=0 keeps F16 KQ; =1 enables int8 KQ for supported shapes.
|
||||
static bool ggml_cl_flash_attn_decompose_dispatch(
|
||||
ggml_backend_t backend,
|
||||
const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst
|
||||
@@ -23514,12 +23683,46 @@ static bool ggml_cl_flash_attn_decompose_dispatch(
|
||||
return false;
|
||||
}
|
||||
|
||||
static const int kq_int8_env = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KQ_INT8");
|
||||
return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
|
||||
}();
|
||||
static const int dk512_env = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KQ_DK512_F16");
|
||||
return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
|
||||
}();
|
||||
const bool dk512 = dk > 256 && dk <= 512 && n_q >= 32 &&
|
||||
(dk512_env == 1 || (dk512_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E));
|
||||
const bool kq_int8_possible = kq_p8 && (dk <= 256 || dk512) && dk % 32 == 0 &&
|
||||
backend_ctx->kernel_fa_q8_rows_f16 != nullptr && backend_ctx->kernel_fa_q8_rows_f32 != nullptr &&
|
||||
n_head <= INT_MAX && n_head_kv <= INT_MAX &&
|
||||
n_q <= INT_MAX / n_head / (dk / 32) && n_kv <= INT_MAX / n_head_kv / (dk / 32) &&
|
||||
(dst->src[4] == nullptr || dk > 64) &&
|
||||
(kq_int8_env == 1 || (kq_int8_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && n_kv >= 2048)) &&
|
||||
backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f16) >= 64 &&
|
||||
backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f32) >= 64;
|
||||
cl_kernel kq_int8_kernel = nullptr;
|
||||
if (kq_int8_possible) {
|
||||
kq_int8_kernel = dk <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
|
||||
}
|
||||
const bool kq_int8 = kq_int8_kernel != nullptr && backend_ctx->get_kernel_workgroup_size(kq_int8_kernel) >= 64;
|
||||
|
||||
// Fewer query tiles per KV block reduce cache pressure in the int8 KQ kernel.
|
||||
static const int64_t kq_int8_max_chunk = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KQ_INT8_MAX_CHUNK");
|
||||
const int v = (e && e[0]) ? atoi(e) : 96;
|
||||
return (int64_t) (v >= 32 ? (v / 32) * 32 : 96);
|
||||
}();
|
||||
if (kq_int8) {
|
||||
n_q_chunk = MIN(n_q_chunk, kq_int8_max_chunk);
|
||||
}
|
||||
|
||||
// even out the chunks, then align to the 32-wide n-tile
|
||||
const int64_t n_chunks = (n_q + n_q_chunk - 1) / n_q_chunk;
|
||||
n_q_chunk = MIN(n_q_chunk, ((n_q + n_chunks - 1) / n_chunks + 31) & ~(int64_t) 31);
|
||||
|
||||
if (kq_p8) {
|
||||
ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct);
|
||||
ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct, kq_int8);
|
||||
} else {
|
||||
ggml_cl_flash_attn_decompose(backend, q, k, dst, n_q_chunk, scale, max_bias);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
|
||||
// Store 32-element blocks as separate int8 values and half scales.
|
||||
|
||||
kernel void kernel_fa_q8_rows_f16(
|
||||
global const char * src, ulong off_src, ulong nb1, ulong nb2,
|
||||
global char * dq, global half * dd,
|
||||
int ne0, int nrow, int nhead
|
||||
) {
|
||||
const int nblk = ne0 / 32;
|
||||
const int gid = get_global_id(0);
|
||||
if (gid >= nblk*nrow*nhead) {
|
||||
return;
|
||||
}
|
||||
const int b = gid % nblk;
|
||||
const int r = (gid / nblk) % nrow;
|
||||
const int h = gid / (nblk*nrow);
|
||||
|
||||
global const half * p = (global const half *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
|
||||
float v[32];
|
||||
float amax = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
const float4 x = convert_float4(vload4(i, p));
|
||||
v[i*4+0] = x.s0;
|
||||
v[i*4+1] = x.s1;
|
||||
v[i*4+2] = x.s2;
|
||||
v[i*4+3] = x.s3;
|
||||
amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
|
||||
}
|
||||
const float d = amax / 127.0f;
|
||||
const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
|
||||
const size_t row = (size_t)h*nrow + r;
|
||||
dd[row*nblk + b] = (half)d;
|
||||
global char * q = dq + row*ne0 + b*32;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_fa_q8_rows_f32(
|
||||
global const char * src, ulong off_src, ulong nb1, ulong nb2,
|
||||
global char * dq, global half * dd,
|
||||
int ne0, int nrow, int nhead
|
||||
) {
|
||||
const int nblk = ne0 / 32;
|
||||
const int gid = get_global_id(0);
|
||||
if (gid >= nblk*nrow*nhead) {
|
||||
return;
|
||||
}
|
||||
const int b = gid % nblk;
|
||||
const int r = (gid / nblk) % nrow;
|
||||
const int h = gid / (nblk*nrow);
|
||||
|
||||
global const float * p = (global const float *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
|
||||
float v[32];
|
||||
float amax = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
const float4 x = vload4(i, p);
|
||||
v[i*4+0] = x.s0;
|
||||
v[i*4+1] = x.s1;
|
||||
v[i*4+2] = x.s2;
|
||||
v[i*4+3] = x.s3;
|
||||
amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
|
||||
}
|
||||
const float d = amax / 127.0f;
|
||||
const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
|
||||
const size_t row = (size_t)h*nrow + r;
|
||||
dd[row*nblk + b] = (half)d;
|
||||
global char * q = dq + row*ne0 + b*32;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
|
||||
#ifdef cl_khr_integer_dot_product
|
||||
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
|
||||
#endif
|
||||
|
||||
#define KQ_TN 32
|
||||
#define KQ_MB 8
|
||||
#define KQ_TM 64
|
||||
#define KQ_WG 64
|
||||
#define KQ_DK_MAX 256
|
||||
|
||||
__attribute__((qcom_wave_pair_mode(1)))
|
||||
kernel void kernel_mul_mm_q8_kq_p8(
|
||||
global const uint * kq, // K int8, [head_kv][n_kv][dk] as packed uints
|
||||
global const half * kd, // K scales, [head_kv][n_kv][dk/32]
|
||||
global const uint * qq, // Q int8, [head][n_q][dk] as packed uints
|
||||
global const half * qd, // Q scales, [head][n_q][dk/32]
|
||||
global uchar * qp, // u8 P, [head][n_q][p_pitch]
|
||||
global float * bmax, // per 32-row block max, [head][n_q][p_pitch/32]
|
||||
global float * bsum, // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
|
||||
global const char * mask, // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
|
||||
ulong offset_mask,
|
||||
ulong mask_nb1,
|
||||
float scale,
|
||||
float max_bias,
|
||||
float m0,
|
||||
float m1,
|
||||
int n_head_log2,
|
||||
int dk,
|
||||
int n_kv,
|
||||
int n_q,
|
||||
int n_head,
|
||||
int n_head_kv,
|
||||
int p_pitch // bytes per P row, >= n_kv, multiple of 32
|
||||
) {
|
||||
const int lid = get_local_id(0);
|
||||
const int gsz = n_head / n_head_kv;
|
||||
const int g0 = get_group_id(0);
|
||||
const int head_kv = get_group_id(2);
|
||||
const int head = head_kv*gsz + (g0 % gsz);
|
||||
const int qn0 = (g0 / gsz) * KQ_TN;
|
||||
const int kv0 = get_group_id(1)*KQ_TM*KQ_MB + lid;
|
||||
|
||||
const int nblk = dk / 32;
|
||||
const int nu = dk / 4;
|
||||
|
||||
__local uint sh_q [KQ_TN][KQ_DK_MAX/4];
|
||||
__local half sh_qd[KQ_TN][KQ_DK_MAX/32];
|
||||
__local float p8_lds[1024];
|
||||
|
||||
const size_t qbase = (size_t)head*n_q;
|
||||
for (int i = lid; i < KQ_TN*(KQ_DK_MAX/16); i += KQ_WG) {
|
||||
const int t = i / (KQ_DK_MAX/16);
|
||||
const int u = (i % (KQ_DK_MAX/16)) * 4;
|
||||
const int qi = qn0 + t;
|
||||
uint4 v = (uint4)(0u);
|
||||
if (qi < n_q && u < nu) {
|
||||
v = vload4(0, &qq[(qbase + qi)*nu + u]);
|
||||
}
|
||||
vstore4(v, 0, &sh_q[t][u]);
|
||||
}
|
||||
for (int i = lid; i < KQ_TN*(KQ_DK_MAX/32); i += KQ_WG) {
|
||||
const int t = i / (KQ_DK_MAX/32);
|
||||
const int b = i % (KQ_DK_MAX/32);
|
||||
const int qi = qn0 + t;
|
||||
sh_qd[t][b] = (qi < n_q && b < nblk) ? qd[(qbase + qi)*nblk + b] : (half)0.0f;
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
|
||||
float slope = 1.0f;
|
||||
if (max_bias > 0.0f) {
|
||||
const float base = head < n_head_log2 ? m0 : m1;
|
||||
const int ex = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
|
||||
slope = pow(base, ex);
|
||||
}
|
||||
global const half * mrow = (global const half *)(mask + offset_mask);
|
||||
const uint mstride = (uint)(mask_nb1 / 2);
|
||||
const int pblk = p_pitch / 32;
|
||||
|
||||
for (int m = 0; m < KQ_MB; ++m) {
|
||||
const int kv = kv0 + m*KQ_TM;
|
||||
if (kv - lid >= n_kv) {
|
||||
break; // uniform across the workgroup
|
||||
}
|
||||
|
||||
float acc[KQ_TN];
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
acc[t] = 0.0f;
|
||||
}
|
||||
|
||||
const int kl = min(kv, n_kv - 1);
|
||||
const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
|
||||
const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
|
||||
|
||||
for (int b = 0; b < nblk; ++b) {
|
||||
const uint4 w0 = vload4(0, &kq[kbase + b*8]);
|
||||
const uint4 w1 = vload4(0, &kq[kbase + b*8 + 4]);
|
||||
const float dks = (float)kd[kdbas + b];
|
||||
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
const uint4 a0 = vload4(0, &sh_q[t][b*8]);
|
||||
const uint4 a1 = vload4(0, &sh_q[t][b*8 + 4]);
|
||||
int raw = 0;
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
|
||||
acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
|
||||
}
|
||||
}
|
||||
|
||||
// Each lane owns one complete 32-row softmax block after the transpose.
|
||||
const int kvblk0 = kv - lid;
|
||||
#pragma unroll
|
||||
for (int half_ = 0; half_ < 2; ++half_) {
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 16; ++j) {
|
||||
p8_lds[j*64 + lid] = acc[half_*16 + j];
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
if (lid < 32) {
|
||||
const int j = lid >> 1;
|
||||
const int b = lid & 1;
|
||||
const int n = qn0 + half_*16 + j;
|
||||
if (n < n_q) {
|
||||
const int kvb = kvblk0 + b*32;
|
||||
const __local float * src = p8_lds + j*64 + b*32;
|
||||
float16 s0 = vload16(0, src);
|
||||
float16 s1 = vload16(1, src);
|
||||
const global half * mp = mrow + (size_t)n*mstride + kvb;
|
||||
const float16 m0v = convert_float16(vload16(0, mp));
|
||||
const float16 m1v = convert_float16(vload16(1, mp));
|
||||
s0 = s0*scale + slope*m0v;
|
||||
s1 = s1*scale + slope*m1v;
|
||||
const float16 mx16 = fmax(s0, s1);
|
||||
const float8 mx8 = fmax(mx16.lo, mx16.hi);
|
||||
const float4 mx4 = fmax(mx8.lo, mx8.hi);
|
||||
const float2 mx2 = fmax(mx4.lo, mx4.hi);
|
||||
const float amax = fmax(mx2.x, mx2.y);
|
||||
const size_t prow = qbase + n;
|
||||
const int blk = kvb >> 5;
|
||||
float sum = 0.0f;
|
||||
uchar16 q0 = (uchar16)(0);
|
||||
uchar16 q1 = (uchar16)(0);
|
||||
// finite sentinel: the program builds with -cl-finite-math-only
|
||||
if (amax > -1.0e30f) {
|
||||
const float16 e0 = exp(s0 - amax);
|
||||
const float16 e1 = exp(s1 - amax);
|
||||
const float16 a16 = e0 + e1;
|
||||
const float8 a8 = a16.lo + a16.hi;
|
||||
const float4 a4 = a8.lo + a8.hi;
|
||||
const float2 a2 = a4.lo + a4.hi;
|
||||
sum = a2.x + a2.y;
|
||||
q0 = convert_uchar16_sat_rte(e0*255.0f);
|
||||
q1 = convert_uchar16_sat_rte(e1*255.0f);
|
||||
}
|
||||
global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
|
||||
vstore16(q0, 0, qdst);
|
||||
vstore16(q1, 1, qdst);
|
||||
bmax[prow*pblk + blk] = amax;
|
||||
bsum[prow*pblk + blk] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
|
||||
#ifdef cl_khr_integer_dot_product
|
||||
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
|
||||
#endif
|
||||
|
||||
#define KQ_TN 32
|
||||
#define KQ_MB 8
|
||||
#define KQ_TM 64
|
||||
#define KQ_WG 64
|
||||
#define KQ_DKC 256
|
||||
|
||||
__attribute__((qcom_wave_pair_mode(1)))
|
||||
kernel void kernel_mul_mm_q8_kq_p8_dk512(
|
||||
global const uint * kq, // K int8, [head_kv][n_kv][dk] as packed uints
|
||||
global const half * kd, // K scales, [head_kv][n_kv][dk/32]
|
||||
global const uint * qq, // Q int8, [head][n_q][dk] as packed uints
|
||||
global const half * qd, // Q scales, [head][n_q][dk/32]
|
||||
global uchar * qp, // u8 P, [head][n_q][p_pitch]
|
||||
global float * bmax, // per 32-row block max, [head][n_q][p_pitch/32]
|
||||
global float * bsum, // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
|
||||
global const char * mask, // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
|
||||
ulong offset_mask,
|
||||
ulong mask_nb1,
|
||||
float scale,
|
||||
float max_bias,
|
||||
float m0,
|
||||
float m1,
|
||||
int n_head_log2,
|
||||
int dk,
|
||||
int n_kv,
|
||||
int n_q,
|
||||
int n_head,
|
||||
int n_head_kv,
|
||||
int p_pitch // bytes per P row, >= n_kv, multiple of 32
|
||||
) {
|
||||
const int lid = get_local_id(0);
|
||||
const int gsz = n_head / n_head_kv;
|
||||
const int g0 = get_group_id(0);
|
||||
const int head_kv = get_group_id(2);
|
||||
const int head = head_kv*gsz + (g0 % gsz);
|
||||
const int qn0 = (g0 / gsz) * KQ_TN;
|
||||
const int kv0 = get_group_id(1)*KQ_TM*KQ_MB + lid;
|
||||
|
||||
const int nblk = dk / 32;
|
||||
const int nu = dk / 4;
|
||||
|
||||
// Reuse the 256-wide Q slice for the F32 softmax scores after the contraction.
|
||||
__local uint sh_q [KQ_TN*KQ_DKC/4];
|
||||
__local half sh_qd[KQ_TN][KQ_DKC/32];
|
||||
__local float * p8_lds = (__local float *)sh_q;
|
||||
|
||||
const size_t qbase = (size_t)head*n_q;
|
||||
|
||||
float slope = 1.0f;
|
||||
if (max_bias > 0.0f) {
|
||||
const float base = head < n_head_log2 ? m0 : m1;
|
||||
const int ex = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
|
||||
slope = pow(base, ex);
|
||||
}
|
||||
global const half * mrow = (global const half *)(mask + offset_mask);
|
||||
const uint mstride = (uint)(mask_nb1 / 2);
|
||||
const int pblk = p_pitch / 32;
|
||||
|
||||
for (int m = 0; m < KQ_MB; ++m) {
|
||||
const int kv = kv0 + m*KQ_TM;
|
||||
if (kv - lid >= n_kv) {
|
||||
break; // uniform across the workgroup
|
||||
}
|
||||
|
||||
float acc[KQ_TN];
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
acc[t] = 0.0f;
|
||||
}
|
||||
|
||||
const int kl = min(kv, n_kv - 1);
|
||||
const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
|
||||
const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
|
||||
|
||||
for (int c0 = 0; c0 < nblk; c0 += KQ_DKC/32) {
|
||||
const int cb = min(KQ_DKC/32, nblk - c0);
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
for (int i = lid; i < KQ_TN*(KQ_DKC/16); i += KQ_WG) {
|
||||
const int t = i / (KQ_DKC/16);
|
||||
const int u = (i % (KQ_DKC/16)) * 4;
|
||||
const int qi = qn0 + t;
|
||||
uint4 v = (uint4)(0u);
|
||||
if (qi < n_q && u < cb*8) {
|
||||
v = vload4(0, &qq[(qbase + qi)*nu + c0*8 + u]);
|
||||
}
|
||||
vstore4(v, 0, &sh_q[t*(KQ_DKC/4) + u]);
|
||||
}
|
||||
for (int i = lid; i < KQ_TN*(KQ_DKC/32); i += KQ_WG) {
|
||||
const int t = i / (KQ_DKC/32);
|
||||
const int b = i % (KQ_DKC/32);
|
||||
const int qi = qn0 + t;
|
||||
sh_qd[t][b] = (qi < n_q && b < cb) ? qd[(qbase + qi)*nblk + c0 + b] : (half)0.0f;
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
|
||||
for (int b = 0; b < cb; ++b) {
|
||||
const uint4 w0 = vload4(0, &kq[kbase + (c0 + b)*8]);
|
||||
const uint4 w1 = vload4(0, &kq[kbase + (c0 + b)*8 + 4]);
|
||||
const float dks = (float)kd[kdbas + c0 + b];
|
||||
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
const uint4 a0 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8]);
|
||||
const uint4 a1 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8 + 4]);
|
||||
int raw = 0;
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
|
||||
acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Finish reading the Q slice before writing scores to the same local memory.
|
||||
const int kvblk0 = kv - lid;
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
p8_lds[t*64 + lid] = acc[t];
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
{
|
||||
const int j = lid >> 1;
|
||||
const int b = lid & 1;
|
||||
const int n = qn0 + j;
|
||||
if (n < n_q) {
|
||||
const int kvb = kvblk0 + b*32;
|
||||
__local float * src = p8_lds + j*64 + b*32;
|
||||
const global half * mp = mrow + (size_t)n*mstride + kvb;
|
||||
float4 s = vload4(0, src)*scale + slope*convert_float4(vload4(0, mp));
|
||||
vstore4(s, 0, src);
|
||||
float amax = fmax(fmax(s.x, s.y), fmax(s.z, s.w));
|
||||
for (int c = 1; c < 8; ++c) {
|
||||
s = vload4(c, src)*scale + slope*convert_float4(vload4(c, mp));
|
||||
vstore4(s, c, src);
|
||||
amax = fmax(amax, fmax(fmax(s.x, s.y), fmax(s.z, s.w)));
|
||||
}
|
||||
const size_t prow = qbase + n;
|
||||
const int blk = kvb >> 5;
|
||||
global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
|
||||
float sum = 0.0f;
|
||||
// finite sentinel: the program builds with -cl-finite-math-only
|
||||
if (amax > -1.0e30f) {
|
||||
for (int c = 0; c < 8; ++c) {
|
||||
const float4 e = exp(vload4(c, src) - amax);
|
||||
sum += (e.x + e.y) + (e.z + e.w);
|
||||
vstore4(convert_uchar4_sat_rte(e*255.0f), c, qdst);
|
||||
}
|
||||
} else {
|
||||
for (int c = 0; c < 8; ++c) {
|
||||
vstore4((uchar4)(0), c, qdst);
|
||||
}
|
||||
}
|
||||
bmax[prow*pblk + blk] = amax;
|
||||
bsum[prow*pblk + blk] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -107,7 +107,8 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, std::map<std::string, std::sh
|
||||
m_naive = true;
|
||||
for (int i = 0; i < cgraph->n_nodes; ++i) {
|
||||
if ((cgraph->nodes[i]->op == GGML_OP_GATED_DELTA_NET && cgraph->nodes[i]->op_params[0] > 1) ||
|
||||
is_sampling_view(cgraph->nodes[i]) || is_conv_state_snapshot_writeback(cgraph->nodes[i])) {
|
||||
is_sampling_view(cgraph->nodes[i]) || is_conv_state_snapshot_writeback(cgraph->nodes[i]) ||
|
||||
is_ssm_state_writeback(cgraph->nodes[i])) {
|
||||
std::tie(m_model_params, m_compute_params) = compute_llm_params(cgraph, false);
|
||||
add_extra_inputs();
|
||||
break;
|
||||
@@ -494,6 +495,10 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
|
||||
}
|
||||
} else if (node->ne[0] == 1 && src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) {
|
||||
op_case = 3;
|
||||
} else if (node->ne[0] * node->ne[1] == src->ne[0] * src->ne[1] && node->ne[2] == src->ne[2] &&
|
||||
node->ne[3] == src->ne[3]) {
|
||||
// head regroup, e.g. mamba2's [head_dim, n_head, n_tokens, n_seqs] -> [d_inner/n_group, n_group, n_tokens, n_seqs]
|
||||
op_case = 1;
|
||||
} else if (name.find("linear_attn_qkv_mixed") == 0 || name.find("alpha") == 0) {
|
||||
op_case = 6;
|
||||
} else if (name.find("linear_attn_out") == 0) {
|
||||
@@ -597,6 +602,10 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
|
||||
op_case = 3;
|
||||
break;
|
||||
}
|
||||
if (is_column_set_rows(node)) {
|
||||
op_case = 4;
|
||||
break;
|
||||
}
|
||||
// `-fa off` writes the V cache transposed, through a [1, n] reshape of the cache: one element per row
|
||||
if (node->src[2] != nullptr && node->src[2]->op == GGML_OP_RESHAPE && node->src[2]->ne[0] == 1 &&
|
||||
is_kvcache(node->src[2]->src[0], nullptr)) {
|
||||
@@ -797,6 +806,9 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
|
||||
} else {
|
||||
op_case = 1;
|
||||
}
|
||||
} else if (is_ssm_state_writeback(node)) {
|
||||
op_case = 13;
|
||||
break;
|
||||
} else if (GgmlOvDecoder::is_conv_state_writeback(node)) {
|
||||
op_case = 2;
|
||||
break;
|
||||
@@ -1296,6 +1308,11 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
|
||||
writeback.snapshot_stride = destination->nb[2] / destination->nb[1];
|
||||
compute_params.rs_writebacks[get_tensor_ov_name(cgraph, node)] = writeback;
|
||||
}
|
||||
if (is_ssm_state_writeback(node)) {
|
||||
ComputeParams::RsWriteback writeback;
|
||||
writeback.slot_begin = (int64_t) (node->src[1]->view_offs / node->src[1]->nb[1]);
|
||||
compute_params.rs_writebacks[get_tensor_ov_name(cgraph, node)] = writeback;
|
||||
}
|
||||
if (is_conv_state_snapshot_writeback(node)) {
|
||||
const auto source = node->src[0];
|
||||
const auto dest = node->src[2];
|
||||
@@ -1406,6 +1423,14 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
|
||||
compute_params.attention_size =
|
||||
ggml_openvino_static_kv_window(compute_params.attention_size, model_params.ctx_per_seq);
|
||||
}
|
||||
// The GPU plugin keeps device memory for each KV length it sees, and llama.cpp grows the KV length in steps of
|
||||
// 256. Round it up to a power of two the same way, so that a long context uses a few lengths and not one per
|
||||
// step. The KQ mask input is padded to this length with -inf (convert_ggml_input_to_ov).
|
||||
if (!is_static && ggml_openvino_is_gpu() && !ggml_openvino_getenv_int("GGML_OPENVINO_STATEFUL_EXECUTION") &&
|
||||
!compute_params.mask_name.empty() && compute_params.attention_size > 0 && model_params.ctx_per_seq > 0) {
|
||||
compute_params.attention_size =
|
||||
ggml_openvino_static_kv_window(compute_params.attention_size, model_params.ctx_per_seq);
|
||||
}
|
||||
model_params.ctx = model_params.ctx_per_seq * model_params.n_seq;
|
||||
return {model_params, compute_params};
|
||||
}
|
||||
@@ -1416,6 +1441,15 @@ void GgmlOvDecoder::validate_cgraph() const {
|
||||
}
|
||||
}
|
||||
|
||||
bool GgmlOvDecoder::has_inp_embd_leaf_use(const ggml_tensor * tensor) const {
|
||||
for (int i = 0; i < m_cgraph->n_nodes; i++) {
|
||||
if (is_inp_embd_leaf(tensor, m_cgraph->nodes[i])) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
|
||||
const ggml_tensor * input,
|
||||
int dynamic_dim_index) const {
|
||||
@@ -1530,7 +1564,7 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op,
|
||||
// index list has a static length; on CPU/GPU it may change across compiles (defrag).
|
||||
input_shape = m_is_static ? ov::PartialShape{get_shape(input)} : ov::PartialShape{1, 1, 1, -1};
|
||||
|
||||
} else if (input != nullptr && op != nullptr && is_inp_emb(input, op)) {
|
||||
} else if (input != nullptr && op != nullptr && (is_inp_emb(input, op) || has_inp_embd_leaf_use(input))) {
|
||||
input_shape = ov::PartialShape{get_shape(input)};
|
||||
if (!m_is_static) {
|
||||
input_shape[2] = -1;
|
||||
@@ -1914,7 +1948,11 @@ void GgmlOvDecoder::compute_model_outputs() {
|
||||
continue;
|
||||
}
|
||||
const bool inactive_top_k = is_sampling_top_k(cur_node) && !(cur_node->flags & GGML_TENSOR_FLAG_COMPUTE);
|
||||
if ((cur_node->flags & GGML_TENSOR_FLAG_OUTPUT) || inactive_top_k) {
|
||||
auto cur_node_use_count = m_cgraph->use_counts[ggml_hash_find(&m_cgraph->visited_hash_set, cur_node)];
|
||||
// llama flags result_norm in every graph but reads it only with embeddings on, when it has no consumer
|
||||
// and is kept by the rule below.
|
||||
const bool unread_embd = strcmp(cur_node->name, "result_norm") == 0 && cur_node_use_count > 0;
|
||||
if (((cur_node->flags & GGML_TENSOR_FLAG_OUTPUT) && !unread_embd) || inactive_top_k) {
|
||||
const auto name = get_tensor_ov_name(m_cgraph, cur_node);
|
||||
m_model_outputs[name] = cur_node;
|
||||
m_model_output_names.insert(name);
|
||||
@@ -1925,7 +1963,12 @@ void GgmlOvDecoder::compute_model_outputs() {
|
||||
m_model_outputs[name] = cache;
|
||||
m_model_output_names.insert(name);
|
||||
}
|
||||
auto cur_node_use_count = m_cgraph->use_counts[ggml_hash_find(&m_cgraph->visited_hash_set, cur_node)];
|
||||
if (is_ssm_state_writeback(cur_node)) {
|
||||
auto * cache = cur_node->view_src;
|
||||
const auto name = get_tensor_ov_name(m_cgraph, cache);
|
||||
m_model_outputs[name] = cache;
|
||||
m_model_output_names.insert(name);
|
||||
}
|
||||
if (cur_node_use_count == 0) {
|
||||
// The output of in-place ops is the view_src tensor, which is updated in place. We should use the view_src name as the output name to make sure it can be correctly matched with the later ops that use the view_src.
|
||||
if (cur_node != nullptr && ::is_inplace_op(cur_node) &&
|
||||
@@ -2641,6 +2684,10 @@ bool GgmlOvDecoder::is_direct_gdn_view(const ggml_tensor * node) const {
|
||||
|
||||
std::vector<std::string> GgmlOvDecoder::get_output_names(int node_idx) const {
|
||||
auto * node = m_node_info_list[node_idx].node;
|
||||
if (is_ssm_state_writeback(node)) {
|
||||
// the written state block plus the full updated cache (see translate_cpy op_case 13)
|
||||
return {m_node_info_list[node_idx].node_name, get_tensor_ov_name(m_cgraph, node->view_src)};
|
||||
}
|
||||
if (node->op == GGML_OP_GATED_DELTA_NET && (!m_model_params.has_rs_rollback || node->op_params[0] > 1)) {
|
||||
std::string attn_name;
|
||||
std::string state_name;
|
||||
@@ -2669,6 +2716,10 @@ std::vector<std::string> GgmlOvDecoder::get_output_names(int node_idx) const {
|
||||
|
||||
std::string GgmlOvDecoder::get_inplace_op_src(int node_idx) const {
|
||||
auto * node = m_node_info_list[node_idx].node;
|
||||
if (is_ssm_state_writeback(node)) {
|
||||
// translate_cpy publishes the updated cache as a second named output
|
||||
return "";
|
||||
}
|
||||
// Reset lengths (also in static mode) and dynamic remainder views can become nonempty
|
||||
// after compilation. Keep their alias updates in the translated cache chain.
|
||||
const bool runtime_recurrent_write = !m_naive && is_recurrent_cache(node->view_src) &&
|
||||
@@ -2786,7 +2837,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
|
||||
m_node_dynamic_dims[src] = 0;
|
||||
continue;
|
||||
}
|
||||
if (is_inp_scale_rows(src, node)) {
|
||||
if (is_inp_scale_rows(src, node) || is_inp_embd_leaf(src, node)) {
|
||||
m_node_dynamic_dims[src] = 1;
|
||||
continue;
|
||||
}
|
||||
@@ -2865,6 +2916,11 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
|
||||
m_node_dynamic_dims[node] = 2;
|
||||
break;
|
||||
}
|
||||
if (node->src[0]->op == GGML_OP_SSM_SCAN) {
|
||||
// y part [head_dim, n_head, n_tokens, n_seqs] at offset 0; the state part is token-independent
|
||||
m_node_dynamic_dims[node] = node->view_offs == 0 ? 2 : -1;
|
||||
break;
|
||||
}
|
||||
// Use stride-based matching: the stride of a VIEW dimension directly
|
||||
// encodes which source dimension it indexes into, so it uniquely
|
||||
// identifies the dynamic dim even when two dims share the same size.
|
||||
@@ -3030,6 +3086,10 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
|
||||
case GGML_OP_GATED_DELTA_NET:
|
||||
m_node_dynamic_dims[node] = 1;
|
||||
break;
|
||||
case GGML_OP_SSM_SCAN:
|
||||
// packed 1-D [y | state]; the y part grows with tokens
|
||||
m_node_dynamic_dims[node] = 0;
|
||||
break;
|
||||
case GGML_OP_RMS_NORM:
|
||||
case GGML_OP_L2_NORM:
|
||||
case GGML_OP_NORM:
|
||||
@@ -3080,12 +3140,15 @@ void GgmlOvDecoder::compute_node_dynamic_dims() {
|
||||
case GGML_OP_CPY:
|
||||
case GGML_OP_SUM:
|
||||
case GGML_OP_MEAN:
|
||||
case GGML_OP_ARANGE:
|
||||
m_node_dynamic_dims[node] = -1;
|
||||
break;
|
||||
case GGML_OP_SET_ROWS:
|
||||
// a cache write returns the cache; other rows writes keep the shape of their destination
|
||||
m_node_dynamic_dims[node] = (is_kvcache_write(node) || is_conv_state_snapshot_writeback(node)) ?
|
||||
-1 : m_node_dynamic_dims[node->src[2]];
|
||||
// a cache or column write returns its whole buffer; other rows writes keep the shape of their destination
|
||||
m_node_dynamic_dims[node] =
|
||||
(is_kvcache_write(node) || is_conv_state_snapshot_writeback(node) || is_column_set_rows(node)) ?
|
||||
-1 :
|
||||
m_node_dynamic_dims[node->src[2]];
|
||||
break;
|
||||
case GGML_OP_IM2COL: {
|
||||
m_node_dynamic_dims[node] = -1;
|
||||
|
||||
@@ -477,6 +477,21 @@ public:
|
||||
return tensor->op == GGML_OP_GET_ROWS && op->op == GGML_OP_RMS_NORM;
|
||||
}
|
||||
|
||||
// Raw float input [n_embd, n_tokens]: DFlash/DSpark/EAGLE3 features into fc, or raw embd batch / EAGLE3 g-embd into RMS_NORM
|
||||
// MUL_MAT needs a weight leaf as src[0], so inp_mean (src[0] is computed) does not match
|
||||
static bool is_inp_embd_leaf(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
if (tensor->op != GGML_OP_NONE || !(tensor->flags & GGML_TENSOR_FLAG_INPUT) || tensor->type != GGML_TYPE_F32) {
|
||||
return false;
|
||||
}
|
||||
if (op->op == GGML_OP_MUL_MAT) {
|
||||
return op->src[1] == tensor && op->src[0] != nullptr && op->src[0]->op == GGML_OP_NONE;
|
||||
}
|
||||
return op->op == GGML_OP_RMS_NORM && op->src[0] == tensor;
|
||||
}
|
||||
|
||||
// true if any node of the current graph uses tensor as a raw embd input (see is_inp_embd_leaf)
|
||||
bool has_inp_embd_leaf_use(const ggml_tensor * tensor) const;
|
||||
|
||||
static bool is_activation_input(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
if (tensor == nullptr || op == nullptr || tensor->type != GGML_TYPE_F32 || tensor->ne[2] != 1 ||
|
||||
tensor->ne[3] != 1 || (tensor->buffer != nullptr && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS)) {
|
||||
@@ -578,11 +593,11 @@ public:
|
||||
|
||||
bool is_direct_gdn_view(const ggml_tensor * node) const;
|
||||
|
||||
inline static bool is_cache(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
static bool is_cache(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
return is_recurrent_cache(tensor) || is_kvcache(tensor, op);
|
||||
}
|
||||
|
||||
inline static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) {
|
||||
if (tensor == nullptr || is_recurrent_cache(tensor)) {
|
||||
return false;
|
||||
}
|
||||
@@ -590,27 +605,68 @@ public:
|
||||
(op != nullptr && op->op == GGML_OP_SET_ROWS && op->src[2] == tensor);
|
||||
}
|
||||
|
||||
inline static bool is_conv_state_writeback(const ggml_tensor * node) {
|
||||
static bool is_conv_state_writeback(const ggml_tensor * node) {
|
||||
return node->op == GGML_OP_CPY && node->view_src != nullptr && is_recurrent_cache(node->view_src) &&
|
||||
node->src[0] != nullptr && node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr &&
|
||||
node->src[0]->src[0]->op == GGML_OP_CONCAT && node->src[1] != nullptr &&
|
||||
node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src;
|
||||
}
|
||||
|
||||
// The final-state writeback of SSM_SCAN (K == 1): the state tail of the packed [y | state]
|
||||
// output copied into a slot block of the recurrent cache.
|
||||
static bool is_ssm_state_writeback(const ggml_tensor * node) {
|
||||
if (!(node != nullptr && node->op == GGML_OP_CPY && node->view_src != nullptr && node->src[0] != nullptr &&
|
||||
node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr &&
|
||||
node->src[0]->src[0]->op == GGML_OP_SSM_SCAN && node->src[0]->src[0]->op_params[0] == 1 &&
|
||||
node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW &&
|
||||
node->src[1]->view_src == node->view_src)) {
|
||||
return false;
|
||||
}
|
||||
const auto * scan = node->src[0]->src[0];
|
||||
const auto * src = node->src[0];
|
||||
const auto * dst = node->src[1];
|
||||
if (node->type != GGML_TYPE_F32 || src->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 ||
|
||||
src->ne[2] != 1 || src->ne[3] != 1) {
|
||||
return false;
|
||||
}
|
||||
for (int i = 0; i < GGML_MAX_DIMS; ++i) {
|
||||
if (src->ne[i] != dst->ne[i]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return dst->nb[1] > 0 && dst->view_offs % dst->nb[1] == 0 &&
|
||||
src->view_offs == (size_t) ggml_nelements(scan->src[1]) * ggml_type_size(scan->type);
|
||||
}
|
||||
|
||||
// build_conv_state_store writes overlapping [history, window, channel, seq] windows into newest-first rollback planes.
|
||||
// Match this layout separately from general SET_ROWS.
|
||||
static bool is_conv_state_snapshot_writeback(const ggml_tensor * node);
|
||||
|
||||
// SET_ROWS into a column range of each row of a plain 2-D buffer, e.g. build_layer_inp_dev(): every enabled layer
|
||||
// input goes to view_2d(layer_inp_dev, n_embd, n_batch, nb1, col * n_embd * elem) at the batch rows of the ubatch.
|
||||
// The buffer sits in a regular device buffer, so is_kvcache() matches it: check this first
|
||||
static bool is_column_set_rows(const ggml_tensor * node) {
|
||||
if (node == nullptr || node->op != GGML_OP_SET_ROWS || node->src[2] == nullptr ||
|
||||
node->src[2]->op != GGML_OP_VIEW) {
|
||||
return false;
|
||||
}
|
||||
const ggml_tensor * dst = node->src[2];
|
||||
const ggml_tensor * buf = dst->view_src;
|
||||
return buf != nullptr && buf->op == GGML_OP_NONE && !is_recurrent_cache(buf) && buf->ne[2] == 1 &&
|
||||
buf->ne[3] == 1 && dst->ne[2] == 1 && dst->ne[3] == 1 && dst->ne[0] < buf->ne[0] &&
|
||||
dst->nb[0] == buf->nb[0] && dst->nb[1] == buf->nb[1] && dst->ne[1] == buf->ne[1];
|
||||
}
|
||||
|
||||
// SET_ROWS into a KV cache, directly or through the [1, n] reshape of the transposed V cache with `-fa off`.
|
||||
// Other SET_ROWS write rows of a compute tensor, e.g. the MoE expert-group mask
|
||||
inline static bool is_kvcache_write(const ggml_tensor * node) {
|
||||
static bool is_kvcache_write(const ggml_tensor * node) {
|
||||
const ggml_tensor * dst = node->src[2];
|
||||
return dst != nullptr && is_kvcache(dst->view_src != nullptr ? dst->view_src : dst, nullptr);
|
||||
}
|
||||
|
||||
// GET_ROWS batched over ggml dims 2 and 3, e.g. the MoE expert-group scores
|
||||
// ggml_get_rows(ggml_reshape_4d(groups, 1, n_exp_per_group, n_groups, n_tokens), top_groups)
|
||||
inline static bool is_get_rows_batched_4d(const ggml_tensor * node) {
|
||||
static bool is_get_rows_batched_4d(const ggml_tensor * node) {
|
||||
return node->op == GGML_OP_GET_ROWS && !ggml_is_quantized(node->src[0]->type) &&
|
||||
node->src[1]->ne[1] == node->src[0]->ne[2] && node->src[1]->ne[2] == node->src[0]->ne[3] &&
|
||||
node->src[1]->ne[3] == 1;
|
||||
|
||||
@@ -616,6 +616,10 @@ bool ggml_openvino_is_npu() {
|
||||
return ggml_openvino_get_device_config().is_npu;
|
||||
}
|
||||
|
||||
bool ggml_openvino_is_static() {
|
||||
return ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC");
|
||||
}
|
||||
|
||||
bool ggml_openvino_is_gpu() {
|
||||
return has_prefix(ggml_openvino_get_device_name(), "GPU");
|
||||
}
|
||||
|
||||
@@ -134,6 +134,9 @@ bool ggml_openvino_release_weights_enabled();
|
||||
// Check if running on NPU
|
||||
bool ggml_openvino_is_npu();
|
||||
|
||||
// Static graphs: NPU, or any device with GGML_OPENVINO_FORCE_STATIC
|
||||
bool ggml_openvino_is_static();
|
||||
|
||||
// KV-cache slicing: bind the KV cache, and size the KQ masks, at the attended length instead of the full context.
|
||||
// One mechanism, two modes:
|
||||
// dynamic graphs (CPU/GPU): the KV input is a view over the cache; the graph is not recompiled.
|
||||
|
||||
@@ -1504,6 +1504,38 @@ static bool has_strides_on_element_grid(const ggml_tensor * t) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// A VIEW that regroups one contiguous span of every row of its contiguous source, e.g. the z/x/B/C
|
||||
// splits of the fused mamba projections. The strides do not nest on an element grid, but
|
||||
// process_view_input_new resolves it as a Slice of the source's innermost axis plus a Reshape.
|
||||
// Keep in sync with the matching branch there; dynamic execution only.
|
||||
static bool is_row_span_view(const ggml_tensor * t) {
|
||||
if (ggml_openvino_is_static() || is_stateful_enabled()) {
|
||||
return false;
|
||||
}
|
||||
const ggml_tensor * src = t->op == GGML_OP_VIEW ? t->src[0] : nullptr;
|
||||
// a RESHAPE source is fine (the translation reads its value); a chained VIEW is not, because
|
||||
// the view metadata then describes the intermediate view instead of the resolved input
|
||||
if (src == nullptr || src->op == GGML_OP_VIEW || !ggml_is_contiguous(src) || t->type != src->type) {
|
||||
return false;
|
||||
}
|
||||
const size_t es = ggml_type_size(t->type);
|
||||
int d = 0;
|
||||
size_t span = 1;
|
||||
while (d < GGML_MAX_DIMS && t->nb[d] == span * es) {
|
||||
span *= t->ne[d];
|
||||
d++;
|
||||
}
|
||||
if (d < 2) {
|
||||
return false;
|
||||
}
|
||||
for (int s = 1; d < GGML_MAX_DIMS; ++d, ++s) {
|
||||
if (t->ne[d] != src->ne[s] || t->nb[d] != src->nb[s]) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return t->view_offs % es == 0 && t->view_offs / es + span <= (size_t) src->ne[0];
|
||||
}
|
||||
|
||||
static bool has_non_contiguous_view_input(const ggml_tensor * op) {
|
||||
for (int i = 0; i < GGML_MAX_SRC; i++) {
|
||||
if (op->src[i] == nullptr) {
|
||||
@@ -1624,7 +1656,7 @@ static bool cpy_output_view_is_supported(const ggml_tensor * op) {
|
||||
}
|
||||
|
||||
return ggml_nbytes(op) == 0 || ggml_is_contiguous(op) || GgmlOvDecoder::is_conv_state_writeback(op) ||
|
||||
GgmlOvDecoder::is_gdn_snapshot_writeback(op);
|
||||
GgmlOvDecoder::is_gdn_snapshot_writeback(op) || GgmlOvDecoder::is_ssm_state_writeback(op);
|
||||
}
|
||||
|
||||
static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) {
|
||||
@@ -1727,14 +1759,16 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
}
|
||||
case GGML_OP_GET_ROWS:
|
||||
case GGML_OP_SET_ROWS: {
|
||||
if (GgmlOvDecoder::is_conv_state_snapshot_writeback(op) &&
|
||||
(ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC"))) {
|
||||
if (GgmlOvDecoder::is_conv_state_snapshot_writeback(op) && ggml_openvino_is_static()) {
|
||||
return {false, "convolution snapshots require dynamic execution"};
|
||||
}
|
||||
if (op->ne[3] != 1 && !GgmlOvDecoder::is_get_rows_batched_4d(op) &&
|
||||
!GgmlOvDecoder::is_conv_state_snapshot_writeback(op)) {
|
||||
return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"};
|
||||
}
|
||||
if (op->op == GGML_OP_SET_ROWS && op->src[0]->type == GGML_TYPE_BF16) {
|
||||
return {false, "SET_ROWS with BF16 src0 is not supported"};
|
||||
}
|
||||
if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K ||
|
||||
op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_1)) {
|
||||
// These are all f16-arithmetic dequant rounding errors that intermittently exceed the
|
||||
@@ -2004,6 +2038,9 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
if (op->view_src != nullptr && !ggml_is_contiguous(op->src[0])) {
|
||||
return {false, "ROPE on VIEW / non-contiguous input is not supported"};
|
||||
}
|
||||
if (op->type == GGML_TYPE_BF16) {
|
||||
return {false, "ROPE with BF16 type is not supported"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_TRANSPOSE: {
|
||||
@@ -2034,7 +2071,7 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
reason = "GATED_DELTA_NET with kda (per-key-dimension gating) is not supported";
|
||||
}
|
||||
else if (op->op_params[0] > 1) {
|
||||
if (ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC")) {
|
||||
if (ggml_openvino_is_static()) {
|
||||
reason = "GATED_DELTA_NET snapshots require dynamic execution";
|
||||
}
|
||||
}
|
||||
@@ -2053,6 +2090,19 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
|
||||
// return true;
|
||||
break;
|
||||
}
|
||||
case GGML_OP_SSM_SCAN: {
|
||||
// SelectiveSSM implements the Mamba-2 recurrence: scalar decay per head, one final state
|
||||
if (op->src[3]->ne[0] != 1) {
|
||||
return {false, "SSM_SCAN with Mamba-1 state-wise decay is not supported"};
|
||||
}
|
||||
if (op->op_params[0] > 1) {
|
||||
return {false, "SSM_SCAN with rollback snapshots (K > 1) is not supported"};
|
||||
}
|
||||
if (ggml_openvino_is_static()) {
|
||||
return {false, "SSM_SCAN requires dynamic execution"};
|
||||
}
|
||||
break;
|
||||
}
|
||||
case GGML_OP_VIEW: {
|
||||
// Skip TOPK_MOE fused tests until it is fully supported.
|
||||
// The argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe.
|
||||
@@ -2274,7 +2324,8 @@ static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(gg
|
||||
if (supported_types.find(src->type) == supported_types.end()) {
|
||||
return {false, "src[" + std::to_string(i) + "] type " + std::string(ggml_type_name(src->type)) + " is not supported"};
|
||||
}
|
||||
if (!has_strides_on_element_grid(src) && !(i == 1 && GgmlOvDecoder::is_gdn_snapshot_writeback(op)) &&
|
||||
if (!has_strides_on_element_grid(src) && !is_row_span_view(src) &&
|
||||
!(i == 1 && GgmlOvDecoder::is_gdn_snapshot_writeback(op)) &&
|
||||
!(i == 2 && GgmlOvDecoder::is_conv_state_snapshot_writeback(op))) {
|
||||
return {false, "src[" + std::to_string(i) + "] strides are not multiples of each other"};
|
||||
}
|
||||
|
||||
@@ -197,8 +197,7 @@ void ggml_openvino_model_cache_init() {
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (cache_only && (ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC") ||
|
||||
ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE") ||
|
||||
if (cache_only && (ggml_openvino_is_static() || ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE") ||
|
||||
ggml_openvino_getenv_int("GGML_OPENVINO_ENABLE_FALLBACK"))) {
|
||||
GGML_ABORT("ggml-openvino: cache-only mode requires dynamic CPU/GPU execution with caching and without fallback");
|
||||
}
|
||||
@@ -368,7 +367,8 @@ uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph,
|
||||
// Version 5 requantizes K-quant MoE experts to Q4_1_64 on GPU by default.
|
||||
// Version 6 clears recurrent-cache slots with Select, so a NaN in a cleared slot does not survive.
|
||||
// Version 7 reads ROPE frequency factors named other than rope_freqs.weight as a Constant, not a Parameter.
|
||||
h = fnv1a_u64(h, 7);
|
||||
// Version 8 drops result_norm from the outputs when it has a consumer, and hashes the output flags.
|
||||
h = fnv1a_u64(h, 8);
|
||||
h = fnv1a(h, graph_signature.data(), graph_signature.size());
|
||||
for (const char * name : cache_settings) {
|
||||
const char * value = ggml_openvino_getenv_str(name, "");
|
||||
@@ -391,6 +391,8 @@ uint64_t ggml_openvino_model_fingerprint(const ggml_cgraph * cgraph,
|
||||
const ggml_tensor * node = cgraph->nodes[i];
|
||||
h = fnv1a_u64(h, static_cast<uint64_t>(node->op));
|
||||
h = fnv1a(h, node->name, strlen(node->name));
|
||||
// Each output is a Result of the model; llama_set_embeddings_layer_inp() changes only the flags.
|
||||
h = fnv1a_u64(h, (node->flags & GGML_TENSOR_FLAG_OUTPUT) ? 1u : 0u);
|
||||
has_gdn = has_gdn || node->op == GGML_OP_GATED_DELTA_NET;
|
||||
}
|
||||
if (has_gdn && ggml_openvino_is_npu()) {
|
||||
|
||||
@@ -75,6 +75,28 @@ OutputVector translate_cpy(const NodeContext & context) {
|
||||
return rename_outputs_with_suffix({result}, context.get_name());
|
||||
}
|
||||
|
||||
if (op_case == 13) {
|
||||
// SSM_SCAN final-state writeback: output the written state block and the full updated cache
|
||||
auto src = process_view_input_new(context, 0);
|
||||
auto base = context.get_input(1);
|
||||
if (src.get_element_type() != context.get_output_type()) {
|
||||
src = std::make_shared<ov::op::v0::Convert>(src, context.get_output_type());
|
||||
}
|
||||
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
|
||||
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
|
||||
auto int_max = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX});
|
||||
auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});
|
||||
auto begin = context.get_input("rs_slot_begin_" + context.get_name());
|
||||
auto src_len = std::make_shared<ov::op::v8::Gather>(
|
||||
std::make_shared<ov::op::v3::ShapeOf>(src, ov::element::i64), axis,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {}, {0}));
|
||||
auto end = std::make_shared<ov::op::v1::Add>(begin, src_len);
|
||||
auto head_part = std::make_shared<ov::op::v8::Slice>(base, zero, begin, one, axis);
|
||||
auto tail_part = std::make_shared<ov::op::v8::Slice>(base, end, int_max, one, axis);
|
||||
auto updated = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{head_part, src, tail_part}, 2);
|
||||
return rename_outputs_with_suffix({src, updated}, context.get_name());
|
||||
}
|
||||
|
||||
if (op_case == 4) {
|
||||
auto src = process_view_input_new(context, 0);
|
||||
auto base = context.get_input(1);
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
#include <openvino/op/add.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/convert.hpp>
|
||||
#include <openvino/op/multiply.hpp>
|
||||
#include <vector>
|
||||
|
||||
@@ -26,15 +27,20 @@ OutputVector translate_scale(const NodeContext & context) {
|
||||
return translate_recurrent_cache_reset(context);
|
||||
}
|
||||
|
||||
auto scaled = std::make_shared<ov::op::v1::Multiply>(process_view_input_new(context, 0), scale_node);
|
||||
|
||||
std::shared_ptr<ov::Node> res;
|
||||
// Multiply and Add need matching types; compute f16 input in f32 and round once, as the CPU backend does.
|
||||
auto input = process_view_input_new(context, 0);
|
||||
const auto input_type = input.get_element_type();
|
||||
if (input_type != ov::element::f32) {
|
||||
input = std::make_shared<ov::op::v0::Convert>(input, ov::element::f32);
|
||||
}
|
||||
ov::Output<ov::Node> res = std::make_shared<ov::op::v1::Multiply>(input, scale_node);
|
||||
if (bias != 0.0f) {
|
||||
auto bias_node =
|
||||
std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{}, std::vector<float>{bias});
|
||||
res = std::make_shared<ov::op::v1::Add>(scaled, bias_node);
|
||||
} else {
|
||||
res = scaled;
|
||||
res = std::make_shared<ov::op::v1::Add>(res, bias_node);
|
||||
}
|
||||
if (input_type != ov::element::f32) {
|
||||
res = std::make_shared<ov::op::v0::Convert>(res, input_type);
|
||||
}
|
||||
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
// Copyright (C) 2018-2026 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
// Local mirror of OpenVINO's ov::op::internal::SelectiveSSM declaration
|
||||
// (src/core/dev_api/openvino/op/selective_ssm.hpp). The implementation comes from the linked
|
||||
// OpenVINO runtime; the class layout must stay in sync with the runtime's declaration.
|
||||
#pragma once
|
||||
|
||||
#include "openvino/op/op.hpp"
|
||||
|
||||
namespace ov::op::internal {
|
||||
/// Mamba2 selective state-space recurrence (arXiv:2405.21060).
|
||||
/// dA = exp(A * dt), dtB = dt * B; per token t:
|
||||
/// dBx_t = x_t (x) dtB_t, state_t = state_{t-1} * dA_t + dBx_t, y_t = sum(state_t * C_t).
|
||||
/// B and C are grouped and shared across heads: head h reads group h / (num_heads / num_groups).
|
||||
class OPENVINO_API SelectiveSSM : public ov::op::Op {
|
||||
public:
|
||||
OPENVINO_OP("SelectiveSSM")
|
||||
|
||||
SelectiveSSM() = default;
|
||||
/// \param A (Negative) log-decay rates per head [num_heads].
|
||||
/// \param dt Per-token, per-head time steps [batch_size, seq_len, num_heads].
|
||||
/// \param B Grouped input projection [batch_size, seq_len, num_groups, state_size].
|
||||
/// \param x Input hidden states [batch_size, seq_len, num_heads, head_dim].
|
||||
/// \param C Grouped output projection [batch_size, seq_len, num_groups, state_size].
|
||||
/// \param recurrent_state Initial state [batch_size, num_heads, head_dim, state_size].
|
||||
SelectiveSSM(const Output<Node> & A,
|
||||
const Output<Node> & dt,
|
||||
const Output<Node> & B,
|
||||
const Output<Node> & x,
|
||||
const Output<Node> & C,
|
||||
const Output<Node> & recurrent_state);
|
||||
|
||||
/// \param args Input tensor vector in order: A, dt, B, x, C, recurrent_state.
|
||||
explicit SelectiveSSM(const ov::OutputVector & args);
|
||||
|
||||
void validate_and_infer_types() override;
|
||||
bool visit_attributes(AttributeVisitor & visitor) override;
|
||||
std::shared_ptr<ov::Node> clone_with_new_inputs(const ov::OutputVector & new_args) const override;
|
||||
};
|
||||
|
||||
} // namespace ov::op::internal
|
||||
@@ -90,12 +90,71 @@ static OutputVector translate_conv_snapshots(const NodeContext & context) {
|
||||
return rename_outputs_with_suffix({result}, name);
|
||||
}
|
||||
|
||||
// SET_ROWS into a column range of a plain 2-D buffer (see GgmlOvDecoder::is_column_set_rows). The result is the
|
||||
// whole buffer: the indexed rows get the new columns, the other columns and rows keep their values, so the next
|
||||
// write into another column range of the same buffer reads this one.
|
||||
static OutputVector translate_column_set_rows(const NodeContext & context) {
|
||||
using namespace ov::op;
|
||||
auto i64_const = [](const std::vector<int64_t> & values) {
|
||||
return v0::Constant::create(ov::element::i64, {values.size()}, values);
|
||||
};
|
||||
|
||||
const auto view_shape = context.get_view_input_ggml_shape(2, 0); // [1, 1, rows, width]
|
||||
FRONT_END_OP_CONVERSION_CHECK(view_shape.size() == 4, "SET_ROWS column write needs a 4-D destination view");
|
||||
const int64_t width = (int64_t) view_shape[3];
|
||||
|
||||
auto base = context.get_input(2);
|
||||
const auto base_shape = base.get_partial_shape();
|
||||
FRONT_END_OP_CONVERSION_CHECK(base_shape.rank().is_static() && base_shape[base_shape.size() - 1].is_static(),
|
||||
"SET_ROWS column write needs a static buffer row size");
|
||||
const int64_t row_size = base_shape[base_shape.size() - 1].get_length();
|
||||
const size_t elem_size = context.get_output_type().size();
|
||||
const int64_t col_begin = (int64_t) (context.get_view_input_offset(2, 0) / elem_size) % row_size;
|
||||
const int64_t col_end = col_begin + width;
|
||||
FRONT_END_OP_CONVERSION_CHECK(col_end <= row_size, "SET_ROWS column range is out of bounds");
|
||||
|
||||
Output<Node> data = std::make_shared<v1::Reshape>(process_view_input_new(context, 0), i64_const({1, 1, -1, width}),
|
||||
false);
|
||||
if (data.get_element_type() != context.get_output_type()) {
|
||||
data = std::make_shared<v0::Convert>(data, context.get_output_type());
|
||||
}
|
||||
Output<Node> rows = std::make_shared<v1::Reshape>(process_view_input_new(context, 1), i64_const({-1}), false);
|
||||
if (rows.get_element_type() != ov::element::i64) {
|
||||
rows = std::make_shared<v0::Convert>(rows, ov::element::i64);
|
||||
}
|
||||
|
||||
auto axis = v0::Constant::create(ov::element::i64, {}, {2});
|
||||
auto buffer = std::make_shared<v1::Reshape>(base, i64_const({1, 1, -1, row_size}), false);
|
||||
Output<Node> updates = data;
|
||||
if (width < row_size) {
|
||||
auto existing = std::make_shared<v8::Gather>(buffer, rows, axis);
|
||||
auto col_axis = i64_const({3});
|
||||
OutputVector parts;
|
||||
if (col_begin > 0) {
|
||||
parts.push_back(std::make_shared<v8::Slice>(existing, i64_const({0}), i64_const({col_begin}),
|
||||
i64_const({1}), col_axis));
|
||||
}
|
||||
parts.push_back(data);
|
||||
if (col_end < row_size) {
|
||||
parts.push_back(std::make_shared<v8::Slice>(existing, i64_const({col_end}), i64_const({row_size}),
|
||||
i64_const({1}), col_axis));
|
||||
}
|
||||
updates = std::make_shared<v0::Concat>(parts, 3);
|
||||
}
|
||||
auto updated = std::make_shared<v3::ScatterUpdate>(buffer, rows, updates, axis);
|
||||
auto res = std::make_shared<v1::Reshape>(updated, std::make_shared<v3::ShapeOf>(base, ov::element::i64), false);
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
}
|
||||
|
||||
OutputVector translate_set_rows(const NodeContext & context) {
|
||||
num_inputs_check(context, 3, 3);
|
||||
|
||||
if (context.get_op_case() == 3) {
|
||||
return translate_conv_snapshots(context);
|
||||
}
|
||||
if (context.get_op_case() == 4) {
|
||||
return translate_column_set_rows(context);
|
||||
}
|
||||
|
||||
auto data = process_view_input_new(context, 0);
|
||||
auto indices = process_view_input_new(context, 1);
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
// Copyright (C) 2018-2026 Intel Corporation
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
//
|
||||
// Adapted from OpenVINO's GGUF frontend (src/frontends/gguf/src/op/ssm_scan.cpp).
|
||||
|
||||
#include "../node_context.h"
|
||||
#include "../op_table.h"
|
||||
#include "../utils.h"
|
||||
#include "selective_ssm.hpp"
|
||||
|
||||
#include <openvino/op/concat.hpp>
|
||||
#include <openvino/op/constant.hpp>
|
||||
#include <openvino/op/gather.hpp>
|
||||
#include <openvino/op/reshape.hpp>
|
||||
#include <openvino/op/shape_of.hpp>
|
||||
#include <openvino/op/softplus.hpp>
|
||||
|
||||
namespace ov {
|
||||
namespace frontend {
|
||||
namespace ggml {
|
||||
namespace op {
|
||||
|
||||
OutputVector translate_ssm_scan(const NodeContext & context) {
|
||||
num_inputs_check(context, 7, 7);
|
||||
using namespace ov::op;
|
||||
|
||||
// ggml: s [slots,H,D,S], x [B,T,H,D], dt [1,B,T,H], A [1,1,H,1],
|
||||
// B/C [B,T,G,S], ids [1,1,1,B]. Only Mamba2 has scalar decay per head.
|
||||
const auto a_shape = context.get_input_shape(3);
|
||||
FRONT_END_OP_CONVERSION_CHECK(a_shape.rank() == 4 && a_shape[0] == 1 && a_shape[1] == 1 && a_shape[3] == 1,
|
||||
"SSM_SCAN: SelectiveSSM requires Mamba2 scalar decay per head; "
|
||||
"Mamba1 state-wise decay is not supported");
|
||||
const auto x = process_view_input_new(context, 1);
|
||||
const auto flat = v0::Constant::create(element::i64, {1}, {-1});
|
||||
const auto axis0 = v0::Constant::create(element::i64, {}, {0});
|
||||
const auto ids = std::make_shared<v1::Reshape>(process_view_input_new(context, 6), flat, false);
|
||||
const auto state = std::make_shared<v8::Gather>(process_view_input_new(context, 0), ids, axis0);
|
||||
const auto a = std::make_shared<v1::Reshape>(process_view_input_new(context, 3), flat, false);
|
||||
const auto dt_shape = std::make_shared<v8::Gather>(std::make_shared<v3::ShapeOf>(x),
|
||||
v0::Constant::create(element::i64, {3}, {0, 1, 2}),
|
||||
axis0);
|
||||
const auto dt =
|
||||
std::make_shared<v4::SoftPlus>(std::make_shared<v1::Reshape>(process_view_input_new(context, 2), dt_shape, false));
|
||||
const auto ssm = std::make_shared<internal::SelectiveSSM>(a,
|
||||
dt,
|
||||
process_view_input_new(context, 4),
|
||||
x,
|
||||
process_view_input_new(context, 5),
|
||||
state);
|
||||
// Pack directly in GGML layout so whole-part views can select either Concat input.
|
||||
const auto packed_shape = v0::Constant::create(element::i64, {4}, {1, 1, 1, -1});
|
||||
const auto y = std::make_shared<v1::Reshape>(ssm->output(0), packed_shape, false);
|
||||
const auto s = std::make_shared<v1::Reshape>(ssm->output(1), packed_shape, false);
|
||||
const auto result = std::make_shared<v0::Concat>(OutputVector{y, s}, 3);
|
||||
result->get_rt_info()[GGML_OV_SSM_SCAN_PACKED] = true;
|
||||
return rename_outputs_with_suffix({result}, context.get_name());
|
||||
}
|
||||
|
||||
} // namespace op
|
||||
} // namespace ggml
|
||||
} // namespace frontend
|
||||
} // namespace ov
|
||||
|
||||
@@ -70,7 +70,8 @@ OutputVector translate_tri(const NodeContext & context) {
|
||||
throw std::runtime_error("translate_tri: invalid tri_type " + std::to_string(tri_type));
|
||||
}
|
||||
|
||||
auto zero = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f});
|
||||
// Select needs the zero in the input's type (f32, f16 or bf16).
|
||||
auto zero = ov::op::v0::Constant::create(x.get_element_type(), {}, {0.0f});
|
||||
auto res = std::make_shared<ov::op::v1::Select>(mask, x, zero);
|
||||
|
||||
return rename_outputs_with_suffix({res}, context.get_name());
|
||||
|
||||
@@ -98,6 +98,7 @@ std::unordered_map<std::string, CreatorFunction> get_supported_ops() {
|
||||
{"GGML_OP_PAD", op::translate_pad },
|
||||
{"GGML_OP_TOP_K", op::translate_top_k },
|
||||
{"GGML_OP_SSM_CONV", op::translate_ssm_conv },
|
||||
{"GGML_OP_SSM_SCAN", op::translate_ssm_scan },
|
||||
{"GGML_OP_GATED_DELTA_NET", op::translate_gated_delta_net },
|
||||
{"GGML_OP_REPEAT", op::translate_repeat },
|
||||
{"GGML_OP_CUMSUM", op::translate_cumsum },
|
||||
|
||||
@@ -49,6 +49,7 @@ GGML_OP_CONVERTER(translate_flash_attn_ext);
|
||||
GGML_OP_CONVERTER(translate_clamp);
|
||||
GGML_OP_CONVERTER(translate_pad);
|
||||
GGML_OP_CONVERTER(translate_ssm_conv);
|
||||
GGML_OP_CONVERTER(translate_ssm_scan);
|
||||
GGML_OP_CONVERTER(translate_gated_delta_net);
|
||||
#ifdef GGML_OPENVINO_HAS_CACHE_UPDATE
|
||||
OutputVector translate_recurrent_gated_delta_net(const NodeContext & context, const OutputVector & cache_inputs);
|
||||
|
||||
@@ -465,6 +465,25 @@ ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int inp
|
||||
return input;
|
||||
}
|
||||
|
||||
// A VIEW over the packed SSM_SCAN output selects one whole Concat part: y at offset 0, the
|
||||
// final state after it. The y length is token-dependent, so a static Slice of the flat
|
||||
// tensor would bake in the captured token count; return the native part instead.
|
||||
if (view_input_size == 1 && input.get_node_shared_ptr()->get_rt_info().count(GGML_OV_SSM_SCAN_PACKED)) {
|
||||
const bool is_y = context.get_view_input_offset(input_index, 0) == 0;
|
||||
auto part = input.get_node_shared_ptr()->input_value(is_y ? 0 : 1).get_node_shared_ptr()->input_value(0);
|
||||
if (is_y) {
|
||||
// y keeps its native [n_seqs, n_tokens, n_head, head_dim] layout
|
||||
return part;
|
||||
}
|
||||
// state [n_seqs, n_head, head_dim, d_state] -> view layout [1, 1, n_seqs, d_state * d_inner]
|
||||
auto n_seqs = get_dimensions(std::make_shared<ov::op::v3::ShapeOf>(part, ov::element::i64), {0});
|
||||
auto target = std::make_shared<ov::op::v0::Concat>(
|
||||
ov::OutputVector{ov::op::v0::Constant::create(ov::element::i64, {2}, {1, 1}), n_seqs,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {-1})},
|
||||
0);
|
||||
return std::make_shared<ov::op::v1::Reshape>(part, target, false);
|
||||
}
|
||||
|
||||
// Integer router views can share these strides when their row count is zero.
|
||||
// Packed projections can store query and gate in separate halves of each head.
|
||||
if (!context.is_static() && view_input_size == 1 && input.get_element_type().is_real()) {
|
||||
@@ -534,6 +553,70 @@ ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int inp
|
||||
}
|
||||
}
|
||||
|
||||
// A VIEW that regroups one contiguous span of every source row, e.g. the z/x/B/C splits of the
|
||||
// fused mamba projections: [head_dim, n_head, nt, ns] over [d_in_proj, nt, ns] rows. The span
|
||||
// dims are packed and the remaining dims carry the source's outer strides, so the view is a
|
||||
// Slice of the source's innermost axis plus a Reshape.
|
||||
if (!context.is_static() && view_input_size == 1 && input.get_element_type().is_real()) {
|
||||
const auto source = context.get_view_input_src_ggml_shape(input_index, 0);
|
||||
const auto view = context.get_view_input_ggml_shape(input_index, 0);
|
||||
const auto source_stride = context.get_view_input_src_stride(input_index, 0);
|
||||
const auto view_stride = context.get_view_input_stride(input_index, 0);
|
||||
const auto in_rank = input.get_partial_shape().rank();
|
||||
const size_t nd = view.size();
|
||||
if (nd == 4 && source.size() == nd && view_stride.size() == nd && source_stride.size() == nd &&
|
||||
in_rank.is_static() && in_rank.get_length() == static_cast<int64_t>(nd) &&
|
||||
view_stride[nd - 1] == source_stride[nd - 1] && source_stride[nd - 1] > 0) {
|
||||
const size_t es = source_stride[nd - 1];
|
||||
int j = static_cast<int>(nd) - 1;
|
||||
size_t span = 1;
|
||||
while (j >= 0 && view_stride[j] == span * es) {
|
||||
span *= view[j];
|
||||
--j;
|
||||
}
|
||||
// at least two packed dims replace the source's innermost dim; a one-dim span keeps the
|
||||
// source rank and is handled by the generic paths below
|
||||
const int shift = static_cast<int>(nd) - 2 - j;
|
||||
bool matches = j >= 0 && shift >= 1;
|
||||
for (int t = 0; matches && t <= j; ++t) {
|
||||
matches = view[t] == source[t + shift] && view_stride[t] == source_stride[t + shift];
|
||||
}
|
||||
for (int t = 0; matches && t < shift; ++t) {
|
||||
matches = source[t] == 1;
|
||||
}
|
||||
const size_t offset = context.get_view_input_offset(input_index, 0);
|
||||
const size_t src_offset = context.get_view_input_src_offset(input_index, 0);
|
||||
if (matches && offset >= src_offset && (offset - src_offset) % es == 0) {
|
||||
const int64_t begin = static_cast<int64_t>((offset - src_offset) / es);
|
||||
if (begin + static_cast<int64_t>(span) <= static_cast<int64_t>(source[nd - 1])) {
|
||||
auto sliced = std::make_shared<ov::op::v8::Slice>(
|
||||
input, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin}),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {begin + static_cast<int64_t>(span)}),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {static_cast<int64_t>(nd) - 1}));
|
||||
const auto view_ov_shape = context.get_view_input_ov_shape(input_index, 0);
|
||||
std::vector<int64_t> pattern(nd);
|
||||
size_t dynamic_dims = 0;
|
||||
for (size_t i = 0; i < nd; ++i) {
|
||||
if (view_ov_shape.rank().is_static() && view_ov_shape[i].is_static()) {
|
||||
pattern[i] = view_ov_shape[i].get_length();
|
||||
} else {
|
||||
pattern[i] = -1;
|
||||
++dynamic_dims;
|
||||
}
|
||||
}
|
||||
if (dynamic_dims > 1) {
|
||||
for (size_t i = 0; i < nd; ++i) {
|
||||
pattern[i] = static_cast<int64_t>(view[i]);
|
||||
}
|
||||
}
|
||||
return std::make_shared<ov::op::v1::Reshape>(
|
||||
sliced, ov::op::v0::Constant::create(ov::element::i64, {nd}, pattern), false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A row range of a 2D weight (BERT's token type row): slice the rows of the [rows, row_size] Constant
|
||||
if (view_input_size == 1 && input.get_partial_shape().rank().is_static() &&
|
||||
input.get_partial_shape().rank().get_length() == 2 &&
|
||||
@@ -671,14 +754,17 @@ ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int inp
|
||||
exact_chunk = i == static_cast<size_t>(split_dim) || view_ggml_shape[i] == 1 ||
|
||||
view_stride_v[i] == view_src_stride_v[i];
|
||||
}
|
||||
// stateful execution drops the leading size-1 axis, so the input can be one rank lower than the
|
||||
// ggml metadata
|
||||
const auto input_rank = input.get_partial_shape().rank();
|
||||
const int64_t rank_shift = input_rank.is_static() ?
|
||||
static_cast<int64_t>(ndims) - input_rank.get_length() :
|
||||
static_cast<int64_t>(ndims);
|
||||
// num_splits is the captured source size: on a dynamic axis (e.g. the token axis of DSpark's
|
||||
// result_norm, read one block position at a time) the graph is reused with other sizes, and
|
||||
// each Split output would then hold several rows or the wrong one
|
||||
const auto input_shape = input.get_partial_shape();
|
||||
const int64_t split_axis = input_shape.rank().is_static() ?
|
||||
split_dim - (int64_t) ndims + input_shape.rank().get_length() :
|
||||
-1;
|
||||
const bool split_axis_static = split_axis >= 0 && input_shape[split_axis].is_static() &&
|
||||
input_shape[split_axis].get_length() == num_splits;
|
||||
if (split_index >= 0 && split_index < num_splits && dynamic_split && exact_chunk &&
|
||||
split_dim >= rank_shift) {
|
||||
split_axis_static) {
|
||||
// the Split keeps its input alive, so only keep a weak reference to it on the input
|
||||
auto src_node = input.get_node_shared_ptr();
|
||||
std::string rt_key =
|
||||
@@ -689,8 +775,7 @@ ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int inp
|
||||
split_node = rt_info[rt_key].as<std::weak_ptr<ov::Node>>().lock();
|
||||
}
|
||||
if (split_node == nullptr) {
|
||||
auto axis_const = ov::op::v0::Constant::create(
|
||||
ov::element::i64, {}, {static_cast<int64_t>(split_dim) - rank_shift});
|
||||
auto axis_const = ov::op::v0::Constant::create(ov::element::i64, {}, {split_axis});
|
||||
split_node =
|
||||
std::make_shared<ov::op::v1::Split>(input, axis_const, static_cast<size_t>(num_splits));
|
||||
split_node->set_friendly_name(src_node->get_friendly_name() + "_split");
|
||||
@@ -722,7 +807,8 @@ ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int inp
|
||||
|
||||
// Lambda function to process a single view operation
|
||||
auto process_single_view =
|
||||
[](ov::Output<ov::Node> current, size_t view_offset, const std::vector<size_t> & view_stride,
|
||||
[is_static = context.is_static()](
|
||||
ov::Output<ov::Node> current, size_t view_offset, const std::vector<size_t> & view_stride,
|
||||
const ov::Shape & view_ggml_shape, const ov::PartialShape & view_ov_shape, const std::string & view_name,
|
||||
size_t view_src_offset, const std::vector<size_t> & view_src_stride, const ov::Shape & view_src_ggml_shape,
|
||||
const ov::PartialShape & view_src_ov_shape, const std::string & view_src_name) -> ov::Output<ov::Node> {
|
||||
@@ -847,7 +933,12 @@ ov::Output<ov::Node> process_view_input_new(const NodeContext & context, int inp
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {1}),
|
||||
ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim - axis_shift}));
|
||||
|
||||
if (view_ov_shape.is_static()) {
|
||||
// In a static graph a rank-preserving slice (diff_dims has one entry) already has the view
|
||||
// shape. Skip the Reshape: its target bakes the static chunk size into the token axis, which
|
||||
// is wrong after the output-ids GET_ROWS of the last layer (1 token in the prefill graph).
|
||||
const bool rank_kept = is_static && current_rank.is_static() &&
|
||||
static_cast<size_t>(current_rank.get_length()) == ndims;
|
||||
if (view_ov_shape.is_static() && !rank_kept) {
|
||||
auto reshaped = std::make_shared<ov::op::v1::Reshape>(
|
||||
sliced,
|
||||
ov::op::v0::Constant::create(ov::element::i64, {ndims}, view_ov_shape.to_shape()),
|
||||
|
||||
@@ -97,6 +97,9 @@ constexpr const char * GGML_OV_GROUPED_ROWS = "ggml_ov_grouped_rows";
|
||||
// rt_info key on the stateful V cache read with `-fa off` (permute): V is [.., kv, head_size], not transposed
|
||||
constexpr const char * GGML_OV_V_TOKEN_MAJOR = "ggml_ov_v_token_major";
|
||||
|
||||
// rt_info key on the packed SSM_SCAN output Concat [1, 1, 1, y | state]; whole-part VIEWs select one Concat input
|
||||
constexpr const char * GGML_OV_SSM_SCAN_PACKED = "ggml_ov_ssm_scan_packed";
|
||||
|
||||
namespace op {
|
||||
template <typename T> OutputVector translate_1to1_match_2_inputs(const NodeContext & context) {
|
||||
num_inputs_check(context, 2, 2);
|
||||
|
||||
@@ -657,6 +657,30 @@ ov::Tensor convert_ggml_input_to_ov(const std::shared_ptr<GgmlOvDecoder> & ggml_
|
||||
return *sliced;
|
||||
}
|
||||
|
||||
// KQ mask with fewer columns than the rounded KV length (see compute_llm_params): pad it with -inf
|
||||
const auto & compute_params = ggml_decoder->get_compute_params();
|
||||
if (!ggml_decoder->is_static() && name == compute_params.mask_name &&
|
||||
compute_params.attention_size > ggml_tensor->ne[0] && ggml_is_contiguous(ggml_tensor)) {
|
||||
const size_t rows = ggml_nrows(ggml_tensor);
|
||||
const size_t cols = ggml_tensor->ne[0];
|
||||
const size_t padded_cols = compute_params.attention_size;
|
||||
ov::Shape shape = GgmlOvDecoder::get_shape(ggml_tensor);
|
||||
shape.back() = padded_cols;
|
||||
ov::Tensor padded(GgmlOvDecoder::get_ov_type(ggml_tensor), shape);
|
||||
auto pad = [&](auto fill) {
|
||||
using T = decltype(fill);
|
||||
auto data = pad_input<T>((const T *) ggml_tensor->data, rows, cols, rows, padded_cols, fill);
|
||||
std::memcpy(padded.data(), data.data(), padded.get_byte_size());
|
||||
};
|
||||
if (ggml_tensor->type == GGML_TYPE_F16) {
|
||||
pad(GGML_FP32_TO_FP16(-INFINITY));
|
||||
} else {
|
||||
GGML_ASSERT(ggml_tensor->type == GGML_TYPE_F32);
|
||||
pad(-INFINITY);
|
||||
}
|
||||
return padded;
|
||||
}
|
||||
|
||||
if (ggml_decoder->is_static() &&
|
||||
GgmlOvDecoder::is_kvcache(ggml_tensor, ggml_decoder->get_tensor_used_op(ggml_tensor))) {
|
||||
const auto type = ggml_decoder->get_ov_type(ggml_tensor);
|
||||
@@ -2192,8 +2216,21 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared
|
||||
old_m_params = ggml_decoder->get_model_params();
|
||||
cache_hit = old_m_params.can_reuse_statically(m_params);
|
||||
if (cache_hit && is_prefill && ggml_openvino_kv_slice_enabled()) {
|
||||
cache_hit = entry->prefill_attention_size == c_params.attention_size &&
|
||||
entry->prefill_attention_size_swa == c_params.attention_size_swa;
|
||||
// A larger compiled prefill also serves a smaller attended length: the extra KV rows are masked, as with
|
||||
// the power-of-two rounding. Without this, a new prompt walks up the sizes again, and since only one
|
||||
// prefill model is kept, every prompt longer than one chunk recompiles. Head-major KV is bound at the
|
||||
// storage size, so it keeps the exact match.
|
||||
// The output count is part of the static prefill shape (more than 1 only for llama-perplexity).
|
||||
const bool reuse_larger = !ggml_openvino_npu_kv_head_major_enabled();
|
||||
cache_hit = entry->prefill_output_len == c_params.output_len &&
|
||||
(reuse_larger ? entry->prefill_attention_size >= c_params.attention_size &&
|
||||
entry->prefill_attention_size_swa >= c_params.attention_size_swa :
|
||||
entry->prefill_attention_size == c_params.attention_size &&
|
||||
entry->prefill_attention_size_swa == c_params.attention_size_swa);
|
||||
if (cache_hit) {
|
||||
c_params.attention_size = entry->prefill_attention_size;
|
||||
c_params.attention_size_swa = entry->prefill_attention_size_swa;
|
||||
}
|
||||
prefill_size_changed = !cache_hit;
|
||||
}
|
||||
}
|
||||
@@ -2260,6 +2297,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared
|
||||
entry->ptr = ggml_decoder;
|
||||
entry->prefill_attention_size = c_params.attention_size;
|
||||
entry->prefill_attention_size_swa = c_params.attention_size_swa;
|
||||
entry->prefill_output_len = c_params.output_len;
|
||||
entry->decode_attention_size = compiled.decode_attention_size;
|
||||
entry->shared_key = shared_key;
|
||||
entry->decode_window_requests.clear();
|
||||
@@ -2274,6 +2312,10 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared
|
||||
r_ctx->ov_output_names_cache_prefill[key] = compiled.prefill_output_names;
|
||||
decoder_end_time = conversion_end_time = compile_end_time = ggml_time_us();
|
||||
GGML_LOG_DEBUG("ggml-openvino: shared compiled model HIT (static)\n");
|
||||
if (prefill_size_changed) {
|
||||
GGML_LOG_INFO("ggml-openvino: reuse the compiled prefill model of attention size %d\n",
|
||||
c_params.attention_size);
|
||||
}
|
||||
} else {
|
||||
auto decode_c_params = c_params;
|
||||
if (ggml_openvino_kv_slice_enabled()) {
|
||||
@@ -2600,6 +2642,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared
|
||||
entry->ptr = ggml_decoder;
|
||||
entry->prefill_attention_size = c_params.attention_size;
|
||||
entry->prefill_attention_size_swa = c_params.attention_size_swa;
|
||||
entry->prefill_output_len = c_params.output_len;
|
||||
entry->decode_attention_size = decode_c_params.attention_size;
|
||||
entry->shared_key = shared_key;
|
||||
entry->decode_window_requests.clear();
|
||||
@@ -3180,7 +3223,7 @@ enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend)
|
||||
GgmlOvDecoder::dump_cgraph(cgraph, filename);
|
||||
}
|
||||
|
||||
const auto is_static = ggml_openvino_is_npu() || ggml_openvino_getenv_int("GGML_OPENVINO_FORCE_STATIC");
|
||||
const auto is_static = ggml_openvino_is_static();
|
||||
|
||||
GGML_ASSERT(ctx->runtime_context != nullptr);
|
||||
std::shared_ptr<ov_runtime_context> r_ctx = std::static_pointer_cast<ov_runtime_context>(ctx->runtime_context);
|
||||
|
||||
@@ -42,6 +42,11 @@ struct graph_key {
|
||||
if (GgmlOvDecoder::is_sampling_view(node)) {
|
||||
input_srcs.push_back("sampling_row_" + std::string(node->name) + "#n" + std::to_string(i));
|
||||
}
|
||||
// The output set is part of the model (one Result per output). Enabling outputs on an existing graph,
|
||||
// e.g. llama_set_embeddings_layer_inp() for a DFlash/DSpark drafter, changes only the flags.
|
||||
if (node->flags & GGML_TENSOR_FLAG_OUTPUT) {
|
||||
input_srcs.push_back("output_" + std::string(node->name) + "#n" + std::to_string(i));
|
||||
}
|
||||
}
|
||||
|
||||
static const bool full_key = ggml_openvino_getenv_int("GGML_OPENVINO_FULL_GRAPH_KEY") != 0;
|
||||
@@ -95,9 +100,11 @@ struct decoder_runtime_ctx {
|
||||
|
||||
std::shared_ptr<std::mutex> mutex;
|
||||
std::shared_ptr<GgmlOvDecoder> ptr;
|
||||
// Attention sizes the cached prefill model was built with; decode calls overwrite the decoder's.
|
||||
// Attention sizes and output count the cached prefill model was built with; decode calls overwrite the
|
||||
// decoder's.
|
||||
int prefill_attention_size = -1;
|
||||
int prefill_attention_size_swa = -1;
|
||||
int prefill_output_len = -1;
|
||||
// KV window of the decode model in the compiled pair (GGML_OPENVINO_NPU_BLOB_PREBUILD).
|
||||
int decode_attention_size = -1;
|
||||
std::string shared_key;
|
||||
|
||||
+113
-6
@@ -3307,6 +3307,85 @@ static int iq2_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, co
|
||||
return grid_index;
|
||||
}
|
||||
|
||||
// same result as iq2_find_best_neighbour: score = s*(s*quad_coef - lin_coef) + const, and the coefs do not depend on the scale s, so cache them per pattern
|
||||
// take the cached winner only if all other neighbours are worse by more than the float rounding bound of the exact search, else run the exact search
|
||||
#define IQ2_NEIGHBOUR_CACHE_MAX 512
|
||||
|
||||
typedef struct {
|
||||
int pattern; // pattern whose coefs are cached, -1 if none
|
||||
int last_best; // last winner for this pattern
|
||||
// sums for the rounding bound: |w|, |w|*|x|, |w|*x^2, and the same with 1 + |w|
|
||||
double sum_w, sum_wx, sum_wxx, sum_w1, sum_w1x, sum_w1xx;
|
||||
double lin_term[8][8], quad_term[8][8]; // 2*w*x*q and w*q*q per element, indexed by the grid level q
|
||||
double lin_coef[IQ2_NEIGHBOUR_CACHE_MAX];
|
||||
double quad_coef[IQ2_NEIGHBOUR_CACHE_MAX];
|
||||
} iq2_neighbour_cache;
|
||||
|
||||
static void iq2_neighbour_cache_init(iq2_neighbour_cache * cache, const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight) {
|
||||
cache->pattern = -1;
|
||||
cache->sum_w = cache->sum_wx = cache->sum_wxx = cache->sum_w1 = cache->sum_w1x = cache->sum_w1xx = 0;
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
const double w = (double)weight[i], x = (double)xval[i], abs_w = fabs(w), abs_x = fabs(x);
|
||||
cache->sum_w += abs_w; cache->sum_wx += abs_w*abs_x; cache->sum_wxx += abs_w*abs_x*abs_x;
|
||||
cache->sum_w1 += 1 + abs_w; cache->sum_w1x += (1 + abs_w)*abs_x; cache->sum_w1xx += (1 + abs_w)*abs_x*abs_x;
|
||||
for (int q = 0; q < 8; ++q) {
|
||||
cache->lin_term[i][q] = 2*w*x*q;
|
||||
cache->quad_term[i][q] = w*q*q;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static int iq2_count_at_most(const double * GGML_RESTRICT values, int n, double limit) {
|
||||
int count = 0;
|
||||
for (int j = 0; j < n; ++j) count += values[j] <= limit;
|
||||
return count;
|
||||
}
|
||||
|
||||
static int iq2_find_best_neighbour_cached(iq2_neighbour_cache * cache, int pattern, const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid,
|
||||
const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, int8_t * GGML_RESTRICT L) {
|
||||
const int num_neighbors = neighbours[0];
|
||||
const double scale_d = (double)scale;
|
||||
const double max_level = 7*scale_d;
|
||||
// size of the numbers in the exact search, its rounding error is a multiple of 2^-24*size (+ size_subnormal for underflow)
|
||||
const double size = max_level*max_level*cache->sum_w + 2*max_level*cache->sum_wx + cache->sum_wxx;
|
||||
const double size_subnormal = (1 + max_level)*(1 + max_level)*cache->sum_w1 + 2*(1 + max_level)*cache->sum_w1x + cache->sum_w1xx;
|
||||
// the cached score is (exact score - const)/s, so the 2^-18*size margin on the exact score is divided by s
|
||||
const double tolerance = (0x1p-18*size + 0x1p-100*size_subnormal)/scale_d;
|
||||
if (num_neighbors <= 0 || num_neighbors > IQ2_NEIGHBOUR_CACHE_MAX || !(scale_d > 0 && size < 0x1p100 && size_subnormal < 0x1p100 && tolerance > 0)) {
|
||||
return iq2_find_best_neighbour(neighbours, grid, xval, weight, scale, L);
|
||||
}
|
||||
if (cache->pattern != pattern) {
|
||||
for (int j = 0; j < num_neighbors; ++j) {
|
||||
const int8_t * levels = (const int8_t *)(grid + neighbours[j+1]);
|
||||
double lin = 0, quad = 0;
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
lin += cache->lin_term[i][levels[i]];
|
||||
quad += cache->quad_term[i][levels[i]];
|
||||
}
|
||||
cache->lin_coef[j] = lin; cache->quad_coef[j] = quad;
|
||||
}
|
||||
cache->pattern = pattern;
|
||||
cache->last_best = 0;
|
||||
}
|
||||
double score[IQ2_NEIGHBOUR_CACHE_MAX];
|
||||
for (int j = 0; j < num_neighbors; ++j) score[j] = scale_d*cache->quad_coef[j] - cache->lin_coef[j];
|
||||
// the winner must be the only neighbour within tolerance, try the previous winner for this pattern first
|
||||
int best = cache->last_best;
|
||||
if (iq2_count_at_most(score, num_neighbors, score[best] + tolerance) != 1) {
|
||||
for (int j = 0; j < num_neighbors; ++j) {
|
||||
if (score[j] < score[best]) best = j;
|
||||
}
|
||||
if (iq2_count_at_most(score, num_neighbors, score[best] + tolerance) != 1) {
|
||||
return iq2_find_best_neighbour(neighbours, grid, xval, weight, scale, L);
|
||||
}
|
||||
cache->last_best = best;
|
||||
}
|
||||
const int grid_index = neighbours[best+1];
|
||||
const int8_t * levels = (const int8_t *)(grid + grid_index);
|
||||
for (int i = 0; i < 8; ++i) L[i] = (levels[i] - 1)/2;
|
||||
return grid_index;
|
||||
}
|
||||
|
||||
static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights) {
|
||||
|
||||
const int gindex = iq2_data_index(GGML_TYPE_IQ2_XXS);
|
||||
@@ -3335,6 +3414,8 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
|
||||
float waux[32];
|
||||
uint8_t block_signs[4];
|
||||
uint32_t q2[2*(QK_K/32)];
|
||||
iq2_neighbour_cache neighbour_cache[4];
|
||||
int8_t Laux_prev[32];
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
@@ -3390,6 +3471,7 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
|
||||
continue;
|
||||
}
|
||||
float best = 0;
|
||||
for (int k = 0; k < 4; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
|
||||
for (int is = -6; is <= 6; ++is) {
|
||||
float id = (2*kMaxQ-1+is*0.1f)/eff_max;
|
||||
float this_scale = 1/id;
|
||||
@@ -3403,9 +3485,14 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
|
||||
int grid_index = kmap_q2xs[u];
|
||||
if (grid_index < 0) {
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
}
|
||||
}
|
||||
// same candidate as for the previous scale: it cannot change the result
|
||||
if (is > -6 && memcmp(Laux, Laux_prev, 32) == 0) {
|
||||
continue;
|
||||
}
|
||||
memcpy(Laux_prev, Laux, 32);
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
float w = weight[i];
|
||||
@@ -3430,7 +3517,7 @@ static void quantize_row_iq2_xxs_impl(const float * GGML_RESTRICT x, void * GGML
|
||||
int grid_index = kmap_q2xs[u];
|
||||
if (grid_index < 0) {
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
}
|
||||
const int8_t * pg = (const int8_t *)(kgrid_q2xs + grid_index);
|
||||
for (int i = 0; i < 8; ++i) L[8*k+i] = (pg[i] - 1)/2;
|
||||
@@ -3515,6 +3602,9 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
|
||||
bool is_on_grid_aux[2];
|
||||
uint8_t block_signs[2];
|
||||
uint16_t q2[2*(QK_K/16)];
|
||||
iq2_neighbour_cache neighbour_cache[2];
|
||||
int8_t Laux_prev[16];
|
||||
bool is_on_grid_prev[2];
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
@@ -3566,6 +3656,7 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
|
||||
float best = 0;
|
||||
float scale = max/(2*kMaxQ-1);
|
||||
is_on_grid[0] = is_on_grid[1] = true;
|
||||
for (int k = 0; k < 2; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
|
||||
for (int is = -9; is <= 9; ++is) {
|
||||
float id = (2*kMaxQ-1+is*0.1f)/max;
|
||||
float this_scale = 1/id;
|
||||
@@ -3581,9 +3672,15 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
|
||||
if (grid_index < 0) {
|
||||
is_on_grid_aux[k] = false;
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
}
|
||||
}
|
||||
// same candidate as for the previous scale: it cannot change the result
|
||||
if (is > -9 && memcmp(Laux, Laux_prev, 16) == 0 && is_on_grid_aux[0] == is_on_grid_prev[0] && is_on_grid_aux[1] == is_on_grid_prev[1]) {
|
||||
continue;
|
||||
}
|
||||
memcpy(Laux_prev, Laux, 16);
|
||||
is_on_grid_prev[0] = is_on_grid_aux[0]; is_on_grid_prev[1] = is_on_grid_aux[1];
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
for (int i = 0; i < 16; ++i) {
|
||||
float w = weight[i];
|
||||
@@ -3613,7 +3710,7 @@ static void quantize_row_iq2_xs_impl(const float * GGML_RESTRICT x, void * GGML_
|
||||
int grid_index = kmap_q2xs[u];
|
||||
if (grid_index < 0) {
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
}
|
||||
}
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
@@ -5155,6 +5252,9 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
|
||||
bool is_on_grid[2];
|
||||
bool is_on_grid_aux[2];
|
||||
uint8_t block_signs[2];
|
||||
iq2_neighbour_cache neighbour_cache[2];
|
||||
int8_t Laux_prev[16];
|
||||
bool is_on_grid_prev[2];
|
||||
|
||||
for (int ibl = 0; ibl < nbl; ++ibl) {
|
||||
|
||||
@@ -5197,6 +5297,7 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
|
||||
float best = 0;
|
||||
float scale = max/(2*kMaxQ-1);
|
||||
is_on_grid[0] = is_on_grid[1] = true;
|
||||
for (int k = 0; k < 2; ++k) iq2_neighbour_cache_init(&neighbour_cache[k], xval + 8*k, waux + 8*k);
|
||||
for (int is = -9; is <= 9; ++is) {
|
||||
float id = (2*kMaxQ-1+is*0.1f)/max;
|
||||
float this_scale = 1/id;
|
||||
@@ -5212,9 +5313,15 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
|
||||
if (grid_index < 0) {
|
||||
is_on_grid_aux[k] = false;
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, this_scale, Laux + 8*k);
|
||||
}
|
||||
}
|
||||
// same candidate as for the previous scale: it cannot change the result
|
||||
if (is > -9 && memcmp(Laux, Laux_prev, 16) == 0 && is_on_grid_aux[0] == is_on_grid_prev[0] && is_on_grid_aux[1] == is_on_grid_prev[1]) {
|
||||
continue;
|
||||
}
|
||||
memcpy(Laux_prev, Laux, 16);
|
||||
is_on_grid_prev[0] = is_on_grid_aux[0]; is_on_grid_prev[1] = is_on_grid_aux[1];
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
for (int i = 0; i < 16; ++i) {
|
||||
float w = weight[i];
|
||||
@@ -5244,7 +5351,7 @@ static void quantize_row_iq2_s_impl(const float * GGML_RESTRICT x, void * GGML_R
|
||||
int grid_index = kmap_q2xs[u];
|
||||
if (grid_index < 0) {
|
||||
const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1;
|
||||
grid_index = iq2_find_best_neighbour(neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
grid_index = iq2_find_best_neighbour_cached(&neighbour_cache[k], u, neighbours, kgrid_q2xs, xval + 8*k, waux + 8*k, scale, L + 8*k);
|
||||
}
|
||||
}
|
||||
float sumqx = 0, sumq2 = 0;
|
||||
|
||||
@@ -7434,8 +7434,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
|
||||
if (op->type == GGML_TYPE_TQ2_0 || op->type == GGML_TYPE_TQ1_0) {
|
||||
return false;
|
||||
}
|
||||
auto res = (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 ||
|
||||
op->src[0]->type == GGML_TYPE_BF16) &&
|
||||
auto res = (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
|
||||
(op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32);
|
||||
return res;
|
||||
}
|
||||
@@ -7624,6 +7623,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
|
||||
}
|
||||
case GGML_OP_ROPE:
|
||||
case GGML_OP_ROPE_BACK:
|
||||
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
|
||||
case GGML_OP_IM2COL:
|
||||
case GGML_OP_IM2COL_3D:
|
||||
case GGML_OP_UPSCALE:
|
||||
|
||||
@@ -101,7 +101,7 @@ void deferred_memset(void * dst, uint32_t val, size_t size, std::vector<vk_stagi
|
||||
|
||||
// matmul
|
||||
vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type type);
|
||||
void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t padded_n, uint64_t b_col_bytes, uint64_t d_col_bytes);
|
||||
void ggml_vk_matmul(ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline, vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer, uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d, uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d, uint32_t split_k, uint32_t stream_k_groups, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3, uint32_t padded_n, uint64_t b_col_bytes, uint64_t d_col_bytes);
|
||||
bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor);
|
||||
vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const ggml_tensor * src, const ggml_tensor * dst, ggml_type to);
|
||||
vk_pipeline ggml_vk_get_quantize_pipeline(ggml_backend_vk_context * ctx, ggml_type type);
|
||||
|
||||
@@ -225,7 +225,7 @@ void ggml_vk_test_matmul(ggml_backend_vk_context * ctx, size_t m, size_t n, size
|
||||
ctx, subctx, p, ggml_vk_subbuffer(ctx, d_X), ggml_vk_subbuffer(ctx, d_Y), ggml_vk_subbuffer(ctx, d_D), ggml_vk_subbuffer(ctx, ctx->prealloc_split_k),
|
||||
m, n, k,
|
||||
k, k, m, k*m, k*n, m*n,
|
||||
split_k, batch, batch, batch, 1, 1, n,
|
||||
split_k, 0, batch, batch, batch, 1, 1, n,
|
||||
k * sizeof(Y_TYPE), m * sizeof(float)
|
||||
);
|
||||
}
|
||||
@@ -693,7 +693,7 @@ void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t
|
||||
ctx, subctx, p, { qx_buf, 0, qx_sz }, { qy_buf, 0, qy_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },
|
||||
m, n, k,
|
||||
k, k, m, k*m, k*n, m*n,
|
||||
split_k, batch, batch, batch, 1, 1, n,
|
||||
split_k, 0, batch, batch, batch, 1, 1, n,
|
||||
(uint64_t)(k / ggml_blck_size(GGML_TYPE_Q8_1)) * ggml_type_size(GGML_TYPE_Q8_1), m * sizeof(float)
|
||||
);
|
||||
}
|
||||
@@ -703,7 +703,7 @@ void ggml_vk_test_dequant_matmul(ggml_backend_vk_context * ctx, size_t m, size_t
|
||||
ctx, subctx, p, { qx_buf, 0, qx_sz }, { y_buf, 0, y_sz }, { d_buf, 0, d_sz }, { ctx->prealloc_split_k, 0, ctx->prealloc_size_split_k },
|
||||
m, n, k,
|
||||
k, k, m, k*m, k*n, m*n,
|
||||
split_k, batch, batch, batch, 1, 1, n,
|
||||
split_k, 0, batch, batch, batch, 1, 1, n,
|
||||
k * sizeof(float), m * sizeof(float)
|
||||
);
|
||||
}
|
||||
@@ -1043,6 +1043,8 @@ void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph
|
||||
} else if (tensor->op == GGML_OP_FILL) {
|
||||
const float value = ggml_get_op_params_f32(tensor, 0);
|
||||
tensor_clone = ggml_fill(ggml_ctx, src_clone[0], value);
|
||||
} else if (tensor->op == GGML_OP_SLEEP) {
|
||||
tensor_clone = ggml_sleep(ggml_ctx, src_clone[0], ggml_get_op_params_i32(tensor, 0));
|
||||
} else if (tensor->op == GGML_OP_SQR) {
|
||||
tensor_clone = ggml_sqr(ggml_ctx, src_clone[0]);
|
||||
} else if (tensor->op == GGML_OP_SQRT) {
|
||||
|
||||
@@ -13,8 +13,38 @@ struct vk_mat_mat_push_constants {
|
||||
uint32_t k_split;
|
||||
uint32_t ne02; uint32_t ne12; uint32_t broadcast2; uint32_t broadcast3;
|
||||
uint32_t padded_N;
|
||||
uint32_t k_units_per_tile;
|
||||
uint32_t work_units_per_group;
|
||||
uint32_t groups_with_extra_unit;
|
||||
uint32_t k_units_mp;
|
||||
uint32_t blocks_m_mp;
|
||||
uint32_t blocks_n_mp;
|
||||
uint32_t ne12_mp;
|
||||
uint32_t broadcast2_mp;
|
||||
uint32_t broadcast3_mp;
|
||||
uint32_t div_Ls0;
|
||||
uint32_t div_Ls1;
|
||||
};
|
||||
|
||||
struct vk_mat_mat_stream_k_fixup_push_constants {
|
||||
uint32_t M; uint32_t N;
|
||||
uint32_t BM; uint32_t BN;
|
||||
uint32_t k_units_per_tile;
|
||||
uint32_t blocks_m;
|
||||
uint32_t blocks_n;
|
||||
uint32_t work_units_per_group;
|
||||
uint32_t groups_with_extra_unit;
|
||||
uint32_t k_units_mp;
|
||||
uint32_t blocks_m_mp;
|
||||
uint32_t blocks_n_mp;
|
||||
uint32_t work_units_mp;
|
||||
uint32_t large_group_work_mp;
|
||||
uint32_t div_Ls0;
|
||||
uint32_t div_Ls1;
|
||||
};
|
||||
|
||||
static_assert(sizeof(vk_mat_mat_push_constants) <= 128);
|
||||
|
||||
struct vk_mat_vec_push_constants {
|
||||
uint32_t ncols;
|
||||
uint32_t stride_a;
|
||||
@@ -159,6 +189,10 @@ struct vk_op_push_constants {
|
||||
float param4;
|
||||
};
|
||||
|
||||
struct vk_op_sleep_push_constants {
|
||||
uint32_t ticks;
|
||||
};
|
||||
|
||||
struct vk_op_fwht_push_constants {
|
||||
uint32_t n_rows;
|
||||
uint32_t src_offset;
|
||||
@@ -403,6 +437,34 @@ template <typename T> void init_pushconst_fastdiv(T &p) {
|
||||
static_assert(!std::is_const<T>::value, "unexpected type");
|
||||
}
|
||||
|
||||
inline void init_pushconst_fastdiv(vk_mat_mat_push_constants &p) {
|
||||
uint32_t ne12_L, broadcast2_L, broadcast3_L;
|
||||
init_fastdiv_values(p.ne12, p.ne12_mp, ne12_L);
|
||||
init_fastdiv_values(p.broadcast2, p.broadcast2_mp, broadcast2_L);
|
||||
init_fastdiv_values(p.broadcast3, p.broadcast3_mp, broadcast3_L);
|
||||
p.div_Ls1 = pack_fastdiv_L(ne12_L, broadcast2_L, broadcast3_L);
|
||||
}
|
||||
|
||||
inline void init_pushconst_fastdiv(vk_mat_mat_push_constants &p, uint32_t blocks_m, uint32_t blocks_n) {
|
||||
uint32_t k_units_L, blocks_m_L, blocks_n_L;
|
||||
init_fastdiv_values(p.k_units_per_tile, p.k_units_mp, k_units_L);
|
||||
init_fastdiv_values(blocks_m, p.blocks_m_mp, blocks_m_L);
|
||||
init_fastdiv_values(blocks_n, p.blocks_n_mp, blocks_n_L);
|
||||
p.div_Ls0 = pack_fastdiv_L(k_units_L, blocks_m_L, blocks_n_L);
|
||||
init_pushconst_fastdiv(p);
|
||||
}
|
||||
|
||||
template <> inline void init_pushconst_fastdiv(vk_mat_mat_stream_k_fixup_push_constants &p) {
|
||||
uint32_t k_units_L, blocks_m_L, blocks_n_L, work_units_L, large_group_work_L;
|
||||
init_fastdiv_values(p.k_units_per_tile, p.k_units_mp, k_units_L);
|
||||
init_fastdiv_values(p.blocks_m, p.blocks_m_mp, blocks_m_L);
|
||||
init_fastdiv_values(p.blocks_n, p.blocks_n_mp, blocks_n_L);
|
||||
init_fastdiv_values(p.work_units_per_group, p.work_units_mp, work_units_L);
|
||||
init_fastdiv_values(p.work_units_per_group + 1, p.large_group_work_mp, large_group_work_L);
|
||||
p.div_Ls0 = pack_fastdiv_L(k_units_L, blocks_m_L, blocks_n_L);
|
||||
p.div_Ls1 = pack_fastdiv_L(work_units_L, large_group_work_L, 0);
|
||||
}
|
||||
|
||||
template <> inline void init_pushconst_fastdiv(vk_op_unary_push_constants &p) {
|
||||
// Compute magic values to divide by these six numbers.
|
||||
uint32_t ne0_012L;
|
||||
|
||||
@@ -259,10 +259,11 @@ struct vk_matmul_pipeline_key {
|
||||
ggml_type type_b;
|
||||
bool mul_mat_id;
|
||||
bool f16acc;
|
||||
bool stream_k = false;
|
||||
|
||||
bool operator<(const vk_matmul_pipeline_key & o) const {
|
||||
return std::tie(type_a, type_b, mul_mat_id, f16acc)
|
||||
< std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc);
|
||||
return std::tie(type_a, type_b, mul_mat_id, f16acc, stream_k)
|
||||
< std::tie(o.type_a, o.type_b, o.mul_mat_id, o.f16acc, o.stream_k);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -738,6 +739,7 @@ struct vk_device_struct {
|
||||
bool shader_int64;
|
||||
bool buffer_device_address;
|
||||
bool vulkan_memory_model;
|
||||
bool shader_clock;
|
||||
|
||||
bool add_rms_fusion;
|
||||
uint32_t partials_binding_alignment;
|
||||
@@ -746,6 +748,10 @@ struct vk_device_struct {
|
||||
bool shader_64b_indexing;
|
||||
|
||||
bool integer_dot_product;
|
||||
// Which 4x8 bit packed integer dot products are accelerated
|
||||
bool integer_dot_signed {};
|
||||
bool integer_dot_mixed {};
|
||||
bool integer_dot_unsigned {};
|
||||
// 0: default, 1: force mmvq, -1: disable mmvq
|
||||
int32_t mmvq_mode;
|
||||
|
||||
@@ -819,6 +825,7 @@ struct vk_device_struct {
|
||||
matmul_tile_selector_t matmul_id_tile_selector;
|
||||
|
||||
vk_pipeline pipeline_matmul_split_k_reduce;
|
||||
vk_pipeline pipeline_matmul_stream_k_fixup;
|
||||
vk_pipeline pipeline_quantize_q8_1_x4;
|
||||
|
||||
vk_pipeline pipeline_dequant[GGML_TYPE_COUNT];
|
||||
@@ -917,6 +924,8 @@ struct vk_device_struct {
|
||||
vk_pipeline pipeline_fill_f32;
|
||||
vk_pipeline pipeline_fill_f16;
|
||||
|
||||
vk_pipeline pipeline_sleep;
|
||||
|
||||
// [src/dst 0=fp32,1=fp16,2=bf16]
|
||||
vk_pipeline pipeline_glu[GGML_GLU_OP_COUNT][3];
|
||||
|
||||
@@ -1023,6 +1032,8 @@ struct vk_device_struct {
|
||||
std::map<std::string, uint64_t> fusion_counts;
|
||||
bool disable_descriptor_reuse;
|
||||
std::atomic<uint64_t> buffer_destroy_count {};
|
||||
bool disable_stream_k;
|
||||
bool force_stream_k;
|
||||
bool disable_host_visible_vidmem;
|
||||
bool allow_sysmem_fallback;
|
||||
bool disable_graph_optimize;
|
||||
|
||||
@@ -1726,6 +1726,11 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec
|
||||
return supported;
|
||||
}
|
||||
|
||||
// MMQ uses the mixed signedness dot product (MMQ_SUDOT) when the signed one is not accelerated
|
||||
static bool ggml_vk_mmq_use_sudot(const vk_device& device) {
|
||||
return device->integer_dot_mixed && !device->integer_dot_signed;
|
||||
}
|
||||
|
||||
static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std::vector<uint32_t>& warptile, bool mul_mat_id, ggml_type src0_type) {
|
||||
|
||||
// FLOAT_TYPE in the shader is float16_t with fp16 support, otherwise float.
|
||||
@@ -1782,7 +1787,12 @@ static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std:
|
||||
const uint32_t buf_b_size = BN * BK_STEP * block_b_size;
|
||||
const uint32_t mmid_row_ids = mul_mat_id ? (BN * 2u * (uint32_t)sizeof(uint16_t)) : 0u;
|
||||
|
||||
const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + lut_size;
|
||||
// Signed quants declare buf_b_qsum: 2 int32_t per B block with MMQ_SUDOT, a single int32_t without
|
||||
const bool b_sums = src0_type == GGML_TYPE_Q8_0 || src0_type == GGML_TYPE_MXFP4 || src0_type == GGML_TYPE_IQ4_XS ||
|
||||
src0_type == GGML_TYPE_IQ3_S || src0_type == GGML_TYPE_Q3_K || src0_type == GGML_TYPE_Q6_K;
|
||||
const uint32_t qsum_size = !b_sums ? 0u : ggml_vk_mmq_use_sudot(device) ? BN * BK_STEP * 2u * 4u : 4u;
|
||||
|
||||
const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + lut_size + qsum_size;
|
||||
const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize;
|
||||
|
||||
VK_LOG_DEBUG("ggml_vk_matmul_int_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), "
|
||||
@@ -1906,7 +1916,7 @@ static bool ggml_vk_fa_type_needs_shmem(ggml_type type) {
|
||||
|
||||
static bool ggml_vk_fa_scalar_uses_mmq(const vk_device& device, ggml_type k_type, ggml_type v_type) {
|
||||
#if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT)
|
||||
return device->integer_dot_product && device->subgroup_clustered &&
|
||||
return device->integer_dot_product && device->integer_dot_signed && device->subgroup_clustered &&
|
||||
!ggml_vk_fa_type_needs_shmem(v_type) &&
|
||||
(k_type == GGML_TYPE_Q4_0 || k_type == GGML_TYPE_Q4_1 ||
|
||||
k_type == GGML_TYPE_Q5_0 || k_type == GGML_TYPE_Q5_1 ||
|
||||
@@ -2467,6 +2477,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
FOR_EACH_LUT_FP4_TYPE(X)
|
||||
|
||||
const int mul_mat_id_param_count = 5;
|
||||
constexpr uint32_t stream_k_spec_id = 15;
|
||||
|
||||
using spec_fn_t = std::function<std::vector<uint32_t>(const std::vector<uint32_t>&, bool)>;
|
||||
auto const &create_mm_pipelines = [&](
|
||||
@@ -2478,6 +2489,11 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
bool disable_robustness = false, bool require_full_subgroups = false, uint32_t required_subgroup_size = 0,
|
||||
bool create_aligned = true, bool pin_subgroup_to_warp = false
|
||||
) {
|
||||
GGML_ASSERT(!key.stream_k || (!key.mul_mat_id && param_count == 3 && push_constant_size == sizeof(vk_mat_mat_push_constants) && create_aligned));
|
||||
// Non-ID matmul shaders declare the Stream-K scratch binding in all specializations.
|
||||
if (!key.mul_mat_id) {
|
||||
++param_count;
|
||||
}
|
||||
auto& vec = device->pipeline_matmul[key];
|
||||
const bool first_call = vec.empty();
|
||||
for (size_t i = 0; i < tile_configs.size(); i++) {
|
||||
@@ -2495,25 +2511,40 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
vk_matmul_pipeline_pair pair{};
|
||||
pair.align = tc.align;
|
||||
pair.narrow_n_max = tc.narrow_n_max;
|
||||
std::string suffix = "_" + std::to_string(i);
|
||||
pair.unaligned = std::make_shared<vk_pipeline_struct>();
|
||||
if (!key.stream_k) {
|
||||
pair.unaligned = std::make_shared<vk_pipeline_struct>();
|
||||
}
|
||||
if (create_aligned) {
|
||||
pair.aligned = std::make_shared<vk_pipeline_struct>();
|
||||
}
|
||||
if (key.stream_k) {
|
||||
// Tile selection reads unaligned even when selecting an aligned pipeline.
|
||||
pair.unaligned = pair.aligned;
|
||||
}
|
||||
vec.push_back(pair);
|
||||
}
|
||||
|
||||
ggml_vk_create_pipeline(device, vec[i].unaligned,
|
||||
vec[i].unaligned->name.empty() ? (shader_name + "_" + std::to_string(i)).c_str() : vec[i].unaligned->name.c_str(),
|
||||
spv_len, spv_data, "main", param_count, push_constant_size,
|
||||
tc.wg_denoms, spec_fn(tc.warptile, false), 1,
|
||||
disable_robustness, rfs, rsgs);
|
||||
if (!key.stream_k) {
|
||||
ggml_vk_create_pipeline(device, vec[i].unaligned,
|
||||
vec[i].unaligned->name.empty() ? (shader_name + "_" + std::to_string(i)).c_str() : vec[i].unaligned->name.c_str(),
|
||||
spv_len, spv_data, "main", param_count, push_constant_size,
|
||||
tc.wg_denoms, spec_fn(tc.warptile, false), 1,
|
||||
disable_robustness, rfs, rsgs);
|
||||
}
|
||||
|
||||
if (vec[i].aligned) {
|
||||
auto spec = spec_fn(tc.warptile, true);
|
||||
if (key.stream_k) {
|
||||
if (spec.size() <= stream_k_spec_id) {
|
||||
spec.resize(stream_k_spec_id + 1);
|
||||
}
|
||||
spec[stream_k_spec_id] = 1;
|
||||
}
|
||||
const std::string name = shader_name + (key.stream_k ? "_stream_k_" : "_aligned_") + std::to_string(i);
|
||||
ggml_vk_create_pipeline(device, vec[i].aligned,
|
||||
vec[i].aligned->name.empty() ? (shader_name + "_aligned_" + std::to_string(i)).c_str() : vec[i].aligned->name.c_str(),
|
||||
vec[i].aligned->name.empty() ? name.c_str() : vec[i].aligned->name.c_str(),
|
||||
spv_len, spv_data, "main", param_count, push_constant_size,
|
||||
tc.wg_denoms, spec_fn(tc.warptile, true), tc.align,
|
||||
tc.wg_denoms, spec, tc.align,
|
||||
disable_robustness, rfs, rsgs);
|
||||
}
|
||||
}
|
||||
@@ -2611,13 +2642,19 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
}
|
||||
|
||||
spec_fn_t cm2_spec = [&](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a); };
|
||||
auto create_cm2_pipelines = [&](vk_matmul_pipeline_key key, const std::vector<vk_tile_config>& tc,
|
||||
const std::string& name, size_t len, const void* data, uint32_t pc_size, uint32_t pc, const spec_fn_t& spec) {
|
||||
create_mm_pipelines(key, tc, name, len, data, pc_size, pc, spec, true);
|
||||
key.stream_k = true;
|
||||
create_mm_pipelines(key, tc, name, len, data, pc_size, pc, spec, true);
|
||||
};
|
||||
|
||||
// F16 x F16
|
||||
create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm2_len, matmul_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_mm_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm2_len, matmul_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_cm2_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, true}, tc_mm, "matmul_f16_f16acc", matmul_f16_f16acc_cm2_len, matmul_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
create_cm2_pipelines({GGML_TYPE_F16, GGML_TYPE_F16, false, false}, tc_mm, "matmul_f16", matmul_f16_cm2_len, matmul_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
#if defined(GGML_VULKAN_BFLOAT16_GLSLC_SUPPORT)
|
||||
if (device->coopmat_bf16_support) {
|
||||
create_mm_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm2_len, matmul_bf16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_cm2_pipelines({GGML_TYPE_BF16, GGML_TYPE_BF16, false, false}, tc_mm, "matmul_bf16", matmul_bf16_cm2_len, matmul_bf16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
}
|
||||
#endif
|
||||
for (const auto type : non_lut_quant_types) {
|
||||
@@ -2627,18 +2664,18 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
}
|
||||
auto& tc = ((type >= GGML_TYPE_Q2_K && type <= GGML_TYPE_Q6_K) || type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0) ? tc_mmq_k : tc_mmq;
|
||||
spec_fn_t qs = [&, type](const std::vector<uint32_t>& wt, bool a) { return ggml_vk_mul_mm_cm2_spec(wt, a, (uint32_t)type); };
|
||||
create_mm_pipelines({type, GGML_TYPE_F16, false, true}, tc, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm2_len, matmul_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true);
|
||||
create_mm_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16", matmul_quant_f16_cm2_len, matmul_quant_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs, true);
|
||||
create_cm2_pipelines({type, GGML_TYPE_F16, false, true}, tc, "matmul_quant_f16_f16acc", matmul_quant_f16_f16acc_cm2_len, matmul_quant_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs);
|
||||
create_cm2_pipelines({type, GGML_TYPE_F16, false, false}, tc, "matmul_quant_f16", matmul_quant_f16_cm2_len, matmul_quant_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, qs);
|
||||
}
|
||||
create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q4_k_f16_f16acc", matmul_q4_k_f16_f16acc_cm2_len, matmul_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_mm_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q4_k_f16", matmul_q4_k_f16_cm2_len, matmul_q4_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q5_k_f16_f16acc", matmul_q5_k_f16_f16acc_cm2_len, matmul_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_mm_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q5_k_f16", matmul_q5_k_f16_cm2_len, matmul_q5_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true);
|
||||
create_cm2_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q4_k_f16_f16acc", matmul_q4_k_f16_f16acc_cm2_len, matmul_q4_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
create_cm2_pipelines({GGML_TYPE_Q4_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q4_k_f16", matmul_q4_k_f16_cm2_len, matmul_q4_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
create_cm2_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, true}, tc_mmq_k, "matmul_q5_k_f16_f16acc", matmul_q5_k_f16_f16acc_cm2_len, matmul_q5_k_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
create_cm2_pipelines({GGML_TYPE_Q5_K, GGML_TYPE_F16, false, false}, tc_mmq_k, "matmul_q5_k_f16", matmul_q5_k_f16_cm2_len, matmul_q5_k_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec);
|
||||
#define X_CM2(TYPE, tstr) \
|
||||
{ auto tc = filter_tc(tc_mmq, TYPE, false); \
|
||||
if (!tc.empty()) { \
|
||||
create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm2_len, matmul_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \
|
||||
create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm2_len, matmul_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \
|
||||
create_cm2_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_f16acc", matmul_##tstr##_f16_f16acc_cm2_len, matmul_##tstr##_f16_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec); \
|
||||
create_cm2_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm2_len, matmul_##tstr##_f16_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec); \
|
||||
} }
|
||||
FOR_EACH_LUT_TYPE_NONFP4(X_CM2)
|
||||
#if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT)
|
||||
@@ -2646,8 +2683,8 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
#define X_CM2_OCP(TYPE, tstr) \
|
||||
{ auto tc = filter_tc(tc_mmq, TYPE, false); \
|
||||
if (!tc.empty()) { \
|
||||
create_mm_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_ocp_f16acc", matmul_##tstr##_f16_ocp_f16acc_cm2_len, matmul_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \
|
||||
create_mm_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16_ocp", matmul_##tstr##_f16_ocp_cm2_len, matmul_##tstr##_f16_ocp_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec, true); \
|
||||
create_cm2_pipelines({TYPE, GGML_TYPE_F16, false, true}, tc, "matmul_" #tstr "_f16_ocp_f16acc", matmul_##tstr##_f16_ocp_f16acc_cm2_len, matmul_##tstr##_f16_ocp_f16acc_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec); \
|
||||
create_cm2_pipelines({TYPE, GGML_TYPE_F16, false, false}, tc, "matmul_" #tstr "_f16_ocp", matmul_##tstr##_f16_ocp_cm2_len, matmul_##tstr##_f16_ocp_cm2_data, sizeof(vk_mat_mat_push_constants), 3, cm2_spec); \
|
||||
} }
|
||||
FOR_EACH_LUT_FP4_TYPE(X_CM2_OCP)
|
||||
#undef X_CM2_OCP
|
||||
@@ -2925,8 +2962,13 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
uint32_t rsgs = 0) {
|
||||
auto tc = filter_tc(tc_base, key.type_a, key.mul_mat_id, true);
|
||||
if (!tc.empty()) {
|
||||
spec_fn_t identity = [](const std::vector<uint32_t>& wt, bool) { return wt; };
|
||||
create_mm_pipelines(key, tc, name, len, data, pc_size, pc, identity, false, rsgs > 0, rsgs, false);
|
||||
// constant_id 11 is MMQ_SUDOT
|
||||
spec_fn_t mmq_spec = [&](const std::vector<uint32_t>& wt, bool) {
|
||||
std::vector<uint32_t> spec = wt;
|
||||
spec.push_back(ggml_vk_mmq_use_sudot(device) ? 1u : 0u);
|
||||
return spec;
|
||||
};
|
||||
create_mm_pipelines(key, tc, name, len, data, pc_size, pc, mmq_spec, false, rsgs > 0, rsgs, false);
|
||||
}
|
||||
};
|
||||
#endif
|
||||
@@ -3515,6 +3557,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_get_rows_back_f32, "get_rows_back_f32", get_rows_back_f32_len, get_rows_back_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {256, 1, 1}, {}, 1, true);
|
||||
|
||||
ggml_vk_create_pipeline(device, device->pipeline_matmul_split_k_reduce, "split_k_reduce", split_k_reduce_len, split_k_reduce_data, "main", 2, 2 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_matmul_stream_k_fixup, "stream_k_fixup", stream_k_fixup_len, stream_k_fixup_data, "main", 2, sizeof(vk_mat_mat_stream_k_fixup_push_constants), {1, 32, 1}, {}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_flash_attn_split_k_reduce, "fa_split_k_reduce", fa_split_k_reduce_len, fa_split_k_reduce_data, "main", 3, sizeof(vk_op_flash_attn_split_k_reduce_push_constants), {1, device->subgroup_size, 1}, {device->subgroup_size}, 1, true);
|
||||
|
||||
#if defined(VK_KHR_cooperative_matrix) && defined(GGML_VULKAN_COOPMAT_GLSLC_SUPPORT)
|
||||
@@ -3849,6 +3892,10 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_fill_f32, "fill_f32", fill_f32_len, fill_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_fill_f16, "fill_f16", fill_f16_len, fill_f16_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
|
||||
|
||||
if (device->shader_clock) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_sleep, "sleep", sleep_len, sleep_data, "main", 1, sizeof(vk_op_sleep_push_constants), {1, 1, 1}, {}, 1);
|
||||
}
|
||||
|
||||
#define CREATE_GLU(name, op) \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
|
||||
@@ -4478,6 +4525,7 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
bool ocp_microscaling_extension = false;
|
||||
bool shader_float8_extension = false;
|
||||
bool workgroup_memory_explicit_layout_support = false;
|
||||
bool shader_clock_support = false;
|
||||
|
||||
for (const auto& properties : ext_props) {
|
||||
if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) {
|
||||
@@ -4528,6 +4576,9 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
} else if (strcmp(VK_EXT_SHADER_FLOAT8_EXTENSION_NAME, properties.extensionName) == 0) {
|
||||
shader_float8_extension = true;
|
||||
#endif
|
||||
} else if (strcmp("VK_KHR_shader_clock", properties.extensionName) == 0 &&
|
||||
!getenv("GGML_VK_DISABLE_SHADER_CLOCK")) {
|
||||
shader_clock_support = true;
|
||||
} else if (strcmp("VK_VALVE_shader_mixed_float_dot_product", properties.extensionName) == 0 &&
|
||||
!getenv("GGML_VK_DISABLE_DOT2")) {
|
||||
dot2_f16_support = true;
|
||||
@@ -4716,7 +4767,10 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
device->coopmat_support = false;
|
||||
}
|
||||
|
||||
device->integer_dot_product = device->integer_dot_product && shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated;
|
||||
device->integer_dot_signed = shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated;
|
||||
device->integer_dot_mixed = shader_integer_dot_product_props.integerDotProduct4x8BitPackedMixedSignednessAccelerated;
|
||||
device->integer_dot_unsigned = shader_integer_dot_product_props.integerDotProduct4x8BitPackedUnsignedAccelerated;
|
||||
device->integer_dot_product = device->integer_dot_product && (device->integer_dot_signed || device->integer_dot_mixed);
|
||||
|
||||
device->min_imported_host_pointer_alignment = external_memory_host_props.minImportedHostPointerAlignment;
|
||||
|
||||
@@ -4873,6 +4927,14 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
device_extensions.push_back("VK_KHR_shader_integer_dot_product");
|
||||
}
|
||||
|
||||
VkPhysicalDeviceShaderClockFeaturesKHR shader_clock_features {};
|
||||
shader_clock_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CLOCK_FEATURES_KHR;
|
||||
if (shader_clock_support) {
|
||||
last_struct->pNext = (VkBaseOutStructure *)&shader_clock_features;
|
||||
last_struct = (VkBaseOutStructure *)&shader_clock_features;
|
||||
device_extensions.push_back("VK_KHR_shader_clock");
|
||||
}
|
||||
|
||||
VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE dot2_features {};
|
||||
dot2_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE;
|
||||
if (dot2_f16_support) {
|
||||
@@ -4979,6 +5041,7 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
device->shader_int64 = device_features2.features.shaderInt64;
|
||||
device->buffer_device_address = vk12_features.bufferDeviceAddress;
|
||||
device->vulkan_memory_model = vk12_features.vulkanMemoryModel;
|
||||
device->shader_clock = shader_clock_support && shader_clock_features.shaderDeviceClock;
|
||||
|
||||
if (device->subgroup_size_control) {
|
||||
device->subgroup_min_size = subgroup_size_control_props.minSubgroupSize;
|
||||
@@ -5375,6 +5438,8 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
device->serialize_submissions = getenv("GGML_VK_SERIALIZE_SUBMISSIONS") != nullptr;
|
||||
|
||||
device->disable_fusion = getenv("GGML_VK_DISABLE_FUSION") != nullptr;
|
||||
device->disable_stream_k = getenv("GGML_VK_DISABLE_STREAM_K") != nullptr;
|
||||
device->force_stream_k = getenv("GGML_VK_FORCE_STREAM_K") != nullptr;
|
||||
|
||||
device->disable_descriptor_reuse = getenv("GGML_VK_DISABLE_DESCRIPTOR_REUSE") != nullptr;
|
||||
|
||||
@@ -5593,7 +5658,8 @@ static void ggml_vk_print_gpu_info(size_t idx) {
|
||||
const bool rebar = ggml_vk_has_rebar(physical_device);
|
||||
|
||||
integer_dot_product = integer_dot_product
|
||||
&& shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated
|
||||
&& (shader_integer_dot_product_props.integerDotProduct4x8BitPackedSignedAccelerated ||
|
||||
shader_integer_dot_product_props.integerDotProduct4x8BitPackedMixedSignednessAccelerated)
|
||||
&& shader_integer_dot_product_features.shaderIntegerDotProduct;
|
||||
|
||||
coopmat_support = coopmat_support
|
||||
@@ -6408,14 +6474,113 @@ static uint32_t ggml_vk_guess_split_k(ggml_backend_vk_context * ctx, uint32_t m,
|
||||
return split_k;
|
||||
}
|
||||
|
||||
static uint32_t ggml_vk_guess_stream_k_groups(
|
||||
ggml_backend_vk_context * ctx, const vk_pipeline & pipeline,
|
||||
uint32_t m, uint32_t n, uint32_t k, uint32_t stride_d, uint32_t batch, uint32_t split_k, bool disable_split_k) {
|
||||
if (disable_split_k) {
|
||||
return 0;
|
||||
}
|
||||
if (ctx->device->disable_stream_k) {
|
||||
return 0;
|
||||
}
|
||||
// TODO: Tune the CM1 and scalar workgroup counts before enabling Stream-K.
|
||||
if (!ctx->device->coopmat2) {
|
||||
return 0;
|
||||
}
|
||||
if (ctx->device->shader_core_count == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const uint32_t bm = pipeline->wg_denoms[0];
|
||||
const uint32_t bn = pipeline->wg_denoms[1];
|
||||
if (bm % 32 != 0 || m % bm != 0 || n % bn != 0 || k % 256 != 0 || stride_d != m) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const uint32_t k_units = k / 256;
|
||||
if (k_units == 1) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const uint64_t tiles = uint64_t(m / bm) * (n / bn) * batch;
|
||||
const uint64_t total_work_units = tiles * k_units;
|
||||
if (total_work_units > UINT32_MAX) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Coopmat2 tiles are tuned for one resident workgroup per SM.
|
||||
const uint32_t target_groups = ctx->device->shader_core_count;
|
||||
const uint64_t regular_groups = tiles * split_k;
|
||||
const uint64_t regular_waves = CEIL_DIV(regular_groups, target_groups);
|
||||
const uint64_t regular_capacity = regular_waves * target_groups;
|
||||
|
||||
const uint32_t stream_k_groups = std::min<uint64_t>(target_groups, total_work_units);
|
||||
if (stream_k_groups > ctx->device->properties.limits.maxComputeWorkGroupCount[0] ||
|
||||
bm / 32 > ctx->device->properties.limits.maxComputeWorkGroupCount[1]) {
|
||||
return 0;
|
||||
}
|
||||
if (!ctx->device->force_stream_k) {
|
||||
if (regular_waves == 1 && stream_k_groups == regular_groups) {
|
||||
return 0;
|
||||
}
|
||||
// Stream-K does not offset its overhead when the regular grid fills most of the last wave.
|
||||
// Use a lower cutoff for short K with no K split and narrow N tiles.
|
||||
if ((split_k == 1 && k <= 4096 && bn <= 128 && regular_groups * 4 >= regular_capacity * 3) ||
|
||||
regular_groups * 10 >= regular_capacity * 9) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
return stream_k_groups;
|
||||
}
|
||||
|
||||
void ggml_vk_matmul(
|
||||
ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline,
|
||||
vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& split_k_buffer,
|
||||
uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d,
|
||||
uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d,
|
||||
uint32_t split_k, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3,
|
||||
uint32_t split_k, uint32_t stream_k_groups, uint32_t batch, uint32_t ne02, uint32_t ne12, uint32_t broadcast2, uint32_t broadcast3,
|
||||
uint32_t padded_n, uint64_t b_col_bytes, uint64_t d_col_bytes) {
|
||||
VK_LOG_DEBUG("ggml_vk_matmul(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), split_k: (" << (split_k_buffer.buffer != nullptr ? split_k_buffer.buffer->buffer : VK_NULL_HANDLE) << ", " << split_k_buffer.offset << ", " << split_k_buffer.size << "), m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", split_k: " << split_k << ", batch: " << batch << ", ne02: " << ne02 << ", ne12: " << ne12 << ", broadcast2: " << broadcast2 << ", broadcast3: " << broadcast3 << ", padded_n: " << padded_n << ")");
|
||||
if (stream_k_groups != 0) {
|
||||
GGML_ASSERT(batch_stride_d == m * n);
|
||||
if (ctx->prealloc_split_k_need_sync) {
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
}
|
||||
|
||||
ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
|
||||
ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_stream_k_fixup, 1);
|
||||
|
||||
const uint32_t bm = pipeline->wg_denoms[0];
|
||||
const uint32_t bn = pipeline->wg_denoms[1];
|
||||
const uint32_t k_units_per_tile = k / 256;
|
||||
const uint32_t blocks_m = m / bm;
|
||||
const uint32_t blocks_n = n / bn;
|
||||
const uint32_t total_work_units = blocks_m * blocks_n * batch * k_units_per_tile;
|
||||
const uint32_t work_units_per_group = total_work_units / stream_k_groups;
|
||||
const uint32_t groups_with_extra_unit = total_work_units % stream_k_groups;
|
||||
|
||||
vk_mat_mat_push_constants pc1 = {
|
||||
m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, 0, batch, k, ne02, ne12, broadcast2, broadcast3, padded_n,
|
||||
k_units_per_tile, work_units_per_group, groups_with_extra_unit,
|
||||
0, 0, 0, 0, 0, 0, 0, 0
|
||||
};
|
||||
init_pushconst_fastdiv(pc1, blocks_m, blocks_n);
|
||||
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, split_k_buffer }, pc1, { stream_k_groups * pipeline->wg_denoms[0], 1, 1 });
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
|
||||
vk_mat_mat_stream_k_fixup_push_constants pc2 = {
|
||||
m, n, bm, bn, k_units_per_tile, blocks_m, blocks_n, work_units_per_group, groups_with_extra_unit,
|
||||
0, 0, 0, 0, 0, 0, 0
|
||||
};
|
||||
init_pushconst_fastdiv(pc2);
|
||||
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, ctx->device->pipeline_matmul_stream_k_fixup, { split_k_buffer, d }, pc2, { stream_k_groups, bm, 1 });
|
||||
ctx->prealloc_split_k_need_sync = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (split_k == 1) {
|
||||
// a small tile can still produce more workgroups than the device allows on the n axis:
|
||||
// split the dispatch into n-chunks, shifting the B and D column windows
|
||||
@@ -6438,8 +6603,12 @@ void ggml_vk_matmul(
|
||||
while (base_work_group_z < batch) {
|
||||
uint32_t groups_z = std::min(batch - base_work_group_z, ctx->device->properties.limits.maxComputeWorkGroupCount[2]);
|
||||
|
||||
const vk_mat_mat_push_constants pc = { m, n_chunk, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k, ne02, ne12, broadcast2, broadcast3, uint32_t(padded_n - n_base) };
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b_chunk, d_chunk }, pc, { m, n_chunk, groups_z });
|
||||
vk_mat_mat_push_constants pc = {
|
||||
m, n_chunk, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k, ne02, ne12, broadcast2, broadcast3, uint32_t(padded_n - n_base),
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
||||
};
|
||||
init_pushconst_fastdiv(pc);
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b_chunk, d_chunk, d_chunk }, pc, { m, n_chunk, groups_z });
|
||||
base_work_group_z += groups_z;
|
||||
}
|
||||
}
|
||||
@@ -6462,9 +6631,13 @@ void ggml_vk_matmul(
|
||||
while (base_work_group_z < batch) {
|
||||
uint32_t groups_z = std::min(batch - base_work_group_z, ctx->device->properties.limits.maxComputeWorkGroupCount[2]);
|
||||
|
||||
const vk_mat_mat_push_constants pc1 = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k_split, ne02, ne12, broadcast2, broadcast3, padded_n };
|
||||
vk_mat_mat_push_constants pc1 = {
|
||||
m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d, base_work_group_z, batch, k_split, ne02, ne12, broadcast2, broadcast3, padded_n,
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
|
||||
};
|
||||
init_pushconst_fastdiv(pc1);
|
||||
// Make sure enough workgroups get assigned for split k to work
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, split_k_buffer }, pc1, { (CEIL_DIV(m, pipeline->wg_denoms[0]) * pipeline->wg_denoms[0]) * split_k, n, groups_z });
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, split_k_buffer, split_k_buffer }, pc1, { (CEIL_DIV(m, pipeline->wg_denoms[0]) * pipeline->wg_denoms[0]) * split_k, n, groups_z });
|
||||
base_work_group_z += groups_z;
|
||||
}
|
||||
ggml_vk_sync_buffers(ctx, subctx);
|
||||
@@ -6518,10 +6691,11 @@ static bool ggml_vk_get_mul_mat_mat_f16acc(ggml_backend_vk_context * ctx, ggml_t
|
||||
return ctx->device->fp16 && prec_allows_f16;
|
||||
}
|
||||
|
||||
static const std::vector<vk_matmul_pipeline_pair>* ggml_vk_get_mul_mat_mat_pipeline_map(
|
||||
ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec, bool mul_mat_id = false) {
|
||||
static const std::vector<vk_matmul_pipeline_pair> * ggml_vk_get_mul_mat_mat_pipeline_map(
|
||||
ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec,
|
||||
bool mul_mat_id = false, bool stream_k = false) {
|
||||
bool f16acc = ggml_vk_get_mul_mat_mat_f16acc(ctx, src0_type, src1_type, prec);
|
||||
vk_matmul_pipeline_key key{src0_type, src1_type, mul_mat_id, f16acc};
|
||||
vk_matmul_pipeline_key key{src0_type, src1_type, mul_mat_id, f16acc, stream_k};
|
||||
auto it = ctx->device->pipeline_matmul.find(key);
|
||||
if (it == ctx->device->pipeline_matmul.end() || it->second.empty()) {
|
||||
// Try without f16acc
|
||||
@@ -6894,25 +7068,50 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
|
||||
const bool y_f32_kernel = y_kernel_type == GGML_TYPE_F32;
|
||||
const bool y_needs_reformat = !quantize_y && (!ggml_vk_dim01_contiguous(src1) || y_kernel_type != src1->type);
|
||||
|
||||
const std::vector<vk_matmul_pipeline_pair>* mmp_map =
|
||||
ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, ggml_prec_get_acc(dst));
|
||||
ggml_type mmp_type_a = src0->type;
|
||||
const ggml_type mmp_type_b = y_kernel_type;
|
||||
const std::vector<vk_matmul_pipeline_pair> * mmp_map =
|
||||
ggml_vk_get_mul_mat_mat_pipeline_map(ctx, mmp_type_a, mmp_type_b, ggml_prec_get_acc(dst));
|
||||
|
||||
const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig;
|
||||
const bool qy_needs_dequant = y_needs_reformat;
|
||||
|
||||
if (qx_needs_dequant) {
|
||||
// dequant src0 to f16
|
||||
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, ggml_prec_get_acc(dst));
|
||||
mmp_type_a = f16_type;
|
||||
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, mmp_type_a, mmp_type_b, ggml_prec_get_acc(dst));
|
||||
}
|
||||
|
||||
GGML_ASSERT(mmp_map != nullptr);
|
||||
|
||||
const std::vector<vk_matmul_pipeline_pair> * regular_mmp_map = mmp_map;
|
||||
bool use_stream_k = ctx->device->coopmat2 && !ctx->device->disable_stream_k && !disable_split_k;
|
||||
if (use_stream_k) {
|
||||
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, mmp_type_a, mmp_type_b, ggml_prec_get_acc(dst), false, true);
|
||||
if (mmp_map == nullptr) {
|
||||
mmp_map = regular_mmp_map;
|
||||
use_stream_k = false;
|
||||
}
|
||||
}
|
||||
|
||||
const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, ne11, false));
|
||||
const bool aligned = !quantize_y && ne10 == kpad && ne01 > 8 && ne11 > 8;
|
||||
const bool needs_64b_indexing = ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange;
|
||||
if (use_stream_k && !aligned) {
|
||||
mmp_map = regular_mmp_map;
|
||||
use_stream_k = false;
|
||||
}
|
||||
|
||||
vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, ne11, aligned, false);
|
||||
|
||||
if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) {
|
||||
const bool narrow_n = mmp_map->back().narrow_n_max != 0 && ne11 <= mmp_map->back().narrow_n_max;
|
||||
const uint32_t split_k = ggml_vk_guess_split_k(ctx, ne01, ne11, ne10, disable_split_k, pipeline, narrow_n);
|
||||
const uint32_t stream_k_groups = use_stream_k ? ggml_vk_guess_stream_k_groups(ctx, pipeline, ne01, ne11, ne10, stride_d, ne12 * ne13, split_k, disable_split_k) : 0;
|
||||
if (use_stream_k && stream_k_groups == 0) {
|
||||
pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *regular_mmp_map, ne01, ne11, aligned, false);
|
||||
}
|
||||
|
||||
if (needs_64b_indexing) {
|
||||
pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline);
|
||||
}
|
||||
|
||||
@@ -6923,9 +7122,6 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
|
||||
const uint64_t y_ne = padded_n * ne10 * ne12 * ne13;
|
||||
const uint64_t d_ne = ggml_nelements(dst);
|
||||
|
||||
const bool narrow_n = mmp_map->back().narrow_n_max != 0 && ne11 <= mmp_map->back().narrow_n_max;
|
||||
const uint32_t split_k = ggml_vk_guess_split_k(ctx, ne01, ne11, ne10, disable_split_k, pipeline, narrow_n);
|
||||
|
||||
const uint64_t qx_sz = ggml_type_size(src0->type) * x_ne / ggml_blck_size(src0->type);
|
||||
const uint64_t qy_sz = ggml_type_size(src1->type) * y_ne / ggml_blck_size(src1->type);
|
||||
const uint64_t x_sz = !qx_needs_dequant ? qx_sz : sizeof(ggml_fp16_t) * x_ne;
|
||||
@@ -6953,12 +7149,13 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
|
||||
to_q8_1 = ggml_vk_get_quantize_pipeline(ctx, GGML_TYPE_Q8_1);
|
||||
}
|
||||
|
||||
const uint64_t stream_k_size = stream_k_groups != 0 ? uint64_t(stream_k_groups) * pipeline->wg_denoms[0] * pipeline->wg_denoms[1] * sizeof(float) : 0;
|
||||
const uint64_t split_k_size = stream_k_groups != 0 ? stream_k_size : (split_k > 1 ? d_sz * split_k : 0);
|
||||
{
|
||||
const uint64_t split_k_size = split_k > 1 ? d_sz * split_k : 0;
|
||||
if (
|
||||
(qx_needs_dequant && x_sz > ctx->device->properties.limits.maxStorageBufferRange) ||
|
||||
(qy_needs_dequant && y_sz > ctx->device->properties.limits.maxStorageBufferRange) ||
|
||||
(split_k > 1 && split_k_size > ctx->device->properties.limits.maxStorageBufferRange)) {
|
||||
(split_k_size > ctx->device->properties.limits.maxStorageBufferRange)) {
|
||||
GGML_ABORT("Requested preallocation size is too large");
|
||||
}
|
||||
if (qx_needs_dequant && ctx->prealloc_size_x < x_sz) {
|
||||
@@ -6969,7 +7166,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
|
||||
ctx->prealloc_size_y = y_sz;
|
||||
ggml_vk_preallocate_buffers(ctx, subctx);
|
||||
}
|
||||
if (split_k > 1 && ctx->prealloc_size_split_k < split_k_size) {
|
||||
if (split_k_size != 0 && ctx->prealloc_size_split_k < split_k_size) {
|
||||
ctx->prealloc_size_split_k = split_k_size;
|
||||
ggml_vk_preallocate_buffers(ctx, subctx);
|
||||
}
|
||||
@@ -6984,7 +7181,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
|
||||
if (quantize_y) {
|
||||
ggml_pipeline_request_descriptor_sets(ctx, to_q8_1, 1);
|
||||
}
|
||||
if (split_k > 1) {
|
||||
if (stream_k_groups == 0 && split_k > 1) {
|
||||
ggml_pipeline_request_descriptor_sets(ctx, ctx->device->pipeline_matmul_split_k_reduce, 1);
|
||||
}
|
||||
}
|
||||
@@ -7092,10 +7289,10 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
|
||||
ggml_vk_matmul(
|
||||
ctx, subctx, pipeline,
|
||||
{ d_X, x_buf_offset, x_range }, { d_Y, y_buf_offset, y_range },
|
||||
ggml_vk_subbuffer(ctx, d_D, d_buf_offset), { ctx->prealloc_split_k, 0, d_sz * split_k },
|
||||
ggml_vk_subbuffer(ctx, d_D, d_buf_offset), { ctx->prealloc_split_k, 0, split_k_size },
|
||||
ne01, ne11, ne10,
|
||||
ne10, ne10, stride_d, stride_batch_x, stride_batch_y, stride_batch_d,
|
||||
split_k, ne12*ne13, ne02, ne12, r2, r3, padded_n,
|
||||
split_k, stream_k_groups, ne12*ne13, ne02, ne12, r2, r3, padded_n,
|
||||
b_col_bytes, (uint64_t)dst->nb[1]
|
||||
); // NOLINT
|
||||
|
||||
@@ -7234,7 +7431,7 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context&
|
||||
|
||||
const ggml_type y_kernel_type = widen_bf16 ? GGML_TYPE_F32 : src1->type;
|
||||
const bool f16_f32_kernel = y_kernel_type == GGML_TYPE_F32;
|
||||
bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && (swap_inputs || ggml_vk_src1_prec_allows_q8(dst)) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type);
|
||||
bool quantize_y = ctx->device->integer_dot_signed && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && (swap_inputs || ggml_vk_src1_prec_allows_q8(dst)) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne11, ne10, src0->type);
|
||||
|
||||
vk_pipeline to_fp16_vk_0 = nullptr;
|
||||
vk_pipeline to_fp16_vk_1 = nullptr;
|
||||
@@ -8302,7 +8499,7 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte
|
||||
const bool y_non_contig = !ggml_vk_dim01_contiguous(src1);
|
||||
|
||||
const bool f16_f32_kernel = src1->type == GGML_TYPE_F32;
|
||||
bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type);
|
||||
bool quantize_y = ctx->device->integer_dot_signed && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0 && ggml_vk_src1_prec_allows_q8(dst) && ggml_vk_should_use_mmvq(ctx->device, ne01, ne12, ne10, src0->type);
|
||||
|
||||
vk_pipeline to_fp16_vk_0 = nullptr;
|
||||
vk_pipeline to_fp16_vk_1 = nullptr;
|
||||
@@ -10055,6 +10252,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
return ctx->device->pipeline_arange_f32;
|
||||
}
|
||||
return nullptr;
|
||||
case GGML_OP_SLEEP:
|
||||
return ctx->device->pipeline_sleep;
|
||||
case GGML_OP_FILL:
|
||||
if (dst->type == GGML_TYPE_F32) {
|
||||
return ctx->device->pipeline_fill_f32;
|
||||
@@ -11291,6 +11490,31 @@ void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements);
|
||||
}
|
||||
|
||||
static void ggml_vk_sleep(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
|
||||
VK_LOG_DEBUG("ggml_vk_sleep(dst=" << dst << ", us=" << ggml_get_op_params_i32(dst, 0) << ")");
|
||||
|
||||
// Vulkan does not specify the period of the shader realtime clock this assumes it matches the timestamp counter.
|
||||
// A driver where this does not hold could sleep too short or too long by that ratio
|
||||
const float ns_per_tick = ctx->device->properties.limits.timestampPeriod;
|
||||
const uint64_t ns = 1000 * (uint64_t) ggml_get_op_params_i32(dst, 0);
|
||||
const uint64_t ticks = std::min<uint64_t>(ns_per_tick > 0.0f ? (uint64_t)(ns / ns_per_tick) : ns, std::numeric_limits<uint32_t>::max());
|
||||
|
||||
vk_op_sleep_push_constants pc = {
|
||||
(uint32_t) ticks,
|
||||
};
|
||||
|
||||
vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, nullptr, nullptr, nullptr, dst, GGML_OP_SLEEP);
|
||||
GGML_ASSERT(pipeline != nullptr);
|
||||
|
||||
ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
|
||||
vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst);
|
||||
|
||||
// a single invocation spins, the buffer is bound only to keep the spin from being optimized out
|
||||
std::array<uint32_t, 3> elements = { 1, 1, 1 };
|
||||
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements);
|
||||
}
|
||||
|
||||
void ggml_vk_sin(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
|
||||
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SIN, vk_op_unary_push_constants_init(src0, dst));
|
||||
}
|
||||
@@ -13137,6 +13361,10 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in
|
||||
case GGML_OP_FILL:
|
||||
ggml_vk_fill(ctx, compute_ctx, node);
|
||||
|
||||
break;
|
||||
case GGML_OP_SLEEP:
|
||||
ggml_vk_sleep(ctx, compute_ctx, node);
|
||||
|
||||
break;
|
||||
case GGML_OP_SCALE:
|
||||
if (ctx->fused_hc_post_gate) {
|
||||
@@ -16501,8 +16729,9 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
case GGML_OP_REPEAT_BACK:
|
||||
return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32;
|
||||
case GGML_OP_ROPE:
|
||||
return ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0]);
|
||||
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0]);
|
||||
case GGML_OP_ROPE_BACK:
|
||||
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
|
||||
case GGML_OP_NONE:
|
||||
case GGML_OP_RESHAPE:
|
||||
case GGML_OP_VIEW:
|
||||
@@ -16588,6 +16817,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
return op->type == GGML_TYPE_F32;
|
||||
case GGML_OP_FILL:
|
||||
return op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;
|
||||
case GGML_OP_SLEEP:
|
||||
return device->shader_clock;
|
||||
case GGML_OP_SCALE:
|
||||
return ggml_is_contiguous(op->src[0]) && ggml_vk_is_float_elementwise(op);
|
||||
case GGML_OP_PAD:
|
||||
|
||||
@@ -22,6 +22,6 @@ void main() {
|
||||
if (i10 == i11) {
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(data_a[get_aoffset() + i13*p.nb03 + i12*p.nb02 + 0*p.nb01 + i10*p.nb00]);
|
||||
} else {
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0);
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0.0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
#version 450
|
||||
|
||||
#include "utils.glsl"
|
||||
|
||||
layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
layout (binding = 0) readonly buffer S4 {vec4 data_s4[];};
|
||||
layout (binding = 1) buffer D4 {vec4 data_d4[];};
|
||||
|
||||
layout (push_constant) uniform parameter {
|
||||
uint M;
|
||||
uint N;
|
||||
uint BM;
|
||||
uint BN;
|
||||
uint k_units_per_tile;
|
||||
uint blocks_m;
|
||||
uint blocks_n;
|
||||
uint work_units_per_group;
|
||||
uint groups_with_extra_unit;
|
||||
uint k_units_mp;
|
||||
uint blocks_m_mp;
|
||||
uint blocks_n_mp;
|
||||
uint work_units_mp;
|
||||
uint large_group_work_mp;
|
||||
uint div_Ls0;
|
||||
uint div_Ls1;
|
||||
} p;
|
||||
|
||||
const uint RESOLVE_ROWS = 32;
|
||||
|
||||
// A normal matmul workgroup accumulates one BM x BN output tile over the full K range.
|
||||
// Stream-K divides that accumulation into work units. Each work unit covers 256 consecutive K values and produces a BM x BN partial sum.
|
||||
// Work units are ordered [K unit, M tile, N tile, batch] and divided into one contiguous range per main workgroup.
|
||||
// A main workgroup writes its partial sum to D when it reaches the end of an output tile, or to scratch when its range stops within a tile.
|
||||
// Scratch has shape [BM, BN, stream_k_groups] and stores float partial sums.
|
||||
// Element [r, c, g] is workgroup g's contribution to local output element [r, c] in the last output tile of its range.
|
||||
// The output tile for each scratch slot is derived from the workgroup's range and is not stored in scratch.
|
||||
// Only the last output tile in a workgroup's range can stop early, so one scratch slot is enough.
|
||||
// Fixup workgroup (x, y) handles main workgroup x and rows 32*y through 32*y+31.
|
||||
// If main workgroup x starts within a tile and completes it, fixup adds the earlier groups' partial sums from scratch to D.
|
||||
|
||||
// Find the main workgroup whose range contains work.
|
||||
// With q = work_units_per_group and r = groups_with_extra_unit, the first r workgroups get q + 1 units and all others get q.
|
||||
// group(work) = work / (q + 1) when work < r*(q + 1), otherwise r + (work - r*(q + 1))/q.
|
||||
uint group_for_work(uint work) {
|
||||
const uint large_group_work = p.work_units_per_group + 1;
|
||||
const uint large_work = large_group_work * p.groups_with_extra_unit;
|
||||
if (work < large_work) {
|
||||
return fastdiv(work, p.large_group_work_mp, fastdiv_L(p.div_Ls1, 1));
|
||||
}
|
||||
return p.groups_with_extra_unit + fastdiv(work - large_work, p.work_units_mp, fastdiv_L(p.div_Ls1, 0));
|
||||
}
|
||||
|
||||
void main() {
|
||||
const uint group_idx = gl_WorkGroupID.x;
|
||||
const uint work_unit = group_idx * p.work_units_per_group + min(group_idx, p.groups_with_extra_unit);
|
||||
const uint work_unit_end = work_unit + p.work_units_per_group + uint(group_idx < p.groups_with_extra_unit);
|
||||
|
||||
const uint tile_idx = fastdiv(work_unit, p.k_units_mp, fastdiv_L(p.div_Ls0, 0));
|
||||
const uint tile_work = tile_idx * p.k_units_per_tile;
|
||||
// Only a range that starts within a tile and completes it needs fixup.
|
||||
if (work_unit == tile_work || work_unit_end < tile_work + p.k_units_per_tile) {
|
||||
return;
|
||||
}
|
||||
|
||||
const uint first_group = group_for_work(tile_work);
|
||||
const uint outer_tile = fastdiv(tile_idx, p.blocks_m_mp, fastdiv_L(p.div_Ls0, 1));
|
||||
const uint ir = tile_idx - outer_tile * p.blocks_m;
|
||||
const uint batch_idx = fastdiv(outer_tile, p.blocks_n_mp, fastdiv_L(p.div_Ls0, 2));
|
||||
const uint ic = outer_tile - batch_idx * p.blocks_n;
|
||||
const uint row_base = gl_WorkGroupID.y * RESOLVE_ROWS;
|
||||
|
||||
for (uint stripe_elem = gl_LocalInvocationIndex * 4; stripe_elem < RESOLVE_ROWS * p.BN; stripe_elem += gl_WorkGroupSize.x * 4) {
|
||||
const uint r = row_base + (stripe_elem & (RESOLVE_ROWS - 1));
|
||||
const uint c = stripe_elem / RESOLVE_ROWS;
|
||||
const uint elem_idx = c * p.BM + r;
|
||||
const uint dst_idx = batch_idx * p.M * p.N + (ic * p.BN + c) * p.M + ir * p.BM + r;
|
||||
|
||||
// D contains the completing segment. Add the earlier segments from scratch.
|
||||
vec4 result = data_d4[dst_idx / 4];
|
||||
for (uint previous_group = first_group; previous_group < group_idx; ++previous_group) {
|
||||
result += data_s4[(previous_group * p.BM * p.BN + elem_idx) / 4];
|
||||
}
|
||||
data_d4[dst_idx / 4] = result;
|
||||
}
|
||||
}
|
||||
@@ -38,6 +38,7 @@ layout (constant_id = 12) const uint MmTypeA = 0;
|
||||
|
||||
#include "types.glsl"
|
||||
#include "dot_product_funcs.glsl"
|
||||
#include "utils.glsl"
|
||||
|
||||
#ifndef MULMAT_QUANT
|
||||
#ifndef LOAD_VEC_A
|
||||
@@ -121,6 +122,9 @@ layout (binding = 0) readonly buffer BUF_Q5_K_P32 { block_q5_K_packed32 data[]
|
||||
layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
|
||||
layout (binding = 1) readonly buffer B_SCALAR {B_TYPE_SCALAR data_b_scalar[];};
|
||||
layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
|
||||
#ifndef MUL_MAT_ID
|
||||
layout (binding = 3) writeonly buffer S {D_TYPE data_s[];};
|
||||
#endif
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
layout (binding = 3) readonly buffer IDS {int data_ids[];};
|
||||
@@ -155,6 +159,18 @@ layout (push_constant) uniform parameter
|
||||
uint ne12;
|
||||
uint broadcast2;
|
||||
uint broadcast3;
|
||||
uint padded_N;
|
||||
uint k_units_per_tile;
|
||||
uint work_units_per_group;
|
||||
uint groups_with_extra_unit;
|
||||
uint k_units_mp;
|
||||
uint blocks_m_mp;
|
||||
uint blocks_n_mp;
|
||||
uint ne12_mp;
|
||||
uint broadcast2_mp;
|
||||
uint broadcast3_mp;
|
||||
uint div_Ls0;
|
||||
uint div_Ls1;
|
||||
#endif
|
||||
} p;
|
||||
|
||||
@@ -189,6 +205,9 @@ layout(constant_id = 13) const bool APPLY_SLM_A_RESHAPE = false;
|
||||
const uint SHMEM_STRIDE_PAD = 1;
|
||||
const bool APPLY_SLM_A_RESHAPE = false;
|
||||
#endif
|
||||
#ifndef MUL_MAT_ID
|
||||
layout(constant_id = 15) const bool stream_k = false;
|
||||
#endif
|
||||
#define SHMEM_STRIDE (BK / 2 + SHMEM_STRIDE_PAD)
|
||||
|
||||
shared FLOAT_TYPEV2 buf_a[BM * SHMEM_STRIDE];
|
||||
@@ -218,27 +237,24 @@ void main() {
|
||||
_ne1 = data_row_map[2 + 2 * tile_idx];
|
||||
const uint ic = 0;
|
||||
#else
|
||||
const uint ic = gl_WorkGroupID.y;
|
||||
const uint group_idx = gl_WorkGroupID.x;
|
||||
uint stream_k_work_unit;
|
||||
uint stream_k_work_unit_end;
|
||||
|
||||
if (stream_k) {
|
||||
stream_k_work_unit = group_idx * p.work_units_per_group + min(group_idx, p.groups_with_extra_unit);
|
||||
stream_k_work_unit_end = stream_k_work_unit + p.work_units_per_group + uint(group_idx < p.groups_with_extra_unit);
|
||||
}
|
||||
#endif
|
||||
#if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT)
|
||||
init_iq_shmem(gl_WorkGroupSize);
|
||||
#endif
|
||||
|
||||
#ifndef MUL_MAT_ID
|
||||
const uint batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
|
||||
|
||||
const uint i13 = batch_idx / p.ne12;
|
||||
const uint i12 = batch_idx % p.ne12;
|
||||
|
||||
const uint i03 = i13 / p.broadcast3;
|
||||
const uint i02 = i12 / p.broadcast2;
|
||||
|
||||
const uint batch_idx_a = i03 * p.ne02 + i02;
|
||||
#endif
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
const uint blocks_m = (p.M + BM - 1) / BM;
|
||||
const uint ir = gl_WorkGroupID.x % blocks_m;
|
||||
const uint ik = gl_WorkGroupID.x / blocks_m;
|
||||
#endif
|
||||
|
||||
const uint WNITER = (WM * WN) / (WARP * TM * TN * WMITER);
|
||||
const uint WSUBM = WM / WMITER;
|
||||
@@ -288,6 +304,41 @@ void main() {
|
||||
const uint loadstride_a = gl_WorkGroupSize.x * LOAD_VEC_A_EFF * LOAD_VEC_BATCH_A / BK;
|
||||
const uint loadstride_b = gl_WorkGroupSize.x * LOAD_VEC_B_EFF * LOAD_VEC_BATCH_B / BK;
|
||||
|
||||
#ifndef MUL_MAT_ID
|
||||
do {
|
||||
uint stream_k_tile_k_unit;
|
||||
uint stream_k_tile_k_units;
|
||||
uint ir;
|
||||
uint ic;
|
||||
uint batch_idx;
|
||||
uint ik;
|
||||
|
||||
if (stream_k) {
|
||||
const uint tile_idx = fastdiv(stream_k_work_unit, p.k_units_mp, fastdiv_L(p.div_Ls0, 0));
|
||||
stream_k_tile_k_unit = stream_k_work_unit - tile_idx * p.k_units_per_tile;
|
||||
stream_k_tile_k_units = min(stream_k_work_unit_end - stream_k_work_unit, p.k_units_per_tile - stream_k_tile_k_unit);
|
||||
|
||||
const uint blocks_m = p.M / BM;
|
||||
const uint blocks_n = p.N / BN;
|
||||
const uint outer_tile = fastdiv(tile_idx, p.blocks_m_mp, fastdiv_L(p.div_Ls0, 1));
|
||||
ir = tile_idx - outer_tile * blocks_m;
|
||||
batch_idx = fastdiv(outer_tile, p.blocks_n_mp, fastdiv_L(p.div_Ls0, 2));
|
||||
ic = outer_tile - batch_idx * blocks_n;
|
||||
} else {
|
||||
const uint blocks_m = (p.M + BM - 1) / BM;
|
||||
ic = gl_WorkGroupID.y;
|
||||
batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
|
||||
ir = group_idx % blocks_m;
|
||||
ik = group_idx / blocks_m;
|
||||
}
|
||||
|
||||
const uint i13 = fastdiv(batch_idx, p.ne12_mp, fastdiv_L(p.div_Ls1, 0));
|
||||
const uint i12 = batch_idx - i13 * p.ne12;
|
||||
const uint i03 = fastdiv(i13, p.broadcast3_mp, fastdiv_L(p.div_Ls1, 2));
|
||||
const uint i02 = fastdiv(i12, p.broadcast2_mp, fastdiv_L(p.div_Ls1, 1));
|
||||
const uint batch_idx_a = i03 * p.ne02 + i02;
|
||||
#endif
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
load_row_ids(row_begin);
|
||||
uint required_work_items = _ne1 * BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B;
|
||||
@@ -298,8 +349,15 @@ void main() {
|
||||
const uint start_k = 0;
|
||||
const uint end_k = p.K;
|
||||
#else
|
||||
const uint start_k = ik * p.k_split;
|
||||
const uint end_k = min(p.K, (ik + 1) * p.k_split);
|
||||
uint start_k;
|
||||
uint end_k;
|
||||
if (stream_k) {
|
||||
start_k = stream_k_tile_k_unit * 256;
|
||||
end_k = (stream_k_tile_k_unit + stream_k_tile_k_units) * 256;
|
||||
} else {
|
||||
start_k = ik * p.k_split;
|
||||
end_k = min(p.K, (ik + 1) * p.k_split);
|
||||
}
|
||||
#endif
|
||||
|
||||
uint pos_a =
|
||||
@@ -438,7 +496,17 @@ void main() {
|
||||
const uint dc = ic * BN + warp_c * WN;
|
||||
|
||||
#ifndef MUL_MAT_ID
|
||||
const uint offsets = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches;
|
||||
uint offsets = batch_idx * p.batch_stride_d;
|
||||
if (!stream_k) {
|
||||
offsets += ik * p.batch_stride_d * p.num_batches;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef MUL_MAT_ID
|
||||
bool stream_k_store_scratch;
|
||||
if (stream_k) {
|
||||
stream_k_store_scratch = stream_k_tile_k_unit + stream_k_tile_k_units != p.k_units_per_tile;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef COOPMAT
|
||||
@@ -462,36 +530,44 @@ void main() {
|
||||
}
|
||||
}
|
||||
#else
|
||||
const bool is_aligned = p.stride_d % 4 == 0; // Assumption: D_TYPE == float
|
||||
|
||||
[[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {
|
||||
[[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) {
|
||||
const bool is_in_bounds = dr + (cm_row + 1) * TM <= p.M && dc + (cm_col + 1) * TN <= p.N;
|
||||
|
||||
if (is_aligned && is_in_bounds) {
|
||||
// Full coopMat is within bounds and stride_d is aligned with 16B
|
||||
coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> cm_dtype = coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(sums[cm_col * cms_per_row + cm_row]);
|
||||
coopMatStore(cm_dtype, data_d, offsets + (dc + cm_col * TN) * p.stride_d + dr + cm_row * TM, p.stride_d, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
} else if (is_in_bounds) {
|
||||
// Full coopMat is within bounds, but stride_d is not aligned
|
||||
coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
[[unroll]] for (uint col = 0; col < TN; col += storestride) {
|
||||
data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
|
||||
coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> cm_dtype = coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(sums[cm_col * cms_per_row + cm_row]);
|
||||
if (stream_k) {
|
||||
if (stream_k_store_scratch) {
|
||||
const uint scratch_offset = group_idx * BM * BN + (warp_c * WN + cm_col * TN) * BM + warp_r * WM + cm_row * TM;
|
||||
coopMatStore(cm_dtype, data_s, scratch_offset, BM, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
} else {
|
||||
coopMatStore(cm_dtype, data_d, offsets + (dc + cm_col * TN) * p.stride_d + dr + cm_row * TM, p.stride_d, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
}
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
} else if (dr + cm_row * TM < p.M && dc + cm_col * TN < p.N) {
|
||||
// Partial coopMat is within bounds
|
||||
coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
} else {
|
||||
const bool is_aligned = p.stride_d % 4 == 0; // Assumption: D_TYPE == float
|
||||
const bool is_in_bounds = dr + (cm_row + 1) * TM <= p.M && dc + (cm_col + 1) * TN <= p.N;
|
||||
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
[[unroll]] for (uint col = 0; col < TN; col += storestride) {
|
||||
if (dr + cm_row * TM + store_r < p.M && dc + cm_col * TN + col + store_c < p.N) {
|
||||
if (is_aligned && is_in_bounds) {
|
||||
// Full coopMat is within bounds and stride_d is aligned with 16B
|
||||
coopMatStore(cm_dtype, data_d, offsets + (dc + cm_col * TN) * p.stride_d + dr + cm_row * TM, p.stride_d, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
} else if (is_in_bounds) {
|
||||
// Full coopMat is within bounds, but stride_d is not aligned
|
||||
coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
[[unroll]] for (uint col = 0; col < TN; col += storestride) {
|
||||
data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
|
||||
}
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
} else if (dr + cm_row * TM < p.M && dc + cm_col * TN < p.N) {
|
||||
// Partial coopMat is within bounds
|
||||
coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
|
||||
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
[[unroll]] for (uint col = 0; col < TN; col += storestride) {
|
||||
if (dr + cm_row * TM + store_r < p.M && dc + cm_col * TN + col + store_c < p.N) {
|
||||
data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
|
||||
}
|
||||
}
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
}
|
||||
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -519,11 +595,22 @@ void main() {
|
||||
data_d[row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
|
||||
}
|
||||
#else
|
||||
if (dr_warp + 2 * cr < p.M && dc_warp + cc < p.N) {
|
||||
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr] = D_TYPE(sums[sums_idx].x);
|
||||
}
|
||||
if (dr_warp + 2 * cr + 1 < p.M && dc_warp + cc < p.N) {
|
||||
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
|
||||
if (stream_k) {
|
||||
const uint scratch_offset = group_idx * BM * BN + (dc_warp + cc - ic * BN) * BM + dr_warp + 2 * cr - ir * BM;
|
||||
if (stream_k_store_scratch) {
|
||||
data_s[scratch_offset] = D_TYPE(sums[sums_idx].x);
|
||||
data_s[scratch_offset + 1] = D_TYPE(sums[sums_idx].y);
|
||||
} else {
|
||||
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr] = D_TYPE(sums[sums_idx].x);
|
||||
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
|
||||
}
|
||||
} else {
|
||||
if (dr_warp + 2 * cr < p.M && dc_warp + cc < p.N) {
|
||||
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr] = D_TYPE(sums[sums_idx].x);
|
||||
}
|
||||
if (dr_warp + 2 * cr + 1 < p.M && dc_warp + cc < p.N) {
|
||||
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
|
||||
}
|
||||
}
|
||||
#endif // MUL_MAT_ID
|
||||
}
|
||||
@@ -531,4 +618,10 @@ void main() {
|
||||
}
|
||||
}
|
||||
#endif // COOPMAT
|
||||
#ifndef MUL_MAT_ID
|
||||
if (stream_k) {
|
||||
stream_k_work_unit += stream_k_tile_k_units;
|
||||
}
|
||||
} while (stream_k && stream_k_work_unit < stream_k_work_unit_end);
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -61,6 +61,10 @@ uint mm_quant_k() {
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef MUL_MAT_ID
|
||||
layout (constant_id = 15) const bool stream_k = false;
|
||||
#endif
|
||||
|
||||
layout (push_constant) uniform parameter
|
||||
{
|
||||
uint M;
|
||||
@@ -91,6 +95,17 @@ layout (push_constant) uniform parameter
|
||||
uint broadcast3;
|
||||
// N dimension for the B matrix can be >= p.N
|
||||
uint padded_N;
|
||||
uint k_units_per_tile;
|
||||
uint work_units_per_group;
|
||||
uint groups_with_extra_unit;
|
||||
uint k_units_mp;
|
||||
uint blocks_m_mp;
|
||||
uint blocks_n_mp;
|
||||
uint ne12_mp;
|
||||
uint broadcast2_mp;
|
||||
uint broadcast3_mp;
|
||||
uint div_Ls0;
|
||||
uint div_Ls1;
|
||||
#endif
|
||||
} p;
|
||||
|
||||
@@ -102,6 +117,9 @@ layout (binding = 0) readonly buffer A {uint8_t data_a[];};
|
||||
#endif
|
||||
layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
|
||||
layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
|
||||
#ifndef MUL_MAT_ID
|
||||
layout (binding = 3) writeonly buffer S {D_TYPE data_s[];};
|
||||
#endif
|
||||
#if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)
|
||||
layout (binding = 1) readonly buffer B4 {B_TYPEV4 data_b_v4[];};
|
||||
#endif
|
||||
@@ -261,6 +279,17 @@ void main() {
|
||||
const uint row_begin = tile_idx * BN;
|
||||
_ne1 = data_row_map[2 + 2 * tile_idx];
|
||||
const uint ic = 0;
|
||||
#else
|
||||
const uint group_idx = gl_WorkGroupID.x;
|
||||
uint stream_k_work_unit;
|
||||
uint stream_k_work_unit_end;
|
||||
|
||||
if (stream_k) {
|
||||
// A work unit covers 256 consecutive K elements of one output tile.
|
||||
// Work units are flattened as [K unit, M tile, N tile, batch].
|
||||
stream_k_work_unit = group_idx * p.work_units_per_group + min(group_idx, p.groups_with_extra_unit);
|
||||
stream_k_work_unit_end = stream_k_work_unit + p.work_units_per_group + uint(group_idx < p.groups_with_extra_unit);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
@@ -278,43 +307,61 @@ void main() {
|
||||
#endif
|
||||
|
||||
#ifndef MUL_MAT_ID
|
||||
const uint batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
|
||||
do {
|
||||
uint stream_k_tile_k_unit;
|
||||
uint stream_k_tile_k_units;
|
||||
uint ir;
|
||||
uint ic;
|
||||
uint batch_idx;
|
||||
uint ik;
|
||||
|
||||
const uint i13 = batch_idx / p.ne12;
|
||||
const uint i12 = batch_idx % p.ne12;
|
||||
if (stream_k) {
|
||||
// The 256-element granularity does not split any supported quant block.
|
||||
const uint tile_idx = fastdiv(stream_k_work_unit, p.k_units_mp, fastdiv_L(p.div_Ls0, 0));
|
||||
stream_k_tile_k_unit = stream_k_work_unit - tile_idx * p.k_units_per_tile;
|
||||
stream_k_tile_k_units = min(stream_k_work_unit_end - stream_k_work_unit, p.k_units_per_tile - stream_k_tile_k_unit);
|
||||
|
||||
const uint i03 = i13 / p.broadcast3;
|
||||
const uint i02 = i12 / p.broadcast2;
|
||||
|
||||
const uint batch_idx_a = i03 * p.ne02 + i02;
|
||||
const uint blocks_m = p.M / BM;
|
||||
const uint blocks_n = p.N / BN;
|
||||
const uint outer_tile = fastdiv(tile_idx, p.blocks_m_mp, fastdiv_L(p.div_Ls0, 1));
|
||||
ir = tile_idx - outer_tile * blocks_m;
|
||||
batch_idx = fastdiv(outer_tile, p.blocks_n_mp, fastdiv_L(p.div_Ls0, 2));
|
||||
ic = outer_tile - batch_idx * blocks_n;
|
||||
} else {
|
||||
const uint blocks_m = (p.M + BM - 1) / BM;
|
||||
// L2-friendly workgroup scheduling
|
||||
const uint blocks_n = (p.N + BN - 1) / BN;
|
||||
#if defined(MULMAT_QUANT)
|
||||
const uint a_panel_bytes = (BM * p.K / mm_quant_k()) * MmABlockBytes;
|
||||
#elif defined(DATA_A_F32)
|
||||
const uint a_panel_bytes = BM * p.K * 4;
|
||||
#elif QUANT_K == 1
|
||||
const uint a_panel_bytes = BM * p.K * 2;
|
||||
#else
|
||||
const uint a_panel_bytes = (BM * p.K) / 2 + (BM * p.K) / 16;
|
||||
#endif
|
||||
const uint group_m = clamp(GROUP_A_BUDGET / max(a_panel_bytes, 1u), 1u, min(blocks_m, 32u));
|
||||
const uint tiles_per_group = group_m * blocks_n;
|
||||
const uint lin = gl_WorkGroupID.y * blocks_m + (gl_WorkGroupID.x % blocks_m);
|
||||
const uint group_id = lin / tiles_per_group;
|
||||
const uint first_m = group_id * group_m;
|
||||
const uint gsize = min(blocks_m - first_m, group_m);
|
||||
const uint in_group = lin - group_id * tiles_per_group;
|
||||
ir = first_m + in_group % gsize;
|
||||
ic = in_group / gsize;
|
||||
batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
|
||||
ik = group_idx / blocks_m;
|
||||
}
|
||||
|
||||
const uint i13 = fastdiv(batch_idx, p.ne12_mp, fastdiv_L(p.div_Ls1, 0));
|
||||
const uint i12 = batch_idx - i13 * p.ne12;
|
||||
const uint i03 = fastdiv(i13, p.broadcast3_mp, fastdiv_L(p.div_Ls1, 2));
|
||||
const uint i02 = fastdiv(i12, p.broadcast2_mp, fastdiv_L(p.div_Ls1, 1));
|
||||
const uint batch_idx_a = i03 * p.ne02 + i02;
|
||||
#else
|
||||
const uint blocks_m = (p.M + BM - 1) / BM;
|
||||
const uint ik = gl_WorkGroupID.x / blocks_m;
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
const uint ir = gl_WorkGroupID.x % blocks_m;
|
||||
#else
|
||||
// L2-friendly workgroup scheduling
|
||||
const uint blocks_n = (p.N + BN - 1) / BN;
|
||||
#if defined(MULMAT_QUANT)
|
||||
const uint a_panel_bytes = (BM * p.K / mm_quant_k()) * MmABlockBytes;
|
||||
#elif defined(DATA_A_F32)
|
||||
const uint a_panel_bytes = BM * p.K * 4;
|
||||
#elif QUANT_K == 1
|
||||
const uint a_panel_bytes = BM * p.K * 2;
|
||||
#else
|
||||
const uint a_panel_bytes = (BM * p.K) / 2 + (BM * p.K) / 16;
|
||||
#endif
|
||||
const uint group_m = clamp(GROUP_A_BUDGET / max(a_panel_bytes, 1u), 1u, min(blocks_m, 32u));
|
||||
const uint tiles_per_group = group_m * blocks_n;
|
||||
const uint lin = gl_WorkGroupID.y * blocks_m + (gl_WorkGroupID.x % blocks_m);
|
||||
const uint group_id = lin / tiles_per_group;
|
||||
const uint first_m = group_id * group_m;
|
||||
const uint gsize = min(blocks_m - first_m, group_m);
|
||||
const uint in_group = lin - group_id * tiles_per_group;
|
||||
const uint ir = first_m + in_group % gsize;
|
||||
const uint ic = in_group / gsize;
|
||||
#endif
|
||||
|
||||
#ifdef MUL_MAT_ID
|
||||
@@ -325,8 +372,15 @@ void main() {
|
||||
uint start_k = 0;
|
||||
const uint end_k = p.K;
|
||||
#else
|
||||
uint start_k = ik * p.k_split;
|
||||
const uint end_k = min(p.K, (ik + 1) * p.k_split);
|
||||
uint start_k;
|
||||
uint end_k;
|
||||
if (stream_k) {
|
||||
start_k = stream_k_tile_k_unit * 256;
|
||||
end_k = (stream_k_tile_k_unit + stream_k_tile_k_units) * 256;
|
||||
} else {
|
||||
start_k = ik * p.k_split;
|
||||
end_k = min(p.K, (ik + 1) * p.k_split);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef MULMAT_QUANT
|
||||
@@ -341,7 +395,10 @@ void main() {
|
||||
#else
|
||||
uint pos_a = batch_idx_a * (p.batch_stride_a / qk);
|
||||
uint pos_b = batch_idx * p.batch_stride_b;
|
||||
uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches;
|
||||
uint pos_d = batch_idx * p.batch_stride_d;
|
||||
if (!stream_k) {
|
||||
pos_d += ik * p.batch_stride_d * p.num_batches;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef MULMAT_QUANT
|
||||
@@ -404,12 +461,15 @@ void main() {
|
||||
const uint UNROLL_K = (qk == 256) ? 256 : (BK * 8);
|
||||
const uint unroll_count = UNROLL_K / BK;
|
||||
|
||||
// Detect a fast path where all loads are entirely in bounds and no clamping is required
|
||||
if ((ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % START_ALIGN_K) == 0 && (end_k % BK) == 0 &&
|
||||
bool fast_path = stream_k;
|
||||
if (!stream_k) {
|
||||
fast_path = (ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % START_ALIGN_K) == 0 && (end_k % BK) == 0 &&
|
||||
#if !defined(MULMAT_QUANT) && QUANT_K == 1
|
||||
(stride_a % 8) == 0 &&
|
||||
(stride_a % 8) == 0 &&
|
||||
#endif
|
||||
(stride_b % 8) == 0) {
|
||||
(stride_b % 8) == 0;
|
||||
}
|
||||
if (fast_path) {
|
||||
// Hint to the compiler that values are aligned (want 16B alignment)
|
||||
start_k &= ~(START_ALIGN_K-1);
|
||||
stride_b &= ~7;
|
||||
@@ -427,7 +487,7 @@ void main() {
|
||||
// The fetches and stores are pipelined to hide the latency.
|
||||
fetch_scales(ir * BM, pos_a, stride_a, start_k, tid, true);
|
||||
|
||||
if (enable_smaller_matrices && ic * BN + BNover4 >= p.N) {
|
||||
if (!stream_k && enable_smaller_matrices && ic * BN + BNover4 >= p.N) {
|
||||
coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator> sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator>(0.0);
|
||||
for (uint i = 0; i < k_iters; ++i) {
|
||||
|
||||
@@ -470,7 +530,7 @@ void main() {
|
||||
|
||||
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BNover4, ir * BM, BM), tensorViewTranspose);
|
||||
return;
|
||||
} else if (enable_smaller_matrices && ic * BN + BNover2 >= p.N) {
|
||||
} else if (!stream_k && enable_smaller_matrices && ic * BN + BNover2 >= p.N) {
|
||||
coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator> sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator>(0.0);
|
||||
for (uint i = 0; i < k_iters; ++i) {
|
||||
|
||||
@@ -555,8 +615,18 @@ void main() {
|
||||
|
||||
coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator> mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator>(sum);
|
||||
|
||||
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
|
||||
return;
|
||||
if (stream_k) {
|
||||
if (stream_k_tile_k_unit + stream_k_tile_k_units != p.k_units_per_tile) {
|
||||
tensorLayoutNV<2> tensorLayoutS = createTensorLayoutNV(2);
|
||||
tensorLayoutS = setTensorLayoutDimensionNV(tensorLayoutS, BN, BM);
|
||||
tensorLayoutS = setTensorLayoutStrideNV(tensorLayoutS, BM, 1);
|
||||
coopMatStoreTensorNV(mat_d, data_s, group_idx * BM * BN, tensorLayoutS, tensorViewTranspose);
|
||||
} else {
|
||||
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
|
||||
}
|
||||
} else {
|
||||
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
|
||||
}
|
||||
}
|
||||
} else
|
||||
#endif // !defined(MUL_MAT_ID)
|
||||
@@ -720,4 +790,10 @@ void main() {
|
||||
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
|
||||
#endif
|
||||
}
|
||||
#ifndef MUL_MAT_ID
|
||||
if (stream_k) {
|
||||
stream_k_work_unit += stream_k_tile_k_units;
|
||||
}
|
||||
} while (stream_k && stream_k_work_unit < stream_k_work_unit_end);
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -82,6 +82,8 @@ layout (constant_id = 7) const uint TM = 4;
|
||||
layout (constant_id = 8) const uint TN = 2;
|
||||
layout (constant_id = 9) const uint TK = 1; // Only needed for coopmat
|
||||
layout (constant_id = 10) const uint WARP = 32;
|
||||
// Use the mixed signedness dot product, for devices that accelerate only that one
|
||||
layout (constant_id = 11) const bool MMQ_SUDOT = false;
|
||||
|
||||
#define BK 32
|
||||
|
||||
@@ -102,6 +104,13 @@ shared block_b_cache buf_b[BN * BK_STEP];
|
||||
block_a_cache cache_a[WMITER * TM];
|
||||
block_b_cache cache_b;
|
||||
|
||||
#if defined(DATA_A_Q8_0) || defined(DATA_A_MXFP4) || defined(DATA_A_IQ4_XS) || defined(DATA_A_IQ3_S) || defined(DATA_A_Q3_K) || defined(DATA_A_Q6_K)
|
||||
// These quants are signed. MMQ_SUDOT gives them a bias and uses the sum of each half of the q8_1 block to remove it.
|
||||
#define MMQ_B_SUMS
|
||||
shared int32_t buf_b_qsum[MMQ_SUDOT ? BN * BK_STEP * 2 : 1];
|
||||
int32_t cache_b_qsum[2];
|
||||
#endif
|
||||
|
||||
#define LOAD_VEC_A (4 * QUANT_R_MMQ)
|
||||
#define LOAD_VEC_B 16
|
||||
|
||||
|
||||
@@ -4,6 +4,19 @@
|
||||
|
||||
#include "types.glsl"
|
||||
|
||||
int32_t mmq_dot(const int32_t a, const int32_t b) {
|
||||
// With MMQ_SUDOT the A quants must be unsigned
|
||||
if (MMQ_SUDOT) {
|
||||
return dotPacked4x8EXT(uint32_t(a), b);
|
||||
}
|
||||
return dotPacked4x8EXT(a, b);
|
||||
}
|
||||
|
||||
// With MMQ_SUDOT, flip the sign bit of each byte. This adds a bias of 128 to signed quants.
|
||||
int32_t mmq_a_store(const int32_t v) {
|
||||
return MMQ_SUDOT ? v ^ int32_t(0x80808080) : v;
|
||||
}
|
||||
|
||||
// Each iqs value maps to a 32-bit integer
|
||||
|
||||
#if defined(DATA_A_Q2_0)
|
||||
@@ -41,7 +54,7 @@ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
||||
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
int32_t q_sum = 0;
|
||||
[[unroll]] for (uint iqs = 0; iqs < 8; ++iqs) {
|
||||
q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(cache_a[ib_a].dm) * (float(q_sum) * float(cache_b.ds.x) - float(cache_b.ds.y)));
|
||||
@@ -102,8 +115,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
const int32_t qs_b0 = cache_b.qs[iqs];
|
||||
const int32_t qs_b1 = cache_b.qs[iqs + 4];
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a.x, qs_b0);
|
||||
q_sum += dotPacked4x8EXT(qs_a.y, qs_b1);
|
||||
q_sum += mmq_dot(qs_a.x, qs_b0);
|
||||
q_sum += mmq_dot(qs_a.y, qs_b1);
|
||||
}
|
||||
|
||||
#ifdef DATA_A_Q4_0
|
||||
@@ -176,8 +189,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
const int32_t qs_b0 = cache_b.qs[iqs];
|
||||
const int32_t qs_b1 = cache_b.qs[iqs + 4];
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a0, qs_b0);
|
||||
q_sum += dotPacked4x8EXT(qs_a1, qs_b1);
|
||||
q_sum += mmq_dot(qs_a0, qs_b0);
|
||||
q_sum += mmq_dot(qs_a1, qs_b1);
|
||||
}
|
||||
|
||||
#ifdef DATA_A_Q5_0
|
||||
@@ -192,8 +205,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
// 2-byte loads for Q8_0 blocks (34 bytes)
|
||||
void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) {
|
||||
if (is_in_bounds) {
|
||||
buf_a[buf_ib].qs[iqs] = pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2],
|
||||
data_a_packed16[ib].qs[iqs * 2 + 1]));
|
||||
buf_a[buf_ib].qs[iqs] = mmq_a_store(pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2],
|
||||
data_a_packed16[ib].qs[iqs * 2 + 1])));
|
||||
|
||||
if (iqs == 0) {
|
||||
buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d);
|
||||
@@ -221,7 +234,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
||||
const int32_t qs_b = cache_b.qs[iqs];
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, qs_b);
|
||||
q_sum += mmq_dot(qs_a, qs_b);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(q_sum) * float(cache_a[ib_a].dm) * float(cache_b.ds.x));
|
||||
@@ -240,8 +256,8 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
|
||||
const u8vec4 i_a0 = unpack8( qs & 0x0F0F0F0F);
|
||||
const u8vec4 i_a1 = unpack8((qs >> 4) & 0x0F0F0F0F);
|
||||
|
||||
buf_a[buf_ib].qs[iqs ] = pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w]));
|
||||
buf_a[buf_ib].qs[iqs + 4] = pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w]));
|
||||
buf_a[buf_ib].qs[iqs ] = mmq_a_store(pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w])));
|
||||
buf_a[buf_ib].qs[iqs + 4] = mmq_a_store(pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w])));
|
||||
|
||||
if (iqs == 0) {
|
||||
buf_a[buf_ib].d = FLOAT_TYPE(e8m0_to_fp32(data_a[ib].e) * 0.5);
|
||||
@@ -269,7 +285,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
|
||||
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
|
||||
@@ -284,8 +303,8 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
|
||||
const uint32_t vui = data_a_packed32[ib_k].qs[4 * ib32 + iqs];
|
||||
const i32vec2 qs = iq4nl_to_i8x8(vui);
|
||||
|
||||
buf_a[buf_ib].qs[iqs ] = qs.x;
|
||||
buf_a[buf_ib].qs[iqs + 4] = qs.y;
|
||||
buf_a[buf_ib].qs[iqs ] = mmq_a_store(qs.x);
|
||||
buf_a[buf_ib].qs[iqs + 4] = mmq_a_store(qs.y);
|
||||
|
||||
if (iqs == 0) {
|
||||
const uint sl = (data_a_packed32[ib_k].scales_l >> (4 * ib32)) & 0xF;
|
||||
@@ -313,7 +332,10 @@ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
||||
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
int32_t q_sum = 0;
|
||||
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
|
||||
q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
|
||||
@@ -372,8 +394,8 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
const int32_t scale_m = int32_t(scale >> 4) * 0x01010101; // Duplicate 8-bit value across 32-bits.
|
||||
const int32_t qs_a = int32_t((cache_a[ib_a].qs[iqs / 4] >> ((iqs % 4) * 2)) & 0x03030303);
|
||||
|
||||
sum_d += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]) * (scale & 0xF);
|
||||
sum_m += dotPacked4x8EXT(scale_m, cache_b.qs[iqs]);
|
||||
sum_d += mmq_dot(qs_a, cache_b.qs[iqs]) * (scale & 0xF);
|
||||
sum_m += mmq_dot(scale_m, cache_b.qs[iqs]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(cache_b.ds.x) * (float(cache_a[ib_a].dm.x) * float(sum_d) - float(cache_a[ib_a].dm.y) * float(sum_m)));
|
||||
@@ -435,18 +457,26 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
int32_t q_sum = 0;
|
||||
|
||||
[[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
|
||||
// Subtract 4 from the quants to correct the 3rd bit offset
|
||||
const int32_t qs_a = pack32(unpack8(int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F)) - int8_t(4));
|
||||
// Subtract 4 from the quants to correct the 3rd bit offset. MMQ_SUDOT keeps them unsigned and subtracts 4 times the B sum.
|
||||
const int32_t qs_u = int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F);
|
||||
const int32_t qs_a = MMQ_SUDOT ? qs_u : pack32(unpack8(qs_u) - int8_t(4));
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 4 * cache_b_qsum[0];
|
||||
}
|
||||
result += float(cache_a[ib_a].d_scales[0]) * float(q_sum);
|
||||
q_sum = 0;
|
||||
|
||||
[[unroll]] for (uint iqs = 4; iqs < 8; iqs++) {
|
||||
const int32_t qs_a = pack32(unpack8(int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F)) - int8_t(4));
|
||||
const int32_t qs_u = int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F);
|
||||
const int32_t qs_a = MMQ_SUDOT ? qs_u : pack32(unpack8(qs_u) - int8_t(4));
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 4 * cache_b_qsum[1];
|
||||
}
|
||||
result += float(cache_a[ib_a].d_scales[1]) * float(q_sum);
|
||||
|
||||
@@ -518,7 +548,7 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
||||
#endif
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(cache_b.ds.x) * float(cache_a[ib_a].dm.x) * float(q_sum) - float(cache_a[ib_a].dm.y) * float(cache_b.ds.y));
|
||||
@@ -539,10 +569,11 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
|
||||
const uint qh_shift = ((iqs_k % 32) / 8) * 2;
|
||||
|
||||
const i8vec2 vals00 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 ] >> ql_shift) & uint16_t(0x0F0F))).xy |
|
||||
unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 ] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32);
|
||||
unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 ] >> qh_shift) & uint16_t(0x0303)) << 4)).xy);
|
||||
const i8vec2 vals01 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 + 1] >> ql_shift) & uint16_t(0x0F0F))).xy |
|
||||
unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 + 1] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32);
|
||||
buf_a[buf_ib].qs[iqs] = pack32(i8vec4(vals00.x, vals00.y, vals01.x, vals01.y));
|
||||
unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 + 1] >> qh_shift) & uint16_t(0x0303)) << 4)).xy);
|
||||
// MMQ_SUDOT keeps the quants unsigned and subtracts 32 times the B sum
|
||||
buf_a[buf_ib].qs[iqs] = MMQ_SUDOT ? pack32(i8vec4(vals00, vals01)) : pack32(i8vec4(vals00 - int8_t(32), vals01 - int8_t(32)));
|
||||
|
||||
if (iqs == 0) {
|
||||
const uint is = iqs_k / 4;
|
||||
@@ -574,7 +605,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
[[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
|
||||
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 32 * cache_b_qsum[0];
|
||||
}
|
||||
result += float(cache_a[ib_a].d_scales[0]) * float(q_sum);
|
||||
q_sum = 0;
|
||||
@@ -582,7 +616,10 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
[[unroll]] for (uint iqs = 4; iqs < 8; iqs++) {
|
||||
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
||||
|
||||
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(qs_a, cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 32 * cache_b_qsum[1];
|
||||
}
|
||||
result += float(cache_a[ib_a].d_scales[1]) * float(q_sum);
|
||||
|
||||
@@ -612,8 +649,8 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
|
||||
const ivec4 m0 = -(ivec4(signs, signs >> 1, signs >> 2, signs >> 3) & 1);
|
||||
const ivec4 m1 = -(ivec4(signs >> 4, signs >> 5, signs >> 6, signs >> 7) & 1);
|
||||
|
||||
buf_a[buf_ib].qs[2 * iqs ] = pack32(i8vec4((vals0 ^ m0) - m0));
|
||||
buf_a[buf_ib].qs[2 * iqs + 1] = pack32(i8vec4((vals1 ^ m1) - m1));
|
||||
buf_a[buf_ib].qs[2 * iqs ] = mmq_a_store(pack32(i8vec4((vals0 ^ m0) - m0)));
|
||||
buf_a[buf_ib].qs[2 * iqs + 1] = mmq_a_store(pack32(i8vec4((vals1 ^ m1) - m1)));
|
||||
|
||||
if (iqs == 0) {
|
||||
const uint scale = (uint(data_a_packed16[ib_k].scales[ib32 / 4]) >> ((ib32 & 3) * 4)) & 0xF;
|
||||
@@ -641,7 +678,10 @@ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
||||
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
||||
int32_t q_sum = 0;
|
||||
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
|
||||
q_sum += dotPacked4x8EXT(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
|
||||
q_sum += mmq_dot(cache_a[ib_a].qs[iqs], cache_b.qs[iqs]);
|
||||
}
|
||||
if (MMQ_SUDOT) {
|
||||
q_sum -= 128 * (cache_b_qsum[0] + cache_b_qsum[1]);
|
||||
}
|
||||
|
||||
return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
|
||||
@@ -662,6 +702,12 @@ void block_b_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
|
||||
buf_b[buf_ib].qs[iqs * 4 + 1] = values.y;
|
||||
buf_b[buf_ib].qs[iqs * 4 + 2] = values.z;
|
||||
buf_b[buf_ib].qs[iqs * 4 + 3] = values.w;
|
||||
#ifdef MMQ_B_SUMS
|
||||
if (MMQ_SUDOT) {
|
||||
buf_b_qsum[buf_ib * 2 + iqs] = dotPacked4x8EXT(values.x, 0x01010101u) + dotPacked4x8EXT(values.y, 0x01010101u) +
|
||||
dotPacked4x8EXT(values.z, 0x01010101u) + dotPacked4x8EXT(values.w, 0x01010101u);
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
if (iqs == 0) {
|
||||
buf_b[buf_ib].ds = FLOAT_TYPEV2(0.0f);
|
||||
@@ -671,6 +717,11 @@ void block_b_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bo
|
||||
buf_b[buf_ib].qs[iqs * 4 + 1] = 0;
|
||||
buf_b[buf_ib].qs[iqs * 4 + 2] = 0;
|
||||
buf_b[buf_ib].qs[iqs * 4 + 3] = 0;
|
||||
#ifdef MMQ_B_SUMS
|
||||
if (MMQ_SUDOT) {
|
||||
buf_b_qsum[buf_ib * 2 + iqs] = 0;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@@ -679,4 +730,10 @@ void block_b_to_registers(const uint ib) {
|
||||
[[unroll]] for (uint iqs = 0; iqs < BK / 4; iqs++) {
|
||||
cache_b.qs[iqs] = buf_b[ib].qs[iqs];
|
||||
}
|
||||
#ifdef MMQ_B_SUMS
|
||||
if (MMQ_SUDOT) {
|
||||
cache_b_qsum[0] = buf_b_qsum[ib * 2];
|
||||
cache_b_qsum[1] = buf_b_qsum[ib * 2 + 1];
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
#version 450
|
||||
|
||||
#extension GL_EXT_shader_realtime_clock : require
|
||||
|
||||
layout(local_size_x = 1, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
// never written, it only anchors the spin below so the compiler cannot drop it
|
||||
layout (binding = 0) writeonly buffer D {uint data_d[];};
|
||||
|
||||
layout (push_constant) uniform parameter {
|
||||
uint ticks;
|
||||
} p;
|
||||
|
||||
// bounds the spin in case the clock does not advance, turning a hang into a too-short sleep
|
||||
const uint MAX_ITER = 1u << 27;
|
||||
|
||||
void main() {
|
||||
const uvec2 t0 = clockRealtime2x32EXT();
|
||||
|
||||
uvec2 t = t0;
|
||||
|
||||
for (uint iter = 0; iter < MAX_ITER; ++iter) {
|
||||
t = clockRealtime2x32EXT();
|
||||
|
||||
const uint lo = t.x - t0.x;
|
||||
const uint hi = t.y - t0.y - (t.x < t0.x ? 1u : 0u);
|
||||
|
||||
if (hi != 0u || lo >= p.ticks) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// the store never runs, but the compiler cannot prove it and has to keep the spin
|
||||
if (t.x == 0xFFFFFFFFu && t.y == 0xFFFFFFFFu) {
|
||||
data_d[0] = t.x;
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,6 @@ void main() {
|
||||
if (pass) {
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(data_a[get_aoffset() + src0_idx(idx)]);
|
||||
} else {
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0);
|
||||
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(0.0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -948,6 +948,7 @@ void process_shaders() {
|
||||
string_to_spv("acc_f32", "acc.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
|
||||
|
||||
string_to_spv("split_k_reduce", "mul_mat_split_k_reduce.comp", {});
|
||||
string_to_spv("stream_k_fixup", "mul_mat_stream_k_fixup.comp", {});
|
||||
string_to_spv("fa_split_k_reduce", "flash_attn_split_k_reduce.comp", {});
|
||||
|
||||
string_to_spv("fa_mask_opt", "flash_attn_mask_opt.comp", {});
|
||||
@@ -1027,6 +1028,7 @@ void process_shaders() {
|
||||
string_to_spv("arange_f32", "arange.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
|
||||
string_to_spv("fill_f32", "fill.comp", {{"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
|
||||
string_to_spv("fill_f16", "fill.comp", {{"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
|
||||
string_to_spv("sleep", "sleep.comp", {});
|
||||
|
||||
string_to_spv("silu_back_f32", "silu_back.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
|
||||
|
||||
@@ -1272,6 +1272,7 @@ class ggml_webgpu_shader_lib {
|
||||
std::unordered_map<int, webgpu_pipeline> arange_pipelines; // key is fixed, no variants yet
|
||||
std::unordered_map<int, webgpu_pipeline> count_equal_pipelines; // key is src_overlap
|
||||
std::unordered_map<int, webgpu_pipeline> timestep_embedding_pipelines; // key is fixed, no variants yet
|
||||
std::unordered_map<int, webgpu_pipeline> pool2d_pipelines; // key is ggml_op_pool
|
||||
std::unordered_map<ggml_webgpu_row_norm_pipeline_key, webgpu_pipeline, ggml_webgpu_row_norm_pipeline_key_hash>
|
||||
row_norm_pipelines; // op/inplace
|
||||
|
||||
@@ -2067,6 +2068,34 @@ class ggml_webgpu_shader_lib {
|
||||
return arange_pipelines[1];
|
||||
}
|
||||
|
||||
webgpu_pipeline get_pool2d_pipeline(const ggml_webgpu_shader_lib_context & context) {
|
||||
const int key = ggml_get_op_params_i32(context.dst, 0);
|
||||
|
||||
auto it = pool2d_pipelines.find(key);
|
||||
if (it != pool2d_pipelines.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
std::vector<std::string> defines;
|
||||
std::string variant = "pool2d";
|
||||
if (key == GGML_OP_POOL_MAX) {
|
||||
defines.push_back("POOL_MAX");
|
||||
variant += "_max";
|
||||
} else {
|
||||
variant += "_avg";
|
||||
}
|
||||
|
||||
defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size));
|
||||
|
||||
auto processed = preprocessor.preprocess(wgsl_pool2d, defines);
|
||||
auto decisions = std::make_shared<ggml_webgpu_generic_shader_decisions>();
|
||||
decisions->wg_size = context.max_wg_size;
|
||||
webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, variant);
|
||||
pipeline.context = decisions;
|
||||
pool2d_pipelines[key] = pipeline;
|
||||
return pool2d_pipelines[key];
|
||||
}
|
||||
|
||||
webgpu_pipeline get_pad_pipeline(const ggml_webgpu_shader_lib_context & context) {
|
||||
ggml_webgpu_pad_pipeline_key key = {};
|
||||
key.reflect = context.dst->op == GGML_OP_PAD_REFLECT_1D;
|
||||
|
||||
@@ -961,6 +961,46 @@ static webgpu_encoded_op ggml_webgpu_pad(webgpu_context & ctx, ggml_tensor * src
|
||||
return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y);
|
||||
}
|
||||
|
||||
static webgpu_encoded_op ggml_webgpu_pool2d(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) {
|
||||
ggml_webgpu_shader_lib_context shader_lib_ctx = {};
|
||||
shader_lib_ctx.src0 = src;
|
||||
shader_lib_ctx.dst = dst;
|
||||
shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup;
|
||||
|
||||
webgpu_pipeline pipeline = ctx->shader_lib->get_pool2d_pipeline(shader_lib_ctx);
|
||||
|
||||
auto * decisions = static_cast<ggml_webgpu_generic_shader_decisions *>(pipeline.context.get());
|
||||
|
||||
const uint32_t ne = (uint32_t) ggml_nelements(dst);
|
||||
|
||||
std::vector<uint32_t> params = {
|
||||
ne,
|
||||
(uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src) / ggml_type_size(src->type)),
|
||||
(uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)),
|
||||
(uint32_t) src->ne[0],
|
||||
(uint32_t) src->ne[1],
|
||||
(uint32_t) dst->ne[0],
|
||||
(uint32_t) dst->ne[1],
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 1), // k0
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 2), // k1
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 3), // s0
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 4), // s1
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 5), // p0
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 6), // p1
|
||||
};
|
||||
|
||||
std::vector<wgpu::BindGroupEntry> entries = {
|
||||
ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src),
|
||||
ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst),
|
||||
};
|
||||
|
||||
uint32_t wg_x;
|
||||
uint32_t wg_y;
|
||||
uint32_t total_wg = CEIL_DIV(ne, decisions->wg_size);
|
||||
compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y);
|
||||
return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y);
|
||||
}
|
||||
|
||||
static webgpu_encoded_op ggml_webgpu_solve_tri(webgpu_context & ctx,
|
||||
ggml_tensor * src0,
|
||||
ggml_tensor * src1,
|
||||
@@ -2106,6 +2146,7 @@ static ggml_webgpu_flash_attn_op ggml_webgpu_flash_attn_prepare(webgpu_context &
|
||||
op.has_mask ? (uint32_t) (mask->nb[3] / ggml_type_size(mask->type)) : 0, // stride of mask dim 3
|
||||
op.has_mask ? (uint32_t) (mask->nb[2] / ggml_type_size(mask->type)) : 0, // stride of mask dim 2
|
||||
op.has_mask ? (uint32_t) mask->ne[2] : 1u, // mask heads, broadcast over the Q heads
|
||||
op.has_mask ? (uint32_t) mask->ne[3] : 1u, // mask batches, broadcast over the Q batches
|
||||
(uint32_t) (Q->ne[2] / K->ne[2]), // repeat factor for K/V in dim 2 (MHA/MQA/GQA)
|
||||
ggml_webgpu_u32_from_f32(scale), // scale (possibly adjusted for logit softcap)
|
||||
ggml_webgpu_u32_from_f32(max_bias),
|
||||
@@ -2215,7 +2256,7 @@ static webgpu_encoded_op ggml_webgpu_flash_attn_vec(webgpu_context & ct
|
||||
const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
|
||||
const uint32_t stride_mask2 = (uint32_t) (mask->nb[2] / ggml_type_size(mask->type));
|
||||
const uint32_t mask_ne2 = (uint32_t) mask->ne[2];
|
||||
blk_batch_count = (stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u) * mask_ne2;
|
||||
blk_batch_count = (uint32_t) (mask->ne[3] * mask_ne2);
|
||||
const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
|
||||
blk_size_bytes = ROUNDUP_POW2(blk_elems * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT);
|
||||
const ggml_webgpu_shader_lib_context blk_shader_ctx = op.shader_lib_ctx;
|
||||
@@ -3669,6 +3710,8 @@ static std::optional<webgpu_encoded_op> ggml_webgpu_encode(webgpu_context ctx,
|
||||
}
|
||||
case GGML_OP_ARANGE:
|
||||
return ggml_webgpu_arange(ctx, node);
|
||||
case GGML_OP_POOL_2D:
|
||||
return ggml_webgpu_pool2d(ctx, src0, node);
|
||||
case GGML_OP_PAD_REFLECT_1D:
|
||||
case GGML_OP_PAD:
|
||||
return ggml_webgpu_pad(ctx, src0, node);
|
||||
@@ -4448,12 +4491,10 @@ static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer
|
||||
res += WEBGPU_STORAGE_BUF_BINDING_MULT + align;
|
||||
}
|
||||
if (mask != nullptr) {
|
||||
const uint32_t blk_nblk0 = CEIL_DIV((uint32_t) K->ne[1], kv_tile);
|
||||
const uint32_t blk_nblk1 = (uint32_t) n_q;
|
||||
const uint32_t stride_mask3 = (uint32_t) (mask->nb[3] / ggml_type_size(mask->type));
|
||||
const uint32_t blk_batch_count =
|
||||
(stride_mask3 > 0 ? (uint32_t) Q->ne[3] : 1u) * (uint32_t) mask->ne[2];
|
||||
const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
|
||||
const uint32_t blk_nblk0 = CEIL_DIV((uint32_t) K->ne[1], kv_tile);
|
||||
const uint32_t blk_nblk1 = (uint32_t) n_q;
|
||||
const uint32_t blk_batch_count = (uint32_t) Q->ne[3] * (uint32_t) mask->ne[2];
|
||||
const uint64_t blk_elems = (uint64_t) blk_nblk0 * blk_nblk1 * blk_batch_count;
|
||||
const size_t blk_size_bytes =
|
||||
ROUNDUP_POW2(blk_elems * sizeof(uint32_t), WEBGPU_STORAGE_BUF_BINDING_MULT);
|
||||
res += blk_size_bytes + align;
|
||||
@@ -5220,6 +5261,9 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
|
||||
case GGML_OP_ARANGE:
|
||||
supports_op = op->type == GGML_TYPE_F32 && ggml_is_contiguous(op);
|
||||
break;
|
||||
case GGML_OP_POOL_2D:
|
||||
supports_op = op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32 && ggml_is_contiguous(src0);
|
||||
break;
|
||||
case GGML_OP_PAD_REFLECT_1D:
|
||||
supports_op = op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32;
|
||||
break;
|
||||
|
||||
@@ -116,7 +116,7 @@ fn main(@builtin(workgroup_id) wg_id: vec3<u32>,
|
||||
|
||||
#ifdef MASK
|
||||
// mask offset
|
||||
let mask_global_offset = params.offset_mask + batch_idx * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
|
||||
let mask_global_offset = params.offset_mask + (batch_idx % params.mask_ne3) * params.stride_mask3 + (head_idx % params.mask_ne2) * params.stride_mask2 + q_row_start * params.seq_len_kv;
|
||||
#endif
|
||||
|
||||
// note that the output is permuted, the layout is [head_dim_v, n_heads, seq_len_q, batch_size]
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user