mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-19 09:17:40 -06:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c7468107ef | ||
|
|
ac830a9cbb | ||
|
|
8af68d486a | ||
|
|
f8caa5ea30 | ||
|
|
4db8b7638f | ||
|
|
01592dc58b | ||
|
|
8bf2132c3f | ||
|
|
5812201a81 | ||
|
|
41d8297fe4 |
@@ -61,6 +61,11 @@ jobs:
|
||||
run: |
|
||||
choco install ninja
|
||||
|
||||
- name: Install LLVM
|
||||
run: |
|
||||
choco install llvm --version=22.1.8
|
||||
echo "C:\Program Files\LLVM\bin" >> $env:GITHUB_PATH
|
||||
|
||||
- name: Install OpenCL Headers and Libs
|
||||
id: install_opencl
|
||||
run: |
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<b>LLM inference in C/C++</b>
|
||||
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v1)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=tag:v1)
|
||||
[](https://github.com/ggml-org/llama.cpp/releases?q=b)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
|
||||
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
||||
|
||||
+1
-1
@@ -90,7 +90,7 @@ Legend:
|
||||
| PAD | ❌ | 🟡 | ✅ | 🟡 | ❌ | ✅ | 🟡 | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ |
|
||||
| PAD_REFLECT_1D | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ |
|
||||
| POOL_1D | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| POOL_2D | ❌ | 🟡 | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REGLU | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ❌ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| RELU | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ |
|
||||
| REPEAT | ❌ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | 🟡 | ❌ | ❌ |
|
||||
|
||||
+128
-128
@@ -1155,134 +1155,134 @@
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i64,ne=[1,8,1,3],nr23=[1,1],r=2,v=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","SET_ROWS","type_src=f16,type_dst=f16,type_idx=i32,ne=[1,8,1,3],nr23=[1,1],r=2,v=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=avg,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=1,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=1,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=1,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=1,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=0,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=0","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_2D","pool_type=max,type_input=f32,ne_input=[10,10,3,1],k0=3,k1=3,s0=2,s1=2,p0=1,p1=1","support","1","yes","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[10,3,2,1],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[11,1,3,2],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
"WebGPU: WebGPU","POOL_1D","pool_type=avg,type_input=f32,ne_input=[128,2,1,3],k0=1,s0=1,p0=0","support","0","no","WebGPU"
|
||||
|
||||
|
Can't render this file because it is too large.
|
@@ -8,10 +8,10 @@ extern "C" {
|
||||
|
||||
#define RPC_PROTO_MAJOR_VERSION 10
|
||||
#define RPC_PROTO_MINOR_VERSION 0
|
||||
#define RPC_PROTO_PATCH_VERSION 0
|
||||
#define RPC_PROTO_PATCH_VERSION 1
|
||||
|
||||
#ifdef __cplusplus
|
||||
static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
|
||||
static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
|
||||
#endif
|
||||
|
||||
#define GGML_RPC_MAX_SERVERS 16
|
||||
|
||||
@@ -429,6 +429,8 @@ extern "C" {
|
||||
|
||||
GGML_OP_GLU,
|
||||
|
||||
GGML_OP_SLEEP,
|
||||
|
||||
GGML_OP_COUNT,
|
||||
};
|
||||
|
||||
@@ -2599,6 +2601,12 @@ extern "C" {
|
||||
int n_tasks,
|
||||
void * userdata);
|
||||
|
||||
// Busy-wait operator for scheduling and backend testing.
|
||||
GGML_API struct ggml_tensor * ggml_sleep(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a, // wait after this tensor has executed, return value is simply a full view of this tensor
|
||||
int32_t us); // amount of time to wait in microseconds
|
||||
|
||||
// loss function
|
||||
|
||||
GGML_API struct ggml_tensor * ggml_cross_entropy_loss(
|
||||
|
||||
@@ -2584,6 +2584,7 @@ bool ggml_op_alloc_size_may_expand(enum ggml_op op) {
|
||||
case GGML_OP_ARGSORT:
|
||||
case GGML_OP_TOP_K:
|
||||
case GGML_OP_ARGMAX:
|
||||
case GGML_OP_CONV_2D:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -2034,6 +2034,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
|
||||
{
|
||||
ggml_compute_forward_fill(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_SLEEP:
|
||||
{
|
||||
ggml_compute_forward_sleep(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_FLASH_ATTN_EXT:
|
||||
{
|
||||
ggml_compute_forward_flash_attn_ext(params, tensor);
|
||||
@@ -2306,6 +2310,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
|
||||
case GGML_OP_REPEAT:
|
||||
case GGML_OP_REPEAT_BACK:
|
||||
case GGML_OP_LEAKY_RELU:
|
||||
case GGML_OP_SLEEP:
|
||||
{
|
||||
n_tasks = 1;
|
||||
} break;
|
||||
|
||||
@@ -2238,6 +2238,19 @@ void ggml_compute_forward_fill(const ggml_compute_params * params, ggml_tensor *
|
||||
}
|
||||
}
|
||||
|
||||
// ggml_compute_forward_sleep
|
||||
|
||||
void ggml_compute_forward_sleep(const ggml_compute_params * params, ggml_tensor * dst) {
|
||||
if (params->ith != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// same clock as ggml_time_us() callers; on macOS steady_clock is CLOCK_MONOTONIC_RAW, which can run faster
|
||||
const int64_t t_end = ggml_time_us() + ggml_get_op_params_i32(dst, 0);
|
||||
|
||||
while (ggml_time_us() < t_end) {}
|
||||
}
|
||||
|
||||
// ggml_compute_tri
|
||||
|
||||
template <typename T>
|
||||
|
||||
@@ -84,6 +84,7 @@ void ggml_compute_forward_top_k(const struct ggml_compute_params * params, struc
|
||||
void ggml_compute_forward_leaky_relu(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_fill(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_sleep(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_flash_attn_ext(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_flash_attn_back(
|
||||
const struct ggml_compute_params * params,
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
#include "batch-get.cuh"
|
||||
#include "batch-copy.cuh"
|
||||
#include "common.cuh"
|
||||
#include "ggml-cpp.h"
|
||||
|
||||
#include <climits>
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
struct ggml_cuda_batch_get_descriptor {
|
||||
struct ggml_cuda_batch_copy_descriptor {
|
||||
const unsigned char * src;
|
||||
unsigned char * dst;
|
||||
size_t size;
|
||||
};
|
||||
|
||||
static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_descriptor * copies, size_t count) {
|
||||
static __global__ void ggml_cuda_batch_copy_kernel(const ggml_cuda_batch_copy_descriptor * copies, size_t count) {
|
||||
// Blocks along y cooperate on each descriptor.
|
||||
const size_t index = blockIdx.x;
|
||||
if (index >= count) {
|
||||
@@ -50,11 +50,11 @@ static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_desc
|
||||
}
|
||||
}
|
||||
|
||||
struct ggml_cuda_batch_get_mapping {
|
||||
struct ggml_cuda_batch_copy_mapping {
|
||||
void * host = nullptr;
|
||||
unsigned char * device = nullptr;
|
||||
|
||||
ggml_cuda_batch_get_mapping(void * host, size_t size) {
|
||||
ggml_cuda_batch_copy_mapping(void * host, size_t size) {
|
||||
const cudaError_t err = cudaHostRegister(host, size, cudaHostRegisterMapped | cudaHostRegisterPortable);
|
||||
if (err != cudaSuccess) {
|
||||
(void) cudaGetLastError();
|
||||
@@ -70,10 +70,10 @@ struct ggml_cuda_batch_get_mapping {
|
||||
}
|
||||
}
|
||||
|
||||
ggml_cuda_batch_get_mapping(const ggml_cuda_batch_get_mapping &) = delete;
|
||||
ggml_cuda_batch_get_mapping & operator=(const ggml_cuda_batch_get_mapping &) = delete;
|
||||
ggml_cuda_batch_copy_mapping(const ggml_cuda_batch_copy_mapping &) = delete;
|
||||
ggml_cuda_batch_copy_mapping & operator=(const ggml_cuda_batch_copy_mapping &) = delete;
|
||||
|
||||
~ggml_cuda_batch_get_mapping() {
|
||||
~ggml_cuda_batch_copy_mapping() {
|
||||
if (host) {
|
||||
// Release only the registration acquired by this object.
|
||||
CUDA_CHECK(cudaHostUnregister(host));
|
||||
@@ -82,11 +82,12 @@ struct ggml_cuda_batch_get_mapping {
|
||||
};
|
||||
#endif
|
||||
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
static bool ggml_cuda_copy_tensor_n(int device, cudaMemcpyKind direction,
|
||||
const ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
GGML_UNUSED(device);
|
||||
GGML_UNUSED(direction);
|
||||
GGML_UNUSED(tensors);
|
||||
GGML_UNUSED(data);
|
||||
GGML_UNUSED(offsets);
|
||||
@@ -94,14 +95,17 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
GGML_UNUSED(n_tensors);
|
||||
return false;
|
||||
#else
|
||||
const bool to_host = direction == cudaMemcpyDeviceToHost;
|
||||
// Provisional dispatch values based on experiments.
|
||||
constexpr size_t min_batch_reads = 128;
|
||||
constexpr size_t max_gather_read_bytes = 128 * 1024;
|
||||
// Collect non-empty CPU destination ranges.
|
||||
constexpr size_t max_kernel_read_bytes = 128 * 1024;
|
||||
// Collect destination ranges; repeated sources are allowed.
|
||||
std::vector<std::pair<uintptr_t, uintptr_t>> ranges;
|
||||
ranges.reserve(n_tensors);
|
||||
size_t payload_bytes = 0;
|
||||
// Number of non-empty reads whose size is at most max_gather_read_bytes.
|
||||
uintptr_t begin = UINTPTR_MAX;
|
||||
uintptr_t end = 0;
|
||||
// Number of non-empty reads whose size is at most max_kernel_read_bytes.
|
||||
size_t n_reads_small = 0;
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
if (sizes[i] == 0) {
|
||||
@@ -111,15 +115,19 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
GGML_ASSERT(address <= UINTPTR_MAX - sizes[i]);
|
||||
GGML_ASSERT(payload_bytes <= SIZE_MAX - sizes[i]);
|
||||
payload_bytes += sizes[i];
|
||||
n_reads_small += sizes[i] <= max_gather_read_bytes;
|
||||
ranges.emplace_back(address, address + sizes[i]);
|
||||
n_reads_small += sizes[i] <= max_kernel_read_bytes;
|
||||
begin = std::min(begin, address);
|
||||
end = std::max(end, address + sizes[i]);
|
||||
const uintptr_t destination = to_host ? address : (uintptr_t) tensors[i]->data + offsets[i];
|
||||
GGML_ASSERT(destination <= UINTPTR_MAX - sizes[i]);
|
||||
ranges.emplace_back(destination, destination + sizes[i]);
|
||||
}
|
||||
if (ranges.empty()) {
|
||||
return true;
|
||||
}
|
||||
// Total number of non-empty reads.
|
||||
const size_t n_reads = ranges.size();
|
||||
if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_get_descriptor) || n_reads > INT_MAX) {
|
||||
if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_copy_descriptor) || n_reads > INT_MAX) {
|
||||
return false;
|
||||
}
|
||||
if (n_reads < 2 || (n_reads < min_batch_reads && n_reads_small == n_reads)) {
|
||||
@@ -127,72 +135,85 @@ bool ggml_cuda_get_tensor_n(int device,
|
||||
}
|
||||
std::sort(ranges.begin(), ranges.end());
|
||||
for (size_t i = 1; i < ranges.size(); ++i) {
|
||||
// Preserve the ordered copy path when CPU destinations overlap.
|
||||
// Preserve the ordered copy path when destinations overlap.
|
||||
if (ranges[i].first < ranges[i - 1].second) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
// Find the CPU span to register, including gaps between destinations.
|
||||
const uintptr_t begin = ranges.front().first;
|
||||
const uintptr_t end = ranges.back().second;
|
||||
if (end - begin - payload_bytes > payload_bytes) {
|
||||
// Limit gaps in the CPU span without underflow when sources overlap.
|
||||
if (end - begin > payload_bytes && end - begin - payload_bytes > payload_bytes) {
|
||||
return false;
|
||||
}
|
||||
// Register and map the existing CPU destination so the kernel can write to it directly.
|
||||
ggml_cuda_batch_get_mapping mapping((void *) begin, end - begin);
|
||||
// Register and map the CPU span for direct kernel access.
|
||||
ggml_cuda_batch_copy_mapping mapping((void *) begin, end - begin);
|
||||
if (!mapping.host) {
|
||||
return false;
|
||||
}
|
||||
// Reads selected for gather; zero means all reads use DMA.
|
||||
size_t n_reads_gather = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0;
|
||||
// Reads selected for the kernel; zero means all reads use DMA.
|
||||
size_t n_reads_kernel = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0;
|
||||
// Keep descriptor storage alive until the final stream synchronization.
|
||||
constexpr size_t max_batch_descriptors = 128 * 1024;
|
||||
const size_t bytes = std::min(n_reads_gather, max_batch_descriptors) * sizeof(ggml_cuda_batch_get_descriptor);
|
||||
ggml_backend_buffer_ptr descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr);
|
||||
const size_t bytes = std::min(n_reads_kernel, max_batch_descriptors) * sizeof(ggml_cuda_batch_copy_descriptor);
|
||||
ggml_backend_buffer_ptr descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr);
|
||||
if (!descriptors) {
|
||||
n_reads_gather = 0;
|
||||
n_reads_kernel = 0;
|
||||
}
|
||||
const auto host_type = ggml_backend_cuda_host_buffer_type();
|
||||
ggml_backend_buffer_ptr host_descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr);
|
||||
ggml_backend_buffer_ptr host_descriptors(n_reads_kernel ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr);
|
||||
if (!host_descriptors || ggml_backend_buffer_get_type(host_descriptors.get()) != host_type) {
|
||||
n_reads_gather = 0;
|
||||
n_reads_kernel = 0;
|
||||
}
|
||||
if (n_reads_gather) {
|
||||
auto * copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get()));
|
||||
auto * device_copies = static_cast<ggml_cuda_batch_get_descriptor *>(ggml_backend_buffer_get_base(descriptors.get()));
|
||||
if (n_reads_kernel) {
|
||||
auto * copies = static_cast<ggml_cuda_batch_copy_descriptor *>(ggml_backend_buffer_get_base(host_descriptors.get()));
|
||||
auto * device_copies = static_cast<ggml_cuda_batch_copy_descriptor *>(ggml_backend_buffer_get_base(descriptors.get()));
|
||||
const int threads = 256;
|
||||
const size_t nsm = ggml_cuda_info().devices[device].nsm;
|
||||
size_t i = 0;
|
||||
for (size_t first = 0; first < n_reads_gather; first += max_batch_descriptors) {
|
||||
const size_t n_batch = std::min(n_reads_gather - first, max_batch_descriptors);
|
||||
for (size_t first = 0; first < n_reads_kernel; first += max_batch_descriptors) {
|
||||
const size_t n_batch = std::min(n_reads_kernel - first, max_batch_descriptors);
|
||||
size_t count = 0;
|
||||
for (; i < n_tensors && count < n_batch; ++i) {
|
||||
if (sizes[i] == 0 || sizes[i] > max_gather_read_bytes) {
|
||||
if (sizes[i] == 0 || sizes[i] > max_kernel_read_bytes) {
|
||||
continue;
|
||||
}
|
||||
copies[count++] = {(const unsigned char *) tensors[i]->data + offsets[i],
|
||||
mapping.device + ((uintptr_t) data[i] - begin), sizes[i]};
|
||||
auto * host = mapping.device + ((uintptr_t) data[i] - begin);
|
||||
auto * tensor = (unsigned char *) tensors[i]->data + offsets[i];
|
||||
copies[count++] = {to_host ? tensor : host, to_host ? host : tensor, sizes[i]};
|
||||
}
|
||||
const size_t blocks_per_copy = std::min<size_t>(UINT16_MAX, std::max<size_t>(1, (nsm + n_batch - 1) / n_batch));
|
||||
const dim3 blocks((unsigned int) n_batch, (unsigned int) blocks_per_copy);
|
||||
// Stream ordering keeps each kernel ahead of the next descriptor overwrite.
|
||||
CUDA_CHECK(cudaMemcpyAsync(device_copies, copies, n_batch * sizeof(copies[0]), cudaMemcpyHostToDevice, cudaStreamPerThread));
|
||||
ggml_cuda_batch_get_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch);
|
||||
ggml_cuda_batch_copy_kernel<<<blocks, threads, 0, cudaStreamPerThread>>>(device_copies, n_batch);
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
if (first + n_batch < n_reads_gather) {
|
||||
if (first + n_batch < n_reads_kernel) {
|
||||
// Wait before refilling the pinned host descriptor buffer.
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
}
|
||||
}
|
||||
}
|
||||
// Keep the destination pinned for DMA too; overlapping destinations already declined above.
|
||||
// Keep the CPU span pinned for DMA too; overlapping destinations already declined above.
|
||||
for (size_t i = 0; i < n_tensors; ++i) {
|
||||
if (sizes[i] && (!n_reads_gather || sizes[i] > max_gather_read_bytes)) {
|
||||
CUDA_CHECK(cudaMemcpyAsync(data[i], (const char *) tensors[i]->data + offsets[i], sizes[i], cudaMemcpyDeviceToHost, cudaStreamPerThread));
|
||||
if (sizes[i] && (!n_reads_kernel || sizes[i] > max_kernel_read_bytes)) {
|
||||
void * host = const_cast<void *>(data[i]);
|
||||
void * tensor = (char *) tensors[i]->data + offsets[i];
|
||||
CUDA_CHECK(cudaMemcpyAsync(to_host ? host : tensor, to_host ? tensor : host, sizes[i], direction, cudaStreamPerThread));
|
||||
}
|
||||
}
|
||||
// Wait for GPU writes before releasing the CPU mapping and descriptor buffers.
|
||||
// Wait for transfers before releasing the CPU mapping and descriptor buffers.
|
||||
CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread));
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
return ggml_cuda_copy_tensor_n(device, cudaMemcpyDeviceToHost, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
|
||||
bool ggml_cuda_set_tensor_n(int device,
|
||||
ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors) {
|
||||
return ggml_cuda_copy_tensor_n(device, cudaMemcpyHostToDevice, tensors, data, offsets, sizes, n_tensors);
|
||||
}
|
||||
@@ -5,3 +5,7 @@
|
||||
bool ggml_cuda_get_tensor_n(int device,
|
||||
const ggml_tensor * const * tensors, void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors);
|
||||
|
||||
bool ggml_cuda_set_tensor_n(int device,
|
||||
ggml_tensor * const * tensors, const void * const * data, const size_t * offsets,
|
||||
const size_t * sizes, size_t n_tensors);
|
||||
@@ -10,7 +10,7 @@
|
||||
#include "ggml-cuda/arange.cuh"
|
||||
#include "ggml-cuda/argmax.cuh"
|
||||
#include "ggml-cuda/argsort.cuh"
|
||||
#include "ggml-cuda/batch-get.cuh"
|
||||
#include "ggml-cuda/batch-copy.cuh"
|
||||
#include "ggml-cuda/binbcast.cuh"
|
||||
#include "ggml-cuda/clamp.cuh"
|
||||
#include "ggml-cuda/col2im-1d.cuh"
|
||||
@@ -48,6 +48,7 @@
|
||||
#include "ggml-cuda/rope.cuh"
|
||||
#include "ggml-cuda/roll.cuh"
|
||||
#include "ggml-cuda/scale.cuh"
|
||||
#include "ggml-cuda/sleep.cuh"
|
||||
#include "ggml-cuda/snake.cuh"
|
||||
#include "ggml-cuda/softcap.cuh"
|
||||
#include "ggml-cuda/softmax.cuh"
|
||||
@@ -1116,6 +1117,11 @@ static void ggml_backend_cuda_buffer_set_tensor_n(ggml_backend_buffer_t buffer,
|
||||
|
||||
ggml_cuda_set_device(ctx->device);
|
||||
|
||||
if (!ggml_cuda_pinned_memory_disabled() &&
|
||||
ggml_cuda_set_tensor_n(ctx->device, tensors, data, offsets, sizes, n_tensors)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < n_tensors; i++) {
|
||||
if (sizes[i] == 0) {
|
||||
continue;
|
||||
@@ -3145,6 +3151,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
|
||||
case GGML_OP_FILL:
|
||||
ggml_cuda_op_fill(ctx, dst);
|
||||
break;
|
||||
case GGML_OP_SLEEP:
|
||||
ggml_cuda_op_sleep(ctx, dst);
|
||||
break;
|
||||
case GGML_OP_LIGHTNING_INDEXER:
|
||||
ggml_cuda_lightning_indexer(ctx, dst);
|
||||
break;
|
||||
@@ -6809,6 +6818,12 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
return ggml_cuda_lightning_indexer_supported(dev_ctx->device, op);
|
||||
case GGML_OP_TOP_K_MASK:
|
||||
return op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32;
|
||||
case GGML_OP_SLEEP:
|
||||
#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
return false;
|
||||
#else
|
||||
return true;
|
||||
#endif // defined(GGML_USE_HIP) || defined(GGML_USE_MUSA)
|
||||
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
#include "sleep.cuh"
|
||||
|
||||
#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
|
||||
// %globaltimer is a nanosecond wall clock, unlike clock64() it is unaffected by the SM clock and by frequency scaling
|
||||
static __device__ __forceinline__ uint64_t globaltimer_ns() {
|
||||
uint64_t t;
|
||||
asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(t));
|
||||
return t;
|
||||
}
|
||||
|
||||
// a single thread is enough, the next kernel on the same stream cannot start before this one retires
|
||||
static __global__ void sleep_ns(const uint64_t ns) {
|
||||
const uint64_t t0 = globaltimer_ns();
|
||||
|
||||
while (globaltimer_ns() - t0 < ns) {}
|
||||
}
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
sleep_ns<<<1, 1, 0, ctx.stream()>>>(1000*(uint64_t) ggml_get_op_params_i32(dst, 0));
|
||||
CUDA_CHECK(cudaGetLastError());
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
GGML_UNUSED(ctx);
|
||||
GGML_UNUSED(dst);
|
||||
GGML_ABORT("GGML_OP_SLEEP requires the %%globaltimer register, which is only available on CUDA");
|
||||
}
|
||||
|
||||
#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA)
|
||||
@@ -0,0 +1,3 @@
|
||||
#include "common.cuh"
|
||||
|
||||
void ggml_cuda_op_sleep(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
@@ -2551,6 +2551,26 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_
|
||||
char base[256];
|
||||
char name[256];
|
||||
|
||||
const bool has_tensor = ggml_metal_device_get_props(ggml_metal_library_get_device(lib))->has_tensor;
|
||||
|
||||
if (has_tensor) {
|
||||
// nr0 output positions per threadgroup, nr1 output channels, same tiles as kernel_mul_mm
|
||||
snprintf(base, 256, "kernel_conv_2d_mm_tensor_%s_%s", ggml_type_name(op->src[0]->type), ggml_type_name(op->src[1]->type));
|
||||
snprintf(name, 256, "%s", base);
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
|
||||
}
|
||||
|
||||
res.nr0 = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y;
|
||||
res.nr1 = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X;
|
||||
res.nsg = N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y;
|
||||
res.smem = res.nr0 * N_MM_NK_TOTAL * sizeof(ggml_fp16_t);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
const int nr0 = op->src[0]->ne[3] <= 32 ? 32 : 64;
|
||||
const int nr1 = 2048/nr0;
|
||||
|
||||
@@ -2564,13 +2584,27 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm(ggml_
|
||||
|
||||
res.nr0 = nr0;
|
||||
res.nr1 = nr1;
|
||||
res.nsg = 4;
|
||||
res.nsg = N_SG_CONV_2D;
|
||||
|
||||
res.smem = nr0*nr1*sizeof(float);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm_reduce(ggml_metal_library_t lib, const ggml_tensor * op) {
|
||||
assert(op->op == GGML_OP_CONV_2D);
|
||||
GGML_UNUSED(op);
|
||||
|
||||
const char * name = "kernel_conv_2d_mm_reduce";
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
res = ggml_metal_library_compile_pipeline(lib, name, name, nullptr);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw(ggml_metal_library_t lib, const ggml_tensor * op, bool tiled) {
|
||||
assert(op->op == GGML_OP_CONV_2D_DW);
|
||||
|
||||
|
||||
@@ -170,6 +170,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_col2im_1d
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_snake (ggml_metal_library_t lib, enum ggml_type type);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_mm_reduce (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_2d_dw (ggml_metal_library_t lib, const struct ggml_tensor * op, bool tiled);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_conv_3d (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_upscale (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
|
||||
@@ -2375,7 +2375,7 @@ void ggml_metal_buffer_memset_tensor(ggml_metal_buffer_t buf, struct ggml_tensor
|
||||
id<MTLBlitCommandEncoder> encoder = [cmd_buf blitCommandEncoder];
|
||||
|
||||
[encoder fillBuffer:bid_dst.metal
|
||||
range:NSMakeRange(bid_dst.offs, bid_dst.offs + size)
|
||||
range:NSMakeRange(bid_dst.offs, size)
|
||||
value:value];
|
||||
|
||||
[encoder endEncoding];
|
||||
|
||||
@@ -17,6 +17,8 @@
|
||||
#define N_ARGMAX_NPART_MAX 256
|
||||
#define N_ARGMAX_CHUNK 4096
|
||||
|
||||
#define N_SG_CONV_2D 4
|
||||
|
||||
// kernel parameters for mat-vec threadgroups
|
||||
//
|
||||
// N_R0: number of src0 rows to process per simdgroup
|
||||
@@ -752,8 +754,14 @@ typedef struct {
|
||||
int32_t p1;
|
||||
int32_t d0;
|
||||
int32_t d1;
|
||||
int32_t nk; // reduction length per split
|
||||
} ggml_metal_kargs_conv_2d;
|
||||
|
||||
typedef struct {
|
||||
int32_t ne; // output elements
|
||||
int32_t nsplit;
|
||||
} ggml_metal_kargs_conv_2d_reduce;
|
||||
|
||||
typedef struct {
|
||||
uint64_t nb00; // kernel strides
|
||||
uint64_t nb01;
|
||||
|
||||
@@ -4959,6 +4959,33 @@ int ggml_metal_op_im2col(ggml_metal_op_t ctx, int idx) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
// number of reduction splits of the conv_2d matmul kernels
|
||||
// A small output with a long reduction leaves the GPU idle, so the reduction is split until there are about 1M partial sums.
|
||||
static int ggml_metal_op_conv_2d_nsplit(const ggml_tensor * op, int32_t * nk) {
|
||||
const int64_t K = op->src[0]->ne[0]*op->src[0]->ne[1]*op->src[0]->ne[2];
|
||||
const int64_t nsteps = (K + 31)/32;
|
||||
const int64_t ne = ggml_nelements(op);
|
||||
|
||||
// at least 3 reduction steps of 32 per split, at most 32 splits
|
||||
const int64_t nsplit = std::max<int64_t>(1, std::min<int64_t>({ (1024*1024 + ne - 1)/ne, nsteps/3, 32 }));
|
||||
const int64_t steps = (nsteps + nsplit - 1)/nsplit;
|
||||
|
||||
if (nk) {
|
||||
*nk = (int32_t) (steps*32);
|
||||
}
|
||||
|
||||
return (int) ((nsteps + steps - 1)/steps);
|
||||
}
|
||||
|
||||
size_t ggml_metal_op_conv_2d_extra_tmp(const ggml_tensor * op) {
|
||||
assert(op->op == GGML_OP_CONV_2D);
|
||||
|
||||
const int nsplit = ggml_metal_op_conv_2d_nsplit(op, nullptr);
|
||||
|
||||
// partial sums of the reduction splits
|
||||
return nsplit > 1 ? nsplit*ggml_nelements(op)*sizeof(float) : 0;
|
||||
}
|
||||
|
||||
int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_tensor * op = ctx->node(idx);
|
||||
|
||||
@@ -5012,11 +5039,17 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
/*.p1 =*/ p1,
|
||||
/*.d0 =*/ d0,
|
||||
/*.d1 =*/ d1,
|
||||
/*.nk =*/ 0,
|
||||
};
|
||||
|
||||
const ggml_metal_device_props * props_dev = ggml_metal_device_get_props(ctx->dev);
|
||||
|
||||
if (props_dev->has_simdgroup_mm) {
|
||||
// the matmul kernels walk the patches with 32-bit byte offsets inside an image
|
||||
// larger images, possibly a strided view, take the generic kernel
|
||||
const uint64_t max_off = (ne12 - 1)*nb12 + (ne11 - 1)*nb11 + (ne10 - 1)*nb10 + sizeof(float);
|
||||
const bool fits_32 = max_off <= UINT32_MAX;
|
||||
|
||||
if (props_dev->has_simdgroup_mm && fits_32) {
|
||||
auto pipeline = ggml_metal_library_get_pipeline_conv_2d_mm(lib, op);
|
||||
|
||||
const int nr0 = pipeline.nr0;
|
||||
@@ -5026,15 +5059,48 @@ int ggml_metal_op_conv_2d(ggml_metal_op_t ctx, int idx) {
|
||||
const int64_t NP = ne3*ne1*ne0;
|
||||
const int64_t OC = ne03;
|
||||
|
||||
const int nsplit = ggml_metal_op_conv_2d_nsplit(op, &args.nk);
|
||||
|
||||
ggml_metal_buffer_id bid_dst = ggml_metal_get_buffer_id(op);
|
||||
ggml_metal_buffer_id bid_tmp = bid_dst;
|
||||
bid_tmp.offs += ggml_nbytes(op);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline);
|
||||
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), 1);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[1]), 2);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), 3);
|
||||
ggml_metal_encoder_set_buffer (enc, nsplit > 1 ? bid_tmp : bid_dst, 3);
|
||||
|
||||
ggml_metal_encoder_set_threadgroup_memory_size(enc, pipeline.smem, 0);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, 1, 32, nsg, 1);
|
||||
if (props_dev->has_tensor) {
|
||||
// per image: nr0 positions times nr1 channels per threadgroup
|
||||
const int64_t OHW = ne1*ne0;
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (OC + nr1 - 1)/nr1, (OHW + nr0 - 1)/nr0, ne3*nsplit, 32, nsg, 1);
|
||||
} else {
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (NP + nr1 - 1)/nr1, (OC + nr0 - 1)/nr0, nsplit, 32, nsg, 1);
|
||||
}
|
||||
|
||||
if (nsplit > 1) {
|
||||
GGML_ASSERT(ggml_is_contiguous(op));
|
||||
|
||||
// sync the 2 kernels
|
||||
ggml_metal_op_concurrency_reset(ctx);
|
||||
|
||||
ggml_metal_kargs_conv_2d_reduce args0 = {
|
||||
/*.ne =*/ (int32_t) ggml_nelements(op),
|
||||
/*.nsplit =*/ nsplit,
|
||||
};
|
||||
|
||||
auto pipeline0 = ggml_metal_library_get_pipeline_conv_2d_mm_reduce(lib, op);
|
||||
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline0);
|
||||
ggml_metal_encoder_set_bytes (enc, &args0, sizeof(args0), 0);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_tmp, 1);
|
||||
ggml_metal_encoder_set_buffer (enc, bid_dst, 2);
|
||||
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, (args0.ne + 255)/256, 1, 1, 256, 1, 1);
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
@@ -49,6 +49,8 @@ size_t ggml_metal_op_flash_attn_ext_extra_rows(const struct ggml_tensor * op);
|
||||
|
||||
size_t ggml_metal_op_argmax_extra(const struct ggml_tensor * op);
|
||||
|
||||
size_t ggml_metal_op_conv_2d_extra_tmp(const struct ggml_tensor * op);
|
||||
|
||||
int ggml_metal_op_concat (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_repeat (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_acc (ggml_metal_op_t ctx, int idx);
|
||||
|
||||
@@ -311,6 +311,10 @@ static size_t ggml_backend_metal_buffer_type_get_alloc_size(ggml_backend_buffer_
|
||||
{
|
||||
res = 2*sizeof(int32_t)*ggml_nelements(tensor->src[0]);
|
||||
} break;
|
||||
case GGML_OP_CONV_2D:
|
||||
{
|
||||
res += ggml_metal_op_conv_2d_extra_tmp(tensor);
|
||||
} break;
|
||||
case GGML_OP_ARGMAX:
|
||||
{
|
||||
res += ggml_metal_op_argmax_extra(tensor);
|
||||
|
||||
@@ -285,7 +285,7 @@ kernel void kernel_conv_2d_mm(
|
||||
ushort tiitg[[thread_index_in_threadgroup]],
|
||||
ushort tiisg[[thread_index_in_simdgroup]],
|
||||
ushort sgitg[[simdgroup_index_in_threadgroup]]) {
|
||||
constexpr short NSG = 4;
|
||||
constexpr short NSG = N_SG_CONV_2D;
|
||||
constexpr short NTH = 32*NSG;
|
||||
constexpr short NK = 32;
|
||||
constexpr short NKB = NK/8;
|
||||
@@ -300,6 +300,10 @@ kernel void kernel_conv_2d_mm(
|
||||
const int K = args.IC*args.KH*args.KW;
|
||||
const int NP = args.N*args.OH*args.OW;
|
||||
|
||||
// this threadgroup reduces [k_begin, k_end), the splits are summed by kernel_conv_2d_mm_reduce
|
||||
const int k_begin = tgpig.z*args.nk;
|
||||
const int k_end = min(K, k_begin + args.nk);
|
||||
|
||||
const int r0 = tgpig.y*NR0;
|
||||
const int r1 = tgpig.x*NR1;
|
||||
|
||||
@@ -308,8 +312,14 @@ kernel void kernel_conv_2d_mm(
|
||||
const short lb_n = 8*sgitg + tiisg%8;
|
||||
const short lb_k = 8*(tiisg/8);
|
||||
|
||||
// the patches are walked with 32-bit byte offsets inside the image, the host checks that an image fits
|
||||
const uint nb10 = args.nb10;
|
||||
const uint nb11 = args.nb11;
|
||||
const uint nb12 = args.nb12;
|
||||
|
||||
int ix0[NPL];
|
||||
int iy0[NPL];
|
||||
uint off0[NPL];
|
||||
bool ip_ok[NPL];
|
||||
|
||||
device const char * srcb[NPL];
|
||||
@@ -324,6 +334,8 @@ kernel void kernel_conv_2d_mm(
|
||||
ix0[t] = iow*args.s0 - args.p0;
|
||||
iy0[t] = ioh*args.s1 - args.p1;
|
||||
|
||||
off0[t] = iy0[t]*nb11 + ix0[t]*nb10;
|
||||
|
||||
ip_ok[t] = ip < NP;
|
||||
|
||||
srcb[t] = src + in*args.nb13;
|
||||
@@ -337,7 +349,7 @@ kernel void kernel_conv_2d_mm(
|
||||
mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
|
||||
}
|
||||
|
||||
for (int k0 = 0; k0 < K; k0 += NK) {
|
||||
for (int k0 = k_begin; k0 < k_end; k0 += NK) {
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
{
|
||||
@@ -349,7 +361,7 @@ kernel void kernel_conv_2d_mm(
|
||||
const int oc = r0 + row;
|
||||
|
||||
half v = 0;
|
||||
if (kk < K && oc < args.OC) {
|
||||
if (kk < k_end && oc < args.OC) {
|
||||
v = *((device const TK *)(weights + oc*args.nb03) + kk);
|
||||
}
|
||||
|
||||
@@ -364,7 +376,11 @@ kernel void kernel_conv_2d_mm(
|
||||
|
||||
int kw = kk % args.KW;
|
||||
int kh = (kk / args.KW) % args.KH;
|
||||
int ic = kk / (args.KW*args.KH);
|
||||
int dx = kw*args.d0;
|
||||
int dy = kh*args.d1;
|
||||
|
||||
// byte offset of the reduction index kk, relative to the patch origin
|
||||
uint offk = (kk / (args.KW*args.KH))*nb12 + dy*nb11 + dx*nb10;
|
||||
|
||||
FOR_UNROLL (short i = 0; i < 8; i++) {
|
||||
const short k = lb_k + i;
|
||||
@@ -372,12 +388,12 @@ kernel void kernel_conv_2d_mm(
|
||||
FOR_UNROLL (short t = 0; t < NPL; t++) {
|
||||
const short n = 32*t + lb_n;
|
||||
|
||||
const int ix = ix0[t] + kw*args.d0;
|
||||
const int iy = iy0[t] + kh*args.d1;
|
||||
const int ix = ix0[t] + dx;
|
||||
const int iy = iy0[t] + dy;
|
||||
|
||||
half v = 0;
|
||||
if (ip_ok[t] && kk < K && ix >= 0 && ix < args.IW && iy >= 0 && iy < args.IH) {
|
||||
v = *(device const float *)(srcb[t] + ic*args.nb12 + iy*args.nb11 + ix*args.nb10);
|
||||
if (ip_ok[t] && kk < k_end && (uint) ix < (uint) args.IW && (uint) iy < (uint) args.IH) {
|
||||
v = *(device const float *)(srcb[t] + (off0[t] + offk));
|
||||
}
|
||||
|
||||
const short ib = NKB*(n/8) + k/8;
|
||||
@@ -386,11 +402,17 @@ kernel void kernel_conv_2d_mm(
|
||||
}
|
||||
|
||||
kk++;
|
||||
dx += args.d0;
|
||||
offk += args.d0*nb10;
|
||||
if (++kw == args.KW) {
|
||||
kw = 0;
|
||||
kw = 0;
|
||||
dx = 0;
|
||||
dy += args.d1;
|
||||
offk += args.d1*nb11 - args.KW*args.d0*nb10;
|
||||
if (++kh == args.KH) {
|
||||
kh = 0;
|
||||
ic++;
|
||||
kh = 0;
|
||||
dy = 0;
|
||||
offk += nb12 - args.KH*args.d1*nb11;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -398,7 +420,7 @@ kernel void kernel_conv_2d_mm(
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
const short nkb = min((K - k0 + 7)/8, (int) NKB);
|
||||
const short nkb = min((k_end - k0 + 7)/8, (int) NKB);
|
||||
|
||||
for (short ik = 0; ik < nkb; ik++) {
|
||||
FOR_UNROLL (short i = 0; i < 4; i++) {
|
||||
@@ -436,7 +458,8 @@ kernel void kernel_conv_2d_mm(
|
||||
const int oh = (p / args.OW) % args.OH;
|
||||
const int n = p / (args.OW*args.OH);
|
||||
|
||||
device char * dstp = dst + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
|
||||
// with several splits, dst is the partial sum buffer, one contiguous output per split
|
||||
device char * dstp = dst + (uint64_t) tgpig.z*args.N*args.nb3 + n*args.nb3 + oh*args.nb1 + ow*args.nb0;
|
||||
|
||||
for (short j = 0; j < NR0*NR1/NTH; j++) {
|
||||
const short row = tiitg/NR1 + (NTH/NR1)*j;
|
||||
@@ -448,6 +471,158 @@ kernel void kernel_conv_2d_mm(
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef GGML_METAL_HAS_TENSOR
|
||||
// conv_2d as a matmul on the tensor cores, like kernel_mul_mm
|
||||
// The patches of NRA output positions are gathered into threadgroup memory, the kernel rows are read from device memory.
|
||||
// One threadgroup computes NRA positions of one image times NRB output channels, over the reduction range of its split.
|
||||
template <typename TK>
|
||||
kernel void kernel_conv_2d_mm_tensor(
|
||||
constant ggml_metal_kargs_conv_2d & args,
|
||||
device const char * weights,
|
||||
device const char * src,
|
||||
device char * dst,
|
||||
threadgroup char * shmem [[threadgroup(0)]],
|
||||
uint3 tgpig[[threadgroup_position_in_grid]],
|
||||
uint3 tgpg[[threadgroups_per_grid]],
|
||||
ushort tiitg[[thread_index_in_threadgroup]]) {
|
||||
constexpr int NRB = SZ_SIMDGROUP * N_MM_BLOCK_X * N_MM_SIMD_GROUP_X; // output channels
|
||||
constexpr int NRA = SZ_SIMDGROUP * N_MM_BLOCK_Y * N_MM_SIMD_GROUP_Y; // output positions
|
||||
constexpr int NK = N_MM_NK_TOTAL;
|
||||
constexpr int NTH = N_SIMDWIDTH * N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y;
|
||||
|
||||
// each thread gathers NK/(NTH/NRA) consecutive reduction indices of one position
|
||||
constexpr int KPT = NK/(NTH/NRA);
|
||||
|
||||
const int K = args.IC*args.KH*args.KW;
|
||||
const int OHW = args.OH*args.OW;
|
||||
|
||||
const int nsplit = tgpg.z/args.N;
|
||||
const int img = tgpig.z/nsplit;
|
||||
const int split = tgpig.z%nsplit;
|
||||
|
||||
const int k_begin = split*args.nk;
|
||||
const int k_end = min(K, k_begin + args.nk);
|
||||
|
||||
const int ra = tgpig.y*NRA;
|
||||
const int rb = tgpig.x*NRB;
|
||||
|
||||
threadgroup half * sa = (threadgroup half *) shmem;
|
||||
|
||||
// the position this thread gathers
|
||||
const int row = tiitg%NRA;
|
||||
const int kc = (tiitg/NRA)*KPT;
|
||||
const int p = ra + row;
|
||||
const bool pok = p < OHW;
|
||||
const int ix0 = (p%args.OW)*args.s0 - args.p0;
|
||||
const int iy0 = (p/args.OW)*args.s1 - args.p1;
|
||||
|
||||
device const char * srcb = src + img*args.nb13;
|
||||
|
||||
// the patch is walked with 32-bit byte offsets inside the image, the host checks that an image fits
|
||||
const uint nb10 = args.nb10;
|
||||
const uint nb11 = args.nb11;
|
||||
const uint nb12 = args.nb12;
|
||||
|
||||
const uint off0 = iy0*nb11 + ix0*nb10;
|
||||
|
||||
device TK * ptrB = (device TK *) weights;
|
||||
const int strideB = args.nb03/sizeof(TK);
|
||||
|
||||
mpp::tensor_ops::matmul2d<
|
||||
mpp::tensor_ops::matmul2d_descriptor(
|
||||
NRB, NRA, static_cast<int>(dynamic_extent), false, true, true,
|
||||
mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
|
||||
execution_simdgroups<N_MM_SIMD_GROUP_X * N_MM_SIMD_GROUP_Y>> mm;
|
||||
|
||||
auto tB0 = tensor(ptrB, dextents<int32_t, 2>(K, args.OC), array<int, 2>({1, strideB}));
|
||||
auto tA0 = tensor(sa, dextents<int32_t, 2>(NK, NRA));
|
||||
auto cT = mm.get_destination_cooperative_tensor<decltype(tB0), decltype(tA0), float>();
|
||||
|
||||
for (int k0 = k_begin; k0 < k_end; k0 += NK) {
|
||||
// the reduction indices past k_end are beyond the extent given to the matmul, they are not gathered
|
||||
if (k0 + kc < k_end) {
|
||||
// gather KPT patch values of this position, walking the reduction index incrementally
|
||||
int kk = k0 + kc;
|
||||
int kw = kk % args.KW;
|
||||
int kh = (kk / args.KW) % args.KH;
|
||||
int dx = kw*args.d0;
|
||||
int dy = kh*args.d1;
|
||||
|
||||
// byte offset of the reduction index kk, relative to the patch origin
|
||||
uint offk = (kk / (args.KW*args.KH))*nb12 + dy*nb11 + dx*nb10;
|
||||
|
||||
FOR_UNROLL (short i = 0; i < KPT; i++) {
|
||||
const int ix = ix0 + dx;
|
||||
const int iy = iy0 + dy;
|
||||
|
||||
half v = 0;
|
||||
if (pok && kk < k_end && (uint) ix < (uint) args.IW && (uint) iy < (uint) args.IH) {
|
||||
v = *(device const float *)(srcb + (off0 + offk));
|
||||
}
|
||||
sa[row*NK + kc + i] = v;
|
||||
|
||||
kk++;
|
||||
dx += args.d0;
|
||||
offk += args.d0*nb10;
|
||||
if (++kw == args.KW) {
|
||||
kw = 0;
|
||||
dx = 0;
|
||||
dy += args.d1;
|
||||
offk += args.d1*nb11 - args.KW*args.d0*nb10;
|
||||
if (++kh == args.KH) {
|
||||
kh = 0;
|
||||
dy = 0;
|
||||
offk += nb12 - args.KH*args.d1*nb11;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
|
||||
// clamp the reduction extent so neither operand is read past the end of the split
|
||||
const int kExt = min(NK, k_end - k0);
|
||||
|
||||
auto tAv = tensor(sa, dextents<int32_t, 2>(kExt, NRA), array<int, 2>({1, NK}));
|
||||
auto tBv = tensor(ptrB + k0 + rb*strideB, dextents<int32_t, 2>(kExt, args.OC - rb), array<int, 2>({1, strideB}));
|
||||
|
||||
mm.run(tBv, tAv, cT);
|
||||
|
||||
threadgroup_barrier(mem_flags::mem_threadgroup);
|
||||
}
|
||||
|
||||
// dst holds [OHW, OC] per image, with several splits it is the partial sum buffer, one contiguous output per split
|
||||
const int tileM = min(NRA, OHW - ra);
|
||||
const int tileN = min(NRB, args.OC - rb);
|
||||
|
||||
device float * dstTile = (device float *) (dst + (uint64_t) split*args.N*args.nb3 + img*args.nb3) + (uint64_t) rb*OHW + ra;
|
||||
|
||||
auto tD = tensor(dstTile, dextents<int32_t, 2>(tileM, tileN), array<int, 2>({1, OHW}));
|
||||
cT.store(tD);
|
||||
}
|
||||
|
||||
typedef decltype(kernel_conv_2d_mm_tensor<float>) kernel_conv_2d_mm_tensor_t;
|
||||
|
||||
template [[host_name("kernel_conv_2d_mm_tensor_f32_f32")]] kernel kernel_conv_2d_mm_tensor_t kernel_conv_2d_mm_tensor<float>;
|
||||
template [[host_name("kernel_conv_2d_mm_tensor_f16_f32")]] kernel kernel_conv_2d_mm_tensor_t kernel_conv_2d_mm_tensor<half>;
|
||||
#endif
|
||||
|
||||
kernel void kernel_conv_2d_mm_reduce(
|
||||
constant ggml_metal_kargs_conv_2d_reduce & args,
|
||||
device const float * tmp,
|
||||
device float * dst,
|
||||
uint tpig[[thread_position_in_grid]]) {
|
||||
if ((int) tpig >= args.ne) {
|
||||
return;
|
||||
}
|
||||
|
||||
float sum = 0.0f;
|
||||
for (int s = 0; s < args.nsplit; ++s) {
|
||||
sum += tmp[(uint64_t) s*args.ne + tpig];
|
||||
}
|
||||
dst[tpig] = sum;
|
||||
}
|
||||
|
||||
typedef decltype(kernel_conv_2d_mm<float, 64, 32>) kernel_conv_2d_mm_t;
|
||||
|
||||
template [[host_name("kernel_conv_2d_mm_f32_f32_64x32")]] kernel kernel_conv_2d_mm_t kernel_conv_2d_mm<float, 64, 32>;
|
||||
|
||||
@@ -1043,6 +1043,8 @@ void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph
|
||||
} else if (tensor->op == GGML_OP_FILL) {
|
||||
const float value = ggml_get_op_params_f32(tensor, 0);
|
||||
tensor_clone = ggml_fill(ggml_ctx, src_clone[0], value);
|
||||
} else if (tensor->op == GGML_OP_SLEEP) {
|
||||
tensor_clone = ggml_sleep(ggml_ctx, src_clone[0], ggml_get_op_params_i32(tensor, 0));
|
||||
} else if (tensor->op == GGML_OP_SQR) {
|
||||
tensor_clone = ggml_sqr(ggml_ctx, src_clone[0]);
|
||||
} else if (tensor->op == GGML_OP_SQRT) {
|
||||
|
||||
@@ -189,6 +189,10 @@ struct vk_op_push_constants {
|
||||
float param4;
|
||||
};
|
||||
|
||||
struct vk_op_sleep_push_constants {
|
||||
uint32_t ticks;
|
||||
};
|
||||
|
||||
struct vk_op_fwht_push_constants {
|
||||
uint32_t n_rows;
|
||||
uint32_t src_offset;
|
||||
|
||||
@@ -739,6 +739,7 @@ struct vk_device_struct {
|
||||
bool shader_int64;
|
||||
bool buffer_device_address;
|
||||
bool vulkan_memory_model;
|
||||
bool shader_clock;
|
||||
|
||||
bool add_rms_fusion;
|
||||
uint32_t partials_binding_alignment;
|
||||
@@ -923,6 +924,8 @@ struct vk_device_struct {
|
||||
vk_pipeline pipeline_fill_f32;
|
||||
vk_pipeline pipeline_fill_f16;
|
||||
|
||||
vk_pipeline pipeline_sleep;
|
||||
|
||||
// [src/dst 0=fp32,1=fp16,2=bf16]
|
||||
vk_pipeline pipeline_glu[GGML_GLU_OP_COUNT][3];
|
||||
|
||||
|
||||
@@ -3892,6 +3892,10 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_fill_f32, "fill_f32", fill_f32_len, fill_f32_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
|
||||
ggml_vk_create_pipeline(device, device->pipeline_fill_f16, "fill_f16", fill_f16_len, fill_f16_data, "main", 1, sizeof(vk_op_push_constants), {512, 1, 1}, {}, 1);
|
||||
|
||||
if (device->shader_clock) {
|
||||
ggml_vk_create_pipeline(device, device->pipeline_sleep, "sleep", sleep_len, sleep_data, "main", 1, sizeof(vk_op_sleep_push_constants), {1, 1, 1}, {}, 1);
|
||||
}
|
||||
|
||||
#define CREATE_GLU(name, op) \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][0], #name "_f32", name ## _f32_len, name ## _f32_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
|
||||
ggml_vk_create_pipeline(device, device->pipeline_glu[GGML_GLU_OP_ ## op][1], #name "_f16", name ## _f16_len, name ## _f16_data, "main", 3, sizeof(vk_op_glu_push_constants), {512, 1, 1}, {}, 1, true); \
|
||||
@@ -4521,6 +4525,7 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
bool ocp_microscaling_extension = false;
|
||||
bool shader_float8_extension = false;
|
||||
bool workgroup_memory_explicit_layout_support = false;
|
||||
bool shader_clock_support = false;
|
||||
|
||||
for (const auto& properties : ext_props) {
|
||||
if (strcmp("VK_KHR_maintenance4", properties.extensionName) == 0) {
|
||||
@@ -4571,6 +4576,9 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
} else if (strcmp(VK_EXT_SHADER_FLOAT8_EXTENSION_NAME, properties.extensionName) == 0) {
|
||||
shader_float8_extension = true;
|
||||
#endif
|
||||
} else if (strcmp("VK_KHR_shader_clock", properties.extensionName) == 0 &&
|
||||
!getenv("GGML_VK_DISABLE_SHADER_CLOCK")) {
|
||||
shader_clock_support = true;
|
||||
} else if (strcmp("VK_VALVE_shader_mixed_float_dot_product", properties.extensionName) == 0 &&
|
||||
!getenv("GGML_VK_DISABLE_DOT2")) {
|
||||
dot2_f16_support = true;
|
||||
@@ -4919,6 +4927,14 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
device_extensions.push_back("VK_KHR_shader_integer_dot_product");
|
||||
}
|
||||
|
||||
VkPhysicalDeviceShaderClockFeaturesKHR shader_clock_features {};
|
||||
shader_clock_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CLOCK_FEATURES_KHR;
|
||||
if (shader_clock_support) {
|
||||
last_struct->pNext = (VkBaseOutStructure *)&shader_clock_features;
|
||||
last_struct = (VkBaseOutStructure *)&shader_clock_features;
|
||||
device_extensions.push_back("VK_KHR_shader_clock");
|
||||
}
|
||||
|
||||
VkPhysicalDeviceShaderMixedFloatDotProductFeaturesVALVE dot2_features {};
|
||||
dot2_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_MIXED_FLOAT_DOT_PRODUCT_FEATURES_VALVE;
|
||||
if (dot2_f16_support) {
|
||||
@@ -5025,6 +5041,7 @@ vk_device ggml_vk_get_device(size_t idx) {
|
||||
device->shader_int64 = device_features2.features.shaderInt64;
|
||||
device->buffer_device_address = vk12_features.bufferDeviceAddress;
|
||||
device->vulkan_memory_model = vk12_features.vulkanMemoryModel;
|
||||
device->shader_clock = shader_clock_support && shader_clock_features.shaderDeviceClock;
|
||||
|
||||
if (device->subgroup_size_control) {
|
||||
device->subgroup_min_size = subgroup_size_control_props.minSubgroupSize;
|
||||
@@ -10235,6 +10252,8 @@ static vk_pipeline ggml_vk_op_get_pipeline(ggml_backend_vk_context * ctx, const
|
||||
return ctx->device->pipeline_arange_f32;
|
||||
}
|
||||
return nullptr;
|
||||
case GGML_OP_SLEEP:
|
||||
return ctx->device->pipeline_sleep;
|
||||
case GGML_OP_FILL:
|
||||
if (dst->type == GGML_TYPE_F32) {
|
||||
return ctx->device->pipeline_fill_f32;
|
||||
@@ -11471,6 +11490,31 @@ void ggml_vk_fill(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements);
|
||||
}
|
||||
|
||||
static void ggml_vk_sleep(ggml_backend_vk_context * ctx, vk_context& subctx, ggml_tensor * dst) {
|
||||
VK_LOG_DEBUG("ggml_vk_sleep(dst=" << dst << ", us=" << ggml_get_op_params_i32(dst, 0) << ")");
|
||||
|
||||
// Vulkan does not specify the period of the shader realtime clock this assumes it matches the timestamp counter.
|
||||
// A driver where this does not hold could sleep too short or too long by that ratio
|
||||
const float ns_per_tick = ctx->device->properties.limits.timestampPeriod;
|
||||
const uint64_t ns = 1000 * (uint64_t) ggml_get_op_params_i32(dst, 0);
|
||||
const uint64_t ticks = std::min<uint64_t>(ns_per_tick > 0.0f ? (uint64_t)(ns / ns_per_tick) : ns, std::numeric_limits<uint32_t>::max());
|
||||
|
||||
vk_op_sleep_push_constants pc = {
|
||||
(uint32_t) ticks,
|
||||
};
|
||||
|
||||
vk_pipeline pipeline = ggml_vk_op_get_pipeline(ctx, nullptr, nullptr, nullptr, dst, GGML_OP_SLEEP);
|
||||
GGML_ASSERT(pipeline != nullptr);
|
||||
|
||||
ggml_pipeline_request_descriptor_sets(ctx, pipeline, 1);
|
||||
vk_subbuffer dst_buf = ggml_vk_tensor_subbuffer(ctx, dst);
|
||||
|
||||
// a single invocation spins, the buffer is bound only to keep the spin from being optimized out
|
||||
std::array<uint32_t, 3> elements = { 1, 1, 1 };
|
||||
|
||||
ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { dst_buf }, pc, elements);
|
||||
}
|
||||
|
||||
void ggml_vk_sin(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, ggml_tensor * dst) {
|
||||
ggml_vk_op_f32(ctx, subctx, src0, nullptr, nullptr, nullptr, dst, GGML_OP_SIN, vk_op_unary_push_constants_init(src0, dst));
|
||||
}
|
||||
@@ -13317,6 +13361,10 @@ bool ggml_vk_build_graph(ggml_backend_vk_context * ctx, ggml_cgraph * cgraph, in
|
||||
case GGML_OP_FILL:
|
||||
ggml_vk_fill(ctx, compute_ctx, node);
|
||||
|
||||
break;
|
||||
case GGML_OP_SLEEP:
|
||||
ggml_vk_sleep(ctx, compute_ctx, node);
|
||||
|
||||
break;
|
||||
case GGML_OP_SCALE:
|
||||
if (ctx->fused_hc_post_gate) {
|
||||
@@ -16768,6 +16816,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
||||
return op->type == GGML_TYPE_F32;
|
||||
case GGML_OP_FILL:
|
||||
return op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16;
|
||||
case GGML_OP_SLEEP:
|
||||
return device->shader_clock;
|
||||
case GGML_OP_SCALE:
|
||||
return ggml_is_contiguous(op->src[0]) && ggml_vk_is_float_elementwise(op);
|
||||
case GGML_OP_PAD:
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
#version 450
|
||||
|
||||
#extension GL_EXT_shader_realtime_clock : require
|
||||
|
||||
layout(local_size_x = 1, local_size_y = 1, local_size_z = 1) in;
|
||||
|
||||
// never written, it only anchors the spin below so the compiler cannot drop it
|
||||
layout (binding = 0) writeonly buffer D {uint data_d[];};
|
||||
|
||||
layout (push_constant) uniform parameter {
|
||||
uint ticks;
|
||||
} p;
|
||||
|
||||
// bounds the spin in case the clock does not advance, turning a hang into a too-short sleep
|
||||
const uint MAX_ITER = 1u << 27;
|
||||
|
||||
void main() {
|
||||
const uvec2 t0 = clockRealtime2x32EXT();
|
||||
|
||||
uvec2 t = t0;
|
||||
|
||||
for (uint iter = 0; iter < MAX_ITER; ++iter) {
|
||||
t = clockRealtime2x32EXT();
|
||||
|
||||
const uint lo = t.x - t0.x;
|
||||
const uint hi = t.y - t0.y - (t.x < t0.x ? 1u : 0u);
|
||||
|
||||
if (hi != 0u || lo >= p.ticks) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// the store never runs, but the compiler cannot prove it and has to keep the spin
|
||||
if (t.x == 0xFFFFFFFFu && t.y == 0xFFFFFFFFu) {
|
||||
data_d[0] = t.x;
|
||||
}
|
||||
}
|
||||
@@ -1028,6 +1028,7 @@ void process_shaders() {
|
||||
string_to_spv("arange_f32", "arange.comp", {{"A_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
|
||||
string_to_spv("fill_f32", "fill.comp", {{"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}});
|
||||
string_to_spv("fill_f16", "fill.comp", {{"D_TYPE", "float16_t"}, {"FLOAT_TYPE", "float"}});
|
||||
string_to_spv("sleep", "sleep.comp", {});
|
||||
|
||||
string_to_spv("silu_back_f32", "silu_back.comp", {{"A_TYPE", "float"}, {"B_TYPE", "float"}, {"D_TYPE", "float"}});
|
||||
|
||||
|
||||
@@ -1272,6 +1272,7 @@ class ggml_webgpu_shader_lib {
|
||||
std::unordered_map<int, webgpu_pipeline> arange_pipelines; // key is fixed, no variants yet
|
||||
std::unordered_map<int, webgpu_pipeline> count_equal_pipelines; // key is src_overlap
|
||||
std::unordered_map<int, webgpu_pipeline> timestep_embedding_pipelines; // key is fixed, no variants yet
|
||||
std::unordered_map<int, webgpu_pipeline> pool2d_pipelines; // key is ggml_op_pool
|
||||
std::unordered_map<ggml_webgpu_row_norm_pipeline_key, webgpu_pipeline, ggml_webgpu_row_norm_pipeline_key_hash>
|
||||
row_norm_pipelines; // op/inplace
|
||||
|
||||
@@ -2067,6 +2068,34 @@ class ggml_webgpu_shader_lib {
|
||||
return arange_pipelines[1];
|
||||
}
|
||||
|
||||
webgpu_pipeline get_pool2d_pipeline(const ggml_webgpu_shader_lib_context & context) {
|
||||
const int key = ggml_get_op_params_i32(context.dst, 0);
|
||||
|
||||
auto it = pool2d_pipelines.find(key);
|
||||
if (it != pool2d_pipelines.end()) {
|
||||
return it->second;
|
||||
}
|
||||
|
||||
std::vector<std::string> defines;
|
||||
std::string variant = "pool2d";
|
||||
if (key == GGML_OP_POOL_MAX) {
|
||||
defines.push_back("POOL_MAX");
|
||||
variant += "_max";
|
||||
} else {
|
||||
variant += "_avg";
|
||||
}
|
||||
|
||||
defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size));
|
||||
|
||||
auto processed = preprocessor.preprocess(wgsl_pool2d, defines);
|
||||
auto decisions = std::make_shared<ggml_webgpu_generic_shader_decisions>();
|
||||
decisions->wg_size = context.max_wg_size;
|
||||
webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, variant);
|
||||
pipeline.context = decisions;
|
||||
pool2d_pipelines[key] = pipeline;
|
||||
return pool2d_pipelines[key];
|
||||
}
|
||||
|
||||
webgpu_pipeline get_pad_pipeline(const ggml_webgpu_shader_lib_context & context) {
|
||||
ggml_webgpu_pad_pipeline_key key = {};
|
||||
key.reflect = context.dst->op == GGML_OP_PAD_REFLECT_1D;
|
||||
|
||||
@@ -961,6 +961,46 @@ static webgpu_encoded_op ggml_webgpu_pad(webgpu_context & ctx, ggml_tensor * src
|
||||
return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y);
|
||||
}
|
||||
|
||||
static webgpu_encoded_op ggml_webgpu_pool2d(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) {
|
||||
ggml_webgpu_shader_lib_context shader_lib_ctx = {};
|
||||
shader_lib_ctx.src0 = src;
|
||||
shader_lib_ctx.dst = dst;
|
||||
shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup;
|
||||
|
||||
webgpu_pipeline pipeline = ctx->shader_lib->get_pool2d_pipeline(shader_lib_ctx);
|
||||
|
||||
auto * decisions = static_cast<ggml_webgpu_generic_shader_decisions *>(pipeline.context.get());
|
||||
|
||||
const uint32_t ne = (uint32_t) ggml_nelements(dst);
|
||||
|
||||
std::vector<uint32_t> params = {
|
||||
ne,
|
||||
(uint32_t) (ggml_webgpu_tensor_misalignment(ctx, src) / ggml_type_size(src->type)),
|
||||
(uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)),
|
||||
(uint32_t) src->ne[0],
|
||||
(uint32_t) src->ne[1],
|
||||
(uint32_t) dst->ne[0],
|
||||
(uint32_t) dst->ne[1],
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 1), // k0
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 2), // k1
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 3), // s0
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 4), // s1
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 5), // p0
|
||||
(uint32_t) ggml_get_op_params_i32(dst, 6), // p1
|
||||
};
|
||||
|
||||
std::vector<wgpu::BindGroupEntry> entries = {
|
||||
ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, src),
|
||||
ggml_webgpu_make_tensor_bind_group_entry(ctx, 1, dst),
|
||||
};
|
||||
|
||||
uint32_t wg_x;
|
||||
uint32_t wg_y;
|
||||
uint32_t total_wg = CEIL_DIV(ne, decisions->wg_size);
|
||||
compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y);
|
||||
return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y);
|
||||
}
|
||||
|
||||
static webgpu_encoded_op ggml_webgpu_solve_tri(webgpu_context & ctx,
|
||||
ggml_tensor * src0,
|
||||
ggml_tensor * src1,
|
||||
@@ -3670,6 +3710,8 @@ static std::optional<webgpu_encoded_op> ggml_webgpu_encode(webgpu_context ctx,
|
||||
}
|
||||
case GGML_OP_ARANGE:
|
||||
return ggml_webgpu_arange(ctx, node);
|
||||
case GGML_OP_POOL_2D:
|
||||
return ggml_webgpu_pool2d(ctx, src0, node);
|
||||
case GGML_OP_PAD_REFLECT_1D:
|
||||
case GGML_OP_PAD:
|
||||
return ggml_webgpu_pad(ctx, src0, node);
|
||||
@@ -5219,6 +5261,9 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const
|
||||
case GGML_OP_ARANGE:
|
||||
supports_op = op->type == GGML_TYPE_F32 && ggml_is_contiguous(op);
|
||||
break;
|
||||
case GGML_OP_POOL_2D:
|
||||
supports_op = op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32 && ggml_is_contiguous(src0);
|
||||
break;
|
||||
case GGML_OP_PAD_REFLECT_1D:
|
||||
supports_op = op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32;
|
||||
break;
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
@group(0) @binding(0)
|
||||
var<storage, read_write> src: array<f32>;
|
||||
|
||||
@group(0) @binding(1)
|
||||
var<storage, read_write> dst: array<f32>;
|
||||
|
||||
struct Params {
|
||||
ne: u32, // total number of elements
|
||||
offset_src: u32, // in elements
|
||||
offset_dst: u32, // in elements
|
||||
|
||||
// Logical shapes
|
||||
src_ne0: u32,
|
||||
src_ne1: u32,
|
||||
|
||||
dst_ne0: u32,
|
||||
dst_ne1: u32,
|
||||
|
||||
k0: u32,
|
||||
k1: u32,
|
||||
s0: u32,
|
||||
s1: u32,
|
||||
p0: u32,
|
||||
p1: u32
|
||||
}
|
||||
|
||||
|
||||
@group(0) @binding(2)
|
||||
var<uniform> params: Params;
|
||||
|
||||
@compute @workgroup_size(WG_SIZE)
|
||||
fn main(@builtin(num_workgroups) num_wg: vec3<u32>,
|
||||
@builtin(global_invocation_id) gid: vec3<u32>) {
|
||||
|
||||
let gid_i = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y;
|
||||
if (gid_i >= params.ne) {
|
||||
return;
|
||||
}
|
||||
|
||||
let out_x = gid_i % params.dst_ne0;
|
||||
let out_y = (gid_i / params.dst_ne0) % params.dst_ne1;
|
||||
let out_z = (gid_i / (params.dst_ne0 * params.dst_ne1));
|
||||
|
||||
let x0 = i32(out_x * params.s0) - i32(params.p0);
|
||||
let y0 = i32(out_y * params.s1) - i32(params.p1);
|
||||
let in_x_start = max(0, x0);
|
||||
let in_x_end = min(i32(params.src_ne0), x0 + i32(params.k0));
|
||||
let in_y_start = max(0, y0);
|
||||
let in_y_end = min(i32(params.src_ne1), y0 + i32(params.k1));
|
||||
|
||||
let base = params.offset_src + out_z * params.src_ne0 * params.src_ne1;
|
||||
|
||||
#ifdef POOL_MAX
|
||||
var res: f32 = bitcast<f32>(0xff7fffffu);
|
||||
#else
|
||||
var res: f32 = 0.0;
|
||||
#endif
|
||||
|
||||
for(var y = in_y_start; y < in_y_end; y++){
|
||||
let row = base + u32(y) * params.src_ne0;
|
||||
for(var x = in_x_start; x < in_x_end; x++){
|
||||
#ifdef POOL_MAX
|
||||
res = max(res, src[row + u32(x)]);
|
||||
#else
|
||||
res += src[row + u32(x)];
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
#ifndef POOL_MAX
|
||||
res = res / f32(params.k0 * params.k1);
|
||||
#endif
|
||||
dst[params.offset_dst + gid_i] = res;
|
||||
}
|
||||
+27
-2
@@ -1112,9 +1112,11 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
|
||||
"OPT_STEP_SGD",
|
||||
|
||||
"GLU",
|
||||
|
||||
"SLEEP",
|
||||
};
|
||||
|
||||
static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99");
|
||||
static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100");
|
||||
|
||||
static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
|
||||
"none",
|
||||
@@ -1225,9 +1227,11 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
|
||||
"sgd(x)",
|
||||
|
||||
"glu(x)",
|
||||
|
||||
"sleep(x)",
|
||||
};
|
||||
|
||||
static_assert(GGML_OP_COUNT == 99, "GGML_OP_COUNT != 99");
|
||||
static_assert(GGML_OP_COUNT == 100, "GGML_OP_COUNT != 100");
|
||||
|
||||
static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
|
||||
|
||||
@@ -6176,6 +6180,27 @@ struct ggml_tensor * ggml_custom_inplace(
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ggml_sleep
|
||||
|
||||
// the result aliases a, the op only delays and never touches the data
|
||||
struct ggml_tensor * ggml_sleep(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a,
|
||||
int32_t us) {
|
||||
GGML_ASSERT(us >= 0);
|
||||
|
||||
struct ggml_tensor * result = ggml_view_tensor(ctx, a);
|
||||
ggml_format_name(result, "%s (sleep)", a->name);
|
||||
|
||||
ggml_set_op_params_i32(result, 0, us);
|
||||
|
||||
result->op = GGML_OP_SLEEP;
|
||||
result->src[0] = a;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// ggml_cross_entropy_loss
|
||||
|
||||
struct ggml_tensor * ggml_cross_entropy_loss(
|
||||
|
||||
@@ -3095,7 +3095,6 @@ llm_graph_params llama_context::graph_params(
|
||||
llm_graph_type gtype) const {
|
||||
return {
|
||||
/*.model =*/ &model,
|
||||
/*.hparams =*/ model.hparams,
|
||||
/*.cparams =*/ cparams,
|
||||
/*.ubatch =*/ ubatch,
|
||||
/*.gtype =*/ gtype,
|
||||
@@ -3106,8 +3105,6 @@ llm_graph_params llama_context::graph_params(
|
||||
/*.mctx =*/ mctx,
|
||||
/*.cross =*/ &cross,
|
||||
/*.moe_cache =*/ moe_cache.get(),
|
||||
/*.prec_policy =*/ &model.prec_policy,
|
||||
/*.hdmd =*/ model.hdmd.rot.empty() ? nullptr : &model.hdmd,
|
||||
/*.layer_inp_dev =*/ layer_inp_dev,
|
||||
/*.layer_inp_src =*/ supports_embd_row() ? cparams.ctx_other->get_layer_inp_dev() : nullptr,
|
||||
/*.h_nextn_dev =*/ cparams.embeddings_nextn_dev ? h_nextn_dev : nullptr,
|
||||
|
||||
+213
-34
@@ -1400,6 +1400,91 @@ bool llm_graph_input_mem_hybrid_iswa::can_reuse(const llm_graph_params & params)
|
||||
return res;
|
||||
}
|
||||
|
||||
void llm_graph_input_mem_hybrid_iswa_rel::set_input(const llama_ubatch * ubatch) {
|
||||
const auto * attn_ctx = mctx->get_attn();
|
||||
|
||||
// base tensors may not be allocated if there are no non-SWA attention layers
|
||||
if (inp_attn->self_k_idxs && inp_attn->self_k_idxs->buffer) {
|
||||
attn_ctx->get_base()->set_input_k_idxs(inp_attn->self_k_idxs, ubatch);
|
||||
attn_ctx->get_base()->set_input_v_idxs(inp_attn->self_v_idxs, ubatch);
|
||||
}
|
||||
|
||||
if (inp_attn->self_attn_mask.raw && inp_attn->self_attn_mask.raw->buffer) {
|
||||
attn_ctx->get_base()->set_input_attn_mask(inp_attn->self_attn_mask, ubatch, cparams.causal_attn);
|
||||
}
|
||||
|
||||
if (self_rel_pos->buffer) {
|
||||
attn_ctx->get_base()->set_input_relative_pos(self_rel_pos, ubatch, extent);
|
||||
}
|
||||
|
||||
// swa tensors may not be allocated if there are no SWA attention layers
|
||||
if (inp_attn->self_k_idxs_swa && inp_attn->self_k_idxs_swa->buffer) {
|
||||
attn_ctx->get_swa()->set_input_k_idxs(inp_attn->self_k_idxs_swa, ubatch);
|
||||
attn_ctx->get_swa()->set_input_v_idxs(inp_attn->self_v_idxs_swa, ubatch);
|
||||
}
|
||||
|
||||
if (inp_attn->self_attn_mask_swa.raw && inp_attn->self_attn_mask_swa.raw->buffer) {
|
||||
attn_ctx->get_swa()->set_input_attn_mask(inp_attn->self_attn_mask_swa, ubatch, cparams.causal_attn);
|
||||
}
|
||||
|
||||
if (self_rel_pos_swa->buffer) {
|
||||
attn_ctx->get_swa()->set_input_relative_pos(self_rel_pos_swa, ubatch, extent_swa);
|
||||
}
|
||||
|
||||
const int64_t n_rs = mctx->get_recr()->get_n_rs();
|
||||
|
||||
if (inp_rs->s_copy) {
|
||||
GGML_ASSERT(ggml_backend_buffer_is_host(inp_rs->s_copy->buffer));
|
||||
int32_t * data = (int32_t *) inp_rs->s_copy->data;
|
||||
|
||||
// assuming copy destinations ALWAYS happen ONLY on the cells between head and head+n
|
||||
for (uint32_t i = 0; i < n_rs; ++i) {
|
||||
data[i] = mctx->get_recr()->s_copy(i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool llm_graph_input_mem_hybrid_iswa_rel::can_reuse(const llm_graph_params & params) {
|
||||
const auto * mctx = static_cast<const llama_memory_hybrid_iswa_context *>(params.mctx);
|
||||
|
||||
this->mctx = mctx;
|
||||
|
||||
bool res = true;
|
||||
|
||||
const auto * attn_ctx = mctx->get_attn();
|
||||
|
||||
// the masks are always dense, see build_inp_mem_hybrid_iswa_rel()
|
||||
|
||||
// base tensors may not be allocated if there are no non-SWA attention layers
|
||||
if (inp_attn->self_k_idxs && inp_attn->self_k_idxs->buffer) {
|
||||
res &= inp_attn->self_k_idxs->ne[0] == params.ubatch.n_tokens;
|
||||
}
|
||||
|
||||
res &= can_reuse_kq_mask(inp_attn->self_attn_mask.raw, attn_ctx->get_base(), params.ubatch, params.cparams);
|
||||
|
||||
res &= self_rel_pos->ne[0] == attn_ctx->get_base()->get_n_kv();
|
||||
res &= self_rel_pos->ne[1] == params.ubatch.n_tokens;
|
||||
|
||||
// swa tensors may not be allocated if there are no SWA attention layers
|
||||
if (inp_attn->self_k_idxs_swa && inp_attn->self_k_idxs_swa->buffer) {
|
||||
res &= inp_attn->self_k_idxs_swa->ne[0] == params.ubatch.n_tokens;
|
||||
}
|
||||
|
||||
res &= can_reuse_kq_mask(inp_attn->self_attn_mask_swa.raw, attn_ctx->get_swa(), params.ubatch, params.cparams);
|
||||
|
||||
res &= self_rel_pos_swa->ne[0] == attn_ctx->get_swa()->get_n_kv();
|
||||
res &= self_rel_pos_swa->ne[1] == params.ubatch.n_tokens;
|
||||
|
||||
res &= inp_rs->s_copy->ne[0] == mctx->get_recr()->get_n_rs();
|
||||
|
||||
res &= inp_rs->s_copy_main->ne[0] == params.ubatch.n_seqs;
|
||||
|
||||
res &= inp_rs->head == mctx->get_recr()->get_head();
|
||||
res &= inp_rs->rs_z == mctx->get_recr()->get_rs_z();
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void llm_graph_input_sampling::set_input(const llama_ubatch * ubatch) {
|
||||
// set the inputs only for the active samplers in the current ubatch
|
||||
std::unordered_set<llama_seq_id> active_samplers;
|
||||
@@ -1594,9 +1679,10 @@ void llm_graph_result::set_params(const llm_graph_params & params) {
|
||||
|
||||
llm_graph_context::llm_graph_context(const llm_graph_params & params) :
|
||||
model (*params.model),
|
||||
hparams (params.hparams),
|
||||
hparams (model.hparams),
|
||||
cparams (params.cparams),
|
||||
ubatch (params.ubatch),
|
||||
prec_policy (model.prec_policy),
|
||||
n_embd (hparams.n_embd),
|
||||
n_layer (hparams.n_layer()),
|
||||
n_layer_nextn (hparams.n_layer_nextn),
|
||||
@@ -1630,8 +1716,7 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) :
|
||||
mctx (params.mctx),
|
||||
cross (params.cross),
|
||||
moe_cache (params.moe_cache),
|
||||
prec_policy (params.prec_policy),
|
||||
hdmd (params.hdmd),
|
||||
hdmd (model.hdmd.rot.empty() ? nullptr : &model.hdmd),
|
||||
layer_inp_dev (params.layer_inp_dev),
|
||||
layer_inp_src (params.layer_inp_src),
|
||||
h_nextn_dev (params.h_nextn_dev),
|
||||
@@ -1728,9 +1813,7 @@ ggml_tensor * llm_graph_context::build_lora_mm(
|
||||
ggml_tensor * cur_mm = build_hadamard_input(w, cur);
|
||||
ggml_tensor * res = ggml_mul_mat(ctx0, w, cur_mm);
|
||||
|
||||
if (prec_policy) {
|
||||
prec_policy->apply(res);
|
||||
}
|
||||
prec_policy.apply(res);
|
||||
|
||||
if (w->type == GGML_TYPE_NVFP4) {
|
||||
ggml_prec_set_acc(res, GGML_PREC_BF16);
|
||||
@@ -1812,9 +1895,7 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
|
||||
ggml_mul_mat_id(ctx0, w, cur_mm, ids) :
|
||||
ggml_mul_mat_id(ctx0, cached ? moe_cache->get_experts(w) : w, cur_mm, slots);
|
||||
|
||||
if (prec_policy) {
|
||||
prec_policy->apply(res);
|
||||
}
|
||||
prec_policy.apply(res);
|
||||
|
||||
if (w->type == GGML_TYPE_NVFP4) {
|
||||
ggml_prec_set_acc(res, GGML_PREC_BF16);
|
||||
@@ -3117,7 +3198,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
|
||||
ggml_tensor * cur;
|
||||
|
||||
const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr && (!sinks || ggml_is_vector(sinks));
|
||||
const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr;
|
||||
GGML_ASSERT(use_flash_attn || kv_rows == nullptr);
|
||||
if (use_flash_attn) {
|
||||
GGML_ASSERT(kq_b == nullptr && "Flash attention does not support KQ bias yet");
|
||||
@@ -3200,16 +3281,8 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
cb(kq, "kq_plus_kq_b", il);
|
||||
}
|
||||
|
||||
if (sinks && !ggml_is_vector(sinks)) {
|
||||
// Per-query sinks are extra logits with zero values.
|
||||
const int64_t n_kv = kq->ne[0];
|
||||
kq = ggml_add(ctx0, ggml_scale(ctx0, kq, kq_scale), ggml_cast(ctx0, kq_mask, GGML_TYPE_F32));
|
||||
kq = ggml_soft_max(ctx0, ggml_concat(ctx0, kq, sinks, 0));
|
||||
kq = ggml_cont(ctx0, ggml_view_4d(ctx0, kq, n_kv, kq->ne[1], kq->ne[2], kq->ne[3], kq->nb[1], kq->nb[2], kq->nb[3], 0));
|
||||
} else {
|
||||
kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias);
|
||||
ggml_soft_max_add_sinks(kq, sinks);
|
||||
}
|
||||
kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, hparams.f_max_alibi_bias);
|
||||
ggml_soft_max_add_sinks(kq, sinks);
|
||||
cb(kq, "kq_soft_max", il);
|
||||
|
||||
if (!v_trans) {
|
||||
@@ -3691,7 +3764,9 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_attn(
|
||||
// the iswa build_attn with the KQ mask and kv_rows given by the caller, which has to keep them in sync
|
||||
static ggml_tensor * build_attn_iswa_impl(
|
||||
const llm_graph_context & gctx,
|
||||
llm_graph_input_attn_kv_iswa * inp,
|
||||
ggml_tensor * wo,
|
||||
ggml_tensor * wo_b,
|
||||
@@ -3700,11 +3775,18 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * k_cur,
|
||||
ggml_tensor * v_cur,
|
||||
ggml_tensor * kq_b,
|
||||
ggml_tensor * kq_mask,
|
||||
ggml_tensor * kv_rows,
|
||||
ggml_tensor * sinks,
|
||||
ggml_tensor * v_mla,
|
||||
float kq_scale,
|
||||
int il,
|
||||
ggml_tensor * kq_mask) const {
|
||||
int il) {
|
||||
ggml_context * ctx0 = gctx.ctx0;
|
||||
ggml_cgraph * gf = gctx.gf;
|
||||
|
||||
const auto & hparams = gctx.hparams;
|
||||
const auto & cparams = gctx.cparams;
|
||||
|
||||
const bool is_swa = hparams.is_swa(il);
|
||||
|
||||
auto * k_rot = is_swa ? inp->self_k_rot_swa[il] : inp->self_k_rot[il];
|
||||
@@ -3742,7 +3824,7 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
// whole seq fits into batch in training mode
|
||||
const bool use_kv_cur = cparams.training && k_cur && v_cur;
|
||||
if (use_kv_cur) {
|
||||
GGML_ASSERT(mctx_cur->get_n_kv() == n_tokens);
|
||||
GGML_ASSERT(mctx_cur->get_n_kv() == gctx.n_tokens);
|
||||
}
|
||||
|
||||
const bool store_kv = !use_kv_cur || hparams.n_layer_kv_from_start >= 0;
|
||||
@@ -3760,25 +3842,20 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il));
|
||||
}
|
||||
|
||||
if (!kq_mask) {
|
||||
kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw;
|
||||
}
|
||||
const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows;
|
||||
|
||||
ggml_tensor * q = q_cur;
|
||||
ggml_tensor * k = use_kv_cur ? k_cur : mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = use_kv_cur ? v_cur : mctx_cur->get_v(ctx0, il);
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, 0, kq_scale, il);
|
||||
ggml_tensor * cur = gctx.build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, 0, kq_scale, il);
|
||||
cur = build_attn_scale_kv_output(ctx0, mctx_cur, cur, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
gctx.cb(cur, "kqv_out", il);
|
||||
|
||||
if (v_rot) {
|
||||
cur = llama_mul_mat_hadamard(ctx0, cur, v_rot);
|
||||
}
|
||||
|
||||
if (wo) {
|
||||
cur = build_lora_mm(wo, cur, wo_s);
|
||||
cur = gctx.build_lora_mm(wo, cur, wo_s);
|
||||
}
|
||||
|
||||
if (wo_b) {
|
||||
@@ -3792,6 +3869,27 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
return cur;
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_attn(
|
||||
llm_graph_input_attn_kv_iswa * inp,
|
||||
ggml_tensor * wo,
|
||||
ggml_tensor * wo_b,
|
||||
ggml_tensor * wo_s,
|
||||
ggml_tensor * q_cur,
|
||||
ggml_tensor * k_cur,
|
||||
ggml_tensor * v_cur,
|
||||
ggml_tensor * kq_b,
|
||||
ggml_tensor * sinks,
|
||||
ggml_tensor * v_mla,
|
||||
float kq_scale,
|
||||
int il) const {
|
||||
const bool is_swa = hparams.is_swa(il);
|
||||
|
||||
const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw;
|
||||
const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows;
|
||||
|
||||
return build_attn_iswa_impl(*this, inp, wo, wo_b, wo_s, q_cur, k_cur, v_cur, kq_b, kq_mask, kv_rows, sinks, v_mla, kq_scale, il);
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_attn(
|
||||
llm_graph_input_attn_k_iswa * inp,
|
||||
ggml_tensor * wo,
|
||||
@@ -4310,6 +4408,43 @@ llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa()
|
||||
|
||||
auto inp_attn = std::make_unique<llm_graph_input_attn_kv_iswa>(hparams, cparams, attn_ctx);
|
||||
|
||||
{
|
||||
inp_attn->self_k_idxs = attn_ctx->get_base()->build_input_k_idxs(ctx0, ubatch);
|
||||
inp_attn->self_v_idxs = attn_ctx->get_base()->build_input_v_idxs(ctx0, ubatch);
|
||||
|
||||
inp_attn->self_attn_mask = build_attn_inp_attn_mask(ctx0, attn_ctx->get_base(), ubatch, cparams);
|
||||
}
|
||||
|
||||
{
|
||||
inp_attn->self_k_idxs_swa = attn_ctx->get_swa()->build_input_k_idxs(ctx0, ubatch);
|
||||
inp_attn->self_v_idxs_swa = attn_ctx->get_swa()->build_input_v_idxs(ctx0, ubatch);
|
||||
|
||||
inp_attn->self_attn_mask_swa = build_attn_inp_attn_mask(ctx0, attn_ctx->get_swa(), ubatch, cparams);
|
||||
}
|
||||
|
||||
inp_attn->self_k_rot = attn_ctx->get_base()->get_k_rot();
|
||||
inp_attn->self_v_rot = attn_ctx->get_base()->get_v_rot();
|
||||
|
||||
inp_attn->self_k_rot_swa = attn_ctx->get_swa()->get_k_rot();
|
||||
inp_attn->self_v_rot_swa = attn_ctx->get_swa()->get_v_rot();
|
||||
|
||||
auto inp = std::make_unique<llm_graph_input_mem_hybrid_iswa>(cparams, std::move(inp_attn), std::move(inp_rs), mctx_cur);
|
||||
|
||||
return (llm_graph_input_mem_hybrid_iswa *) res->add_input(std::move(inp));
|
||||
}
|
||||
|
||||
llm_graph_input_mem_hybrid_iswa_rel * llm_graph_context::build_inp_mem_hybrid_iswa_rel(
|
||||
uint32_t extent,
|
||||
uint32_t extent_swa) const {
|
||||
const auto * mctx_cur = static_cast<const llama_memory_hybrid_iswa_context *>(mctx);
|
||||
|
||||
auto inp_rs = build_rs_inp_impl(ctx0, ubatch, mctx_cur->get_recr());
|
||||
|
||||
const auto * attn_ctx = mctx_cur->get_attn();
|
||||
|
||||
auto inp_attn = std::make_unique<llm_graph_input_attn_kv_iswa>(hparams, cparams, attn_ctx);
|
||||
|
||||
// dense masks: the bias is gathered by the distance to each cell of the K/V view
|
||||
{
|
||||
inp_attn->self_k_idxs = attn_ctx->get_base()->build_input_k_idxs(ctx0, ubatch);
|
||||
inp_attn->self_v_idxs = attn_ctx->get_base()->build_input_v_idxs(ctx0, ubatch);
|
||||
@@ -4330,9 +4465,53 @@ llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa()
|
||||
inp_attn->self_k_rot_swa = attn_ctx->get_swa()->get_k_rot();
|
||||
inp_attn->self_v_rot_swa = attn_ctx->get_swa()->get_v_rot();
|
||||
|
||||
auto inp = std::make_unique<llm_graph_input_mem_hybrid_iswa>(cparams, std::move(inp_attn), std::move(inp_rs), mctx_cur);
|
||||
auto inp = std::make_unique<llm_graph_input_mem_hybrid_iswa_rel>(cparams, std::move(inp_attn), std::move(inp_rs), mctx_cur, extent, extent_swa);
|
||||
|
||||
return (llm_graph_input_mem_hybrid_iswa *) res->add_input(std::move(inp));
|
||||
inp->self_rel_pos = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, attn_ctx->get_base()->get_n_kv(), ubatch.n_tokens);
|
||||
ggml_set_input(inp->self_rel_pos);
|
||||
ggml_set_name(inp->self_rel_pos, "attn_inp_rel_pos");
|
||||
|
||||
inp->self_rel_pos_swa = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, attn_ctx->get_swa()->get_n_kv(), ubatch.n_tokens);
|
||||
ggml_set_input(inp->self_rel_pos_swa);
|
||||
ggml_set_name(inp->self_rel_pos_swa, "attn_inp_rel_pos_swa");
|
||||
|
||||
return (llm_graph_input_mem_hybrid_iswa_rel *) res->add_input(std::move(inp));
|
||||
}
|
||||
|
||||
ggml_tensor * llm_graph_context::build_attn(
|
||||
llm_graph_input_mem_hybrid_iswa_rel * inp,
|
||||
ggml_tensor * wo,
|
||||
ggml_tensor * wo_b,
|
||||
ggml_tensor * wo_s,
|
||||
ggml_tensor * q_cur,
|
||||
ggml_tensor * k_cur,
|
||||
ggml_tensor * v_cur,
|
||||
ggml_tensor * kq_b,
|
||||
ggml_tensor * sinks,
|
||||
float kq_scale,
|
||||
int il) const {
|
||||
const bool is_swa = hparams.is_swa(il);
|
||||
|
||||
auto * inp_attn = inp->get_attn();
|
||||
|
||||
const auto & attn_mask = is_swa ? inp_attn->self_attn_mask_swa : inp_attn->self_attn_mask;
|
||||
|
||||
// TODO: support kv_rows - fill the distances over the kv_rows cell lists and pass the rows to build_attn_mha,
|
||||
// needs a kv_rows flash attention with a per-head mask (CUDA has none)
|
||||
GGML_ASSERT(attn_mask.rows == nullptr);
|
||||
|
||||
// a mask per head, in the dense layout of the input mask, which flash attention takes in F16
|
||||
ggml_tensor * kq_mask = attn_mask.raw;
|
||||
if (kq_mask->type != GGML_TYPE_F32) {
|
||||
kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F32);
|
||||
}
|
||||
kq_mask = ggml_add(ctx0, kq_b, kq_mask);
|
||||
if (cparams.flash_attn) {
|
||||
kq_mask = ggml_cast(ctx0, kq_mask, GGML_TYPE_F16);
|
||||
}
|
||||
cb(kq_mask, "kq_mask_rel_pos", il);
|
||||
|
||||
return build_attn_iswa_impl(*this, inp_attn, wo, wo_b, wo_s, q_cur, k_cur, v_cur, nullptr, kq_mask, nullptr, sinks, nullptr, kq_scale, il);
|
||||
}
|
||||
|
||||
void llm_graph_context::build_dense_out(
|
||||
|
||||
+64
-9
@@ -774,6 +774,48 @@ public:
|
||||
const llama_memory_hybrid_iswa_context * mctx;
|
||||
};
|
||||
|
||||
// hybrid iswa memory with a per-head attention bias that depends on the distance from the query to the cell (Inkling)
|
||||
// the KQ masks are always dense (rows == nullptr) and are filled from the same cells as the distances, so that the bias
|
||||
// gathered by distance lines up with the mask
|
||||
class llm_graph_input_mem_hybrid_iswa_rel : public llm_graph_input_i {
|
||||
public:
|
||||
llm_graph_input_mem_hybrid_iswa_rel(
|
||||
const llama_cparams & cparams,
|
||||
std::unique_ptr<llm_graph_input_attn_kv_iswa> inp_attn,
|
||||
std::unique_ptr<llm_graph_input_rs> inp_rs,
|
||||
const llama_memory_hybrid_iswa_context * mctx,
|
||||
uint32_t extent,
|
||||
uint32_t extent_swa) :
|
||||
inp_attn(std::move(inp_attn)),
|
||||
inp_rs(std::move(inp_rs)),
|
||||
extent(extent),
|
||||
extent_swa(extent_swa),
|
||||
cparams(cparams),
|
||||
mctx(mctx) { }
|
||||
virtual ~llm_graph_input_mem_hybrid_iswa_rel() = default;
|
||||
|
||||
void set_input(const llama_ubatch * ubatch) override;
|
||||
|
||||
bool can_reuse(const llm_graph_params & params) override;
|
||||
|
||||
std::unique_ptr<llm_graph_input_attn_kv_iswa> inp_attn;
|
||||
std::unique_ptr<llm_graph_input_rs> inp_rs;
|
||||
|
||||
llm_graph_input_attn_kv_iswa * get_attn() const { return inp_attn.get(); }
|
||||
llm_graph_input_rs * get_recr() const { return inp_rs.get(); }
|
||||
|
||||
// distance from the token to the cell, extent for the empty, future and more distant cells
|
||||
ggml_tensor * self_rel_pos = nullptr; // I32 [n_kv, n_batch]
|
||||
ggml_tensor * self_rel_pos_swa = nullptr; // I32 [n_kv_swa, n_batch]
|
||||
|
||||
const uint32_t extent;
|
||||
const uint32_t extent_swa;
|
||||
|
||||
const llama_cparams cparams;
|
||||
|
||||
const llama_memory_hybrid_iswa_context * mctx;
|
||||
};
|
||||
|
||||
class llm_graph_input_sampling : public llm_graph_input_i {
|
||||
public:
|
||||
llm_graph_input_sampling(std::map<llama_seq_id, llama_sampler *> samplers) :
|
||||
@@ -804,7 +846,6 @@ class llm_graph_result;
|
||||
struct llm_graph_params {
|
||||
const llama_model * model = nullptr;
|
||||
|
||||
llama_hparams hparams;
|
||||
llama_cparams cparams;
|
||||
|
||||
llama_ubatch ubatch; // note: intentionally make a copy
|
||||
@@ -820,10 +861,6 @@ struct llm_graph_params {
|
||||
const llama_cross * cross;
|
||||
const llama_moe_cache * moe_cache;
|
||||
|
||||
const llama_prec_policy * prec_policy = nullptr;
|
||||
|
||||
const llama_hadamard * hdmd = nullptr;
|
||||
|
||||
ggml_tensor * layer_inp_dev = nullptr; // own layer input device buffer (write)
|
||||
ggml_tensor * layer_inp_src = nullptr; // ctx_other's layer input device buffer (read)
|
||||
ggml_tensor * h_nextn_dev = nullptr; // own h_nextn device buffer (write)
|
||||
@@ -1056,6 +1093,8 @@ struct llm_graph_context {
|
||||
const llama_cparams & cparams;
|
||||
const llama_ubatch & ubatch;
|
||||
|
||||
const llama_prec_policy & prec_policy;
|
||||
|
||||
const int64_t n_embd;
|
||||
const int64_t n_layer;
|
||||
const int64_t n_layer_nextn;
|
||||
@@ -1096,8 +1135,6 @@ struct llm_graph_context {
|
||||
const llama_cross * cross;
|
||||
const llama_moe_cache * moe_cache;
|
||||
|
||||
const llama_prec_policy * prec_policy;
|
||||
|
||||
const llama_hadamard * hdmd;
|
||||
|
||||
ggml_tensor * layer_inp_dev;
|
||||
@@ -1406,8 +1443,7 @@ struct llm_graph_context {
|
||||
ggml_tensor * sinks, // [n_head_q]
|
||||
ggml_tensor * v_mla, // [n_embd_head_v_mla, n_embd_head_v, n_head_v]
|
||||
float kq_scale,
|
||||
int il,
|
||||
ggml_tensor * kq_mask = nullptr) const;
|
||||
int il) const;
|
||||
|
||||
llm_graph_input_attn_k_iswa * build_attn_inp_k_iswa() const;
|
||||
|
||||
@@ -1500,6 +1536,25 @@ struct llm_graph_context {
|
||||
|
||||
llm_graph_input_mem_hybrid_iswa * build_inp_mem_hybrid_iswa() const;
|
||||
|
||||
// extent, extent_swa: the distance that the bias tables of the full and the SWA layers stop at
|
||||
llm_graph_input_mem_hybrid_iswa_rel * build_inp_mem_hybrid_iswa_rel(
|
||||
uint32_t extent,
|
||||
uint32_t extent_swa) const;
|
||||
|
||||
// kq_b is added to the KQ mask, after the KQ scale
|
||||
ggml_tensor * build_attn(
|
||||
llm_graph_input_mem_hybrid_iswa_rel * inp,
|
||||
ggml_tensor * wo,
|
||||
ggml_tensor * wo_b,
|
||||
ggml_tensor * wo_s,
|
||||
ggml_tensor * q_cur, // [n_embd_head_q, n_head_q, n_tokens]
|
||||
ggml_tensor * k_cur, // [n_embd_head_k, n_head_k, n_tokens]
|
||||
ggml_tensor * v_cur, // [n_embd_head_v, n_head_v, n_tokens]
|
||||
ggml_tensor * kq_b, // [n_kv, n_tokens/n_stream, n_head_q, n_stream]
|
||||
ggml_tensor * sinks, // [n_head_q]
|
||||
float kq_scale,
|
||||
int il) const;
|
||||
|
||||
//
|
||||
// pooling
|
||||
//
|
||||
|
||||
+290
-196
@@ -2,111 +2,157 @@
|
||||
#include "../llama-memory-hybrid-iswa.h"
|
||||
|
||||
void llama_model_inkling2::load_arch_hparams(llama_model_loader & ml) {
|
||||
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
|
||||
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
|
||||
ml.get_key(LLM_KV_ATTENTION_QK_CLIP, hparams.f_qk_clip);
|
||||
ml.get_key(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size);
|
||||
ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);
|
||||
|
||||
// relative-position bias, and the log-scaled positions of the full-attention layers
|
||||
ml.get_key(LLM_KV_ATTENTION_RELATIVE_LENGTH, hparams.n_relative);
|
||||
ml.get_key(LLM_KV_ATTENTION_RELATIVE_EXTENT, hparams.n_relative_extent);
|
||||
ml.get_key(LLM_KV_ATTENTION_QK_CLIP, hparams.f_qk_clip);
|
||||
ml.get_key(LLM_KV_ATTENTION_RELATIVE_CLIP, hparams.f_relative_clip);
|
||||
ml.get_key(LLM_KV_ATTENTION_RELATIVE_CLIP, hparams.f_relative_clip);
|
||||
ml.get_key(LLM_KV_ATTENTION_LOG_SCALE_FLOOR, hparams.n_log_scale_floor);
|
||||
ml.get_key(LLM_KV_ATTENTION_LOG_SCALE_ALPHA, hparams.f_log_scale_alpha);
|
||||
ml.get_key(LLM_KV_ATTN_RES_BLOCK_SIZE, hparams.attn_res_block_size);
|
||||
ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);
|
||||
ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab_padded);
|
||||
|
||||
// the embedding and output tensors are padded past the vocabulary
|
||||
ml.get_key(LLM_KV_VOCAB_SIZE, hparams.n_vocab_padded);
|
||||
ml.get_key(LLM_KV_VOCAB_SIZE_UNPADDED, hparams.n_vocab_unpadded);
|
||||
ml.get_key(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta);
|
||||
|
||||
// MoE
|
||||
ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);
|
||||
|
||||
ml.get_key(LLM_KV_ACTIVATION_SITU_BETA, hparams.situ_beta);
|
||||
ml.get_key(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, hparams.situ_linear_beta);
|
||||
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);
|
||||
ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping);
|
||||
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
|
||||
ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);
|
||||
|
||||
hparams.f_final_logit_softcapping = 0.0f;
|
||||
ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false);
|
||||
|
||||
// sliding window
|
||||
ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
|
||||
ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
|
||||
|
||||
uint32_t n_pattern = 0;
|
||||
ml.get_arr_n(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, n_pattern);
|
||||
|
||||
if (n_pattern != hparams.n_layer() || !hparams.n_relative || !hparams.n_relative_extent || !hparams.n_swa ||
|
||||
!hparams.attn_res_block_size || !hparams.n_log_scale_floor || hparams.n_shortconv_l_cache < 2 ||
|
||||
hparams.n_shortconv_l_cache > 64 || hparams.n_relative_extent > INT32_MAX || hparams.n_swa > INT32_MAX ||
|
||||
!hparams.n_vocab_unpadded || hparams.n_vocab_unpadded > hparams.n_vocab_padded) {
|
||||
throw std::runtime_error("inkling2: invalid dimensions, vocabulary, or layer pattern");
|
||||
}
|
||||
for (float value : {hparams.f_norm_rms_eps, hparams.f_qk_clip, hparams.f_relative_clip, hparams.situ_beta,
|
||||
hparams.situ_linear_beta, hparams.expert_weights_scale, hparams.f_final_logit_softcapping}) {
|
||||
|
||||
for (float value : {hparams.f_norm_rms_eps, hparams.f_qk_clip, hparams.f_relative_clip, hparams.situ_beta, hparams.situ_linear_beta}) {
|
||||
if (!std::isfinite(value) || value <= 0.0f) {
|
||||
throw std::runtime_error("inkling2: normalization, clipping and scaling parameters must be finite and positive");
|
||||
}
|
||||
}
|
||||
|
||||
if (!std::isfinite(hparams.f_log_scale_alpha) || hparams.f_log_scale_alpha < 0.0f) {
|
||||
throw std::runtime_error("inkling2: invalid logarithmic scale");
|
||||
}
|
||||
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
|
||||
hparams.expert_router_f32 = true;
|
||||
|
||||
hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
|
||||
hparams.expert_router_f32 = true;
|
||||
hparams.expert_weights_norm_eps = 1e-20f;
|
||||
|
||||
// no RoPE, the relative-position bias carries the positions
|
||||
hparams.rope_pattern.fill(false);
|
||||
uint64_t max_channels = 0;
|
||||
|
||||
// the convolution state keeps K, V, the attention output and the FFN output of the widest layer
|
||||
uint64_t n_channels_max = 0;
|
||||
|
||||
for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
|
||||
if (!hparams.n_head_kv(il) || hparams.n_head(il) % hparams.n_head_kv(il) ||
|
||||
hparams.n_embd_head_k(il) != hparams.n_embd_head_v(il) || !hparams.n_expert_used(il) ||
|
||||
hparams.n_expert_used(il) > hparams.n_expert || !hparams.n_ff_exp(il)) {
|
||||
throw std::runtime_error("inkling2: invalid attention or expert dimensions");
|
||||
}
|
||||
max_channels = std::max<uint64_t>(max_channels, 2ull * hparams.n_embd_k_gqa(il) + 2ull * hparams.n_embd);
|
||||
|
||||
n_channels_max = std::max<uint64_t>(n_channels_max, 2ull * hparams.n_embd_k_gqa(il) + 2ull * hparams.n_embd);
|
||||
}
|
||||
if (max_channels * (hparams.n_shortconv_l_cache - 1) > UINT32_MAX) {
|
||||
|
||||
if (n_channels_max * (hparams.n_shortconv_l_cache - 1) > UINT32_MAX) {
|
||||
throw std::runtime_error("inkling2: convolution state is too large");
|
||||
}
|
||||
hparams.n_sconv_channels = max_channels;
|
||||
|
||||
hparams.n_sconv_channels = n_channels_max;
|
||||
|
||||
type = LLM_TYPE_UNKNOWN;
|
||||
}
|
||||
|
||||
void llama_model_inkling2::load_arch_tensors(llama_model_loader &) {
|
||||
LLAMA_LOAD_LOCALS;
|
||||
|
||||
if (hparams.n_vocab_unpadded != vocab.n_tokens()) {
|
||||
throw std::runtime_error("inkling2: tokenizer size must equal the unpadded vocabulary size");
|
||||
}
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, hparams.n_vocab_padded}, 0);
|
||||
|
||||
const int64_t n_vocab_padded = hparams.n_vocab_padded;
|
||||
const int64_t n_rel = hparams.n_relative;
|
||||
const int64_t n_conv = hparams.n_shortconv_l_cache;
|
||||
|
||||
tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab_padded}, 0);
|
||||
tok_norm = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {n_embd}, 0);
|
||||
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, hparams.n_vocab_padded}, 0);
|
||||
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
|
||||
|
||||
// output
|
||||
output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab_padded}, 0);
|
||||
output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
|
||||
output_res_query = create_tensor(tn(LLM_TENSOR_OUTPUT_RES_QUERY, "weight"), {n_embd}, 0);
|
||||
|
||||
for (int i = 0; i < n_layer; ++i) {
|
||||
auto & l = layers[i];
|
||||
const int64_t nh = hparams.n_head(i);
|
||||
const int64_t nk = hparams.n_head_kv(i);
|
||||
const int64_t hd = hparams.n_embd_head_k(i);
|
||||
const int64_t nr = hparams.n_relative;
|
||||
const int64_t kernel = hparams.n_shortconv_l_cache;
|
||||
const int64_t extent = hparams.is_swa(i) ? hparams.n_swa : hparams.n_relative_extent;
|
||||
l.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, nh*hd}, 0);
|
||||
l.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, nk*hd}, 0);
|
||||
l.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, nk*hd}, 0);
|
||||
l.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {nh*hd, n_embd}, 0);
|
||||
l.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, nh*hd}, 0);
|
||||
l.attn_rel = create_tensor(tn(LLM_TENSOR_ATTN_REL, "weight", i), {n_embd, nh*nr}, 0);
|
||||
l.attn_rel_bias = create_tensor(tn(LLM_TENSOR_ATTN_REL_BIAS, "weight", i), {nr, extent}, 0);
|
||||
l.attn_q_clip = create_tensor(tn(LLM_TENSOR_ATTN_Q_CLIP, "weight", i), {hd, nh}, 0);
|
||||
l.attn_rel_clip = create_tensor(tn(LLM_TENSOR_ATTN_REL_CLIP, "weight", i), {nr, nh}, 0);
|
||||
l.attn_sink_proj = create_tensor(tn(LLM_TENSOR_ATTN_SINK_PROJ, "weight", i), {n_embd, nh}, 0);
|
||||
l.attn_k_conv = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV, "weight", i), {kernel, nk*hd}, 0);
|
||||
l.attn_v_conv = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV, "weight", i), {kernel, nk*hd}, 0);
|
||||
l.attn_post_conv = create_tensor(tn(LLM_TENSOR_ATTN_POST_CONV, "weight", i), {kernel, n_embd}, 0);
|
||||
l.ffn_post_conv = create_tensor(tn(LLM_TENSOR_FFN_POST_CONV, "weight", i), {kernel, n_embd}, 0);
|
||||
auto & layer = layers[i];
|
||||
|
||||
const int64_t n_head = hparams.n_head(i);
|
||||
const int64_t n_head_kv = hparams.n_head_kv(i);
|
||||
const int64_t n_embd_head = hparams.n_embd_head_k(i);
|
||||
const int64_t n_ff_exp = hparams.n_ff_exp(i);
|
||||
const int64_t n_extent = hparams.is_swa(i) ? hparams.n_swa : hparams.n_relative_extent;
|
||||
|
||||
// attention
|
||||
layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_head*n_head}, 0);
|
||||
layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_head*n_head_kv}, 0);
|
||||
layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_head*n_head_kv}, 0);
|
||||
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head*n_head, n_embd}, 0);
|
||||
layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_embd_head*n_head}, 0);
|
||||
|
||||
layer.attn_q_clip = create_tensor(tn(LLM_TENSOR_ATTN_Q_CLIP, "weight", i), {n_embd_head, n_head}, 0);
|
||||
layer.attn_sink_proj = create_tensor(tn(LLM_TENSOR_ATTN_SINK_PROJ, "weight", i), {n_embd, n_head}, 0);
|
||||
|
||||
// relative-position bias
|
||||
layer.attn_rel = create_tensor(tn(LLM_TENSOR_ATTN_REL, "weight", i), {n_embd, n_rel*n_head}, 0);
|
||||
layer.attn_rel_bias = create_tensor(tn(LLM_TENSOR_ATTN_REL_BIAS, "weight", i), {n_rel, n_extent}, 0);
|
||||
layer.attn_rel_clip = create_tensor(tn(LLM_TENSOR_ATTN_REL_CLIP, "weight", i), {n_rel, n_head}, 0);
|
||||
|
||||
// short convolutions
|
||||
layer.attn_k_conv = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV, "weight", i), {n_conv, n_embd_head*n_head_kv}, 0);
|
||||
layer.attn_v_conv = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV, "weight", i), {n_conv, n_embd_head*n_head_kv}, 0);
|
||||
layer.attn_post_conv = create_tensor(tn(LLM_TENSOR_ATTN_POST_CONV, "weight", i), {n_conv, n_embd}, 0);
|
||||
layer.ffn_post_conv = create_tensor(tn(LLM_TENSOR_FFN_POST_CONV, "weight", i), {n_conv, n_embd}, 0);
|
||||
|
||||
if (hparams.is_swa(i)) {
|
||||
l.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
|
||||
layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);
|
||||
} else {
|
||||
l.attn_q_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_INP_NORM, "weight", i), {n_embd}, 0);
|
||||
l.attn_k_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_INP_NORM, "weight", i), {n_embd}, 0);
|
||||
l.attn_v_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_V_INP_NORM, "weight", i), {n_embd}, 0);
|
||||
l.attn_k_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV_PROJ, "weight", i), {n_embd, kernel*nk}, 0);
|
||||
l.attn_v_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV_PROJ, "weight", i), {n_embd, kernel*nk}, 0);
|
||||
// full-attention layers: separate Q, K and V input norms, and input-dependent taps for the K and V convolutions
|
||||
layer.attn_q_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_INP_NORM, "weight", i), {n_embd}, 0);
|
||||
layer.attn_k_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_INP_NORM, "weight", i), {n_embd}, 0);
|
||||
layer.attn_v_inp_norm = create_tensor(tn(LLM_TENSOR_ATTN_V_INP_NORM, "weight", i), {n_embd}, 0);
|
||||
layer.attn_k_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_K_CONV_PROJ, "weight", i), {n_embd, n_conv*n_head_kv}, 0);
|
||||
layer.attn_v_conv_proj = create_tensor(tn(LLM_TENSOR_ATTN_V_CONV_PROJ, "weight", i), {n_embd, n_conv*n_head_kv}, 0);
|
||||
}
|
||||
|
||||
// block attention residuals
|
||||
if (i > 0) {
|
||||
l.attn_res_query = create_tensor(tn(LLM_TENSOR_ATTN_RES_QUERY, "weight", i), {n_embd, hparams.is_swa(i) ? 1 : 3}, 0);
|
||||
layer.attn_res_query = create_tensor(tn(LLM_TENSOR_ATTN_RES_QUERY, "weight", i), {n_embd, hparams.is_swa(i) ? 1 : 3}, 0);
|
||||
}
|
||||
l.ffn_res_query = create_tensor(tn(LLM_TENSOR_FFN_RES_QUERY, "weight", i), {n_embd}, 0);
|
||||
l.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
|
||||
l.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
|
||||
l.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
|
||||
l.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, 2*hparams.n_ff_exp(i), n_expert}, 0);
|
||||
l.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp(i), n_embd, n_expert}, 0);
|
||||
layer.ffn_res_query = create_tensor(tn(LLM_TENSOR_FFN_RES_QUERY, "weight", i), {n_embd}, 0);
|
||||
|
||||
// MoE
|
||||
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
|
||||
layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);
|
||||
layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED);
|
||||
layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", i), {n_embd, 2*n_ff_exp, n_expert}, 0);
|
||||
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,202 +160,250 @@ std::unique_ptr<llm_graph_context> llama_model_inkling2::build_arch_graph(const
|
||||
return std::make_unique<graph>(params);
|
||||
}
|
||||
|
||||
class llm_graph_input_inkling2_pos : public llm_graph_input_i {
|
||||
public:
|
||||
const llama_kv_cache_context * cache[2];
|
||||
ggml_tensor * distance[2];
|
||||
uint32_t extent[2];
|
||||
|
||||
bool can_reuse(const llm_graph_params & params) override {
|
||||
const auto * memory = static_cast<const llama_memory_hybrid_iswa_context *>(params.mctx);
|
||||
cache[0] = memory->get_attn()->get_base();
|
||||
cache[1] = memory->get_attn()->get_swa();
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
if (distance[i]->ne[0] != cache[i]->get_n_kv() || distance[i]->ne[1] != params.ubatch.n_tokens) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void set_input(const llama_ubatch * ubatch) override {
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
if (distance[i]->buffer) {
|
||||
cache[i]->set_input_relative_pos(distance[i], ubatch, extent[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// scale each row down to an L2 norm of at most threshold, after the optional per-channel scale
|
||||
ggml_tensor * llama_model_inkling2::graph::clip(ggml_tensor * cur, ggml_tensor * scale, float threshold) {
|
||||
if (scale) {
|
||||
cur = ggml_mul(ctx0, cur, scale);
|
||||
}
|
||||
auto * norm = ggml_sqrt(ctx0, ggml_sum_rows(ctx0, ggml_sqr(ctx0, cur)));
|
||||
|
||||
ggml_tensor * norm = ggml_sqrt(ctx0, ggml_sum_rows(ctx0, ggml_sqr(ctx0, cur)));
|
||||
norm = ggml_scale_bias(ctx0, norm, 1.0f, 1e-6f);
|
||||
auto * factor = ggml_div(ctx0, ggml_fill(ctx0, norm, threshold), norm);
|
||||
|
||||
ggml_tensor * factor = ggml_div(ctx0, ggml_fill(ctx0, norm, threshold), norm);
|
||||
|
||||
return ggml_mul(ctx0, cur, ggml_clamp(ctx0, factor, 0.0f, 1.0f));
|
||||
}
|
||||
|
||||
// attention over the depth: mix the earlier block outputs and cur with learned queries
|
||||
ggml_tensor * llama_model_inkling2::graph::mix(const std::vector<ggml_tensor *> & blocks, ggml_tensor * cur, ggml_tensor * query) {
|
||||
auto * states = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
|
||||
for (auto * block : blocks) {
|
||||
ggml_tensor * states = ggml_reshape_3d(ctx0, cur, n_embd, 1, n_tokens);
|
||||
for (ggml_tensor * block : blocks) {
|
||||
states = ggml_concat(ctx0, ggml_reshape_3d(ctx0, block, n_embd, 1, n_tokens), states, 1);
|
||||
}
|
||||
states = ggml_rms_norm(ctx0, states, 1e-6f);
|
||||
auto * queries = ggml_repeat(ctx0, query, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd, query->ne[1], n_tokens));
|
||||
auto * scores = ggml_mul_mat(ctx0, states, queries);
|
||||
|
||||
ggml_tensor * queries = ggml_repeat(ctx0, query, ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_embd, query->ne[1], n_tokens));
|
||||
|
||||
ggml_tensor * scores = ggml_mul_mat(ctx0, states, queries);
|
||||
scores = ggml_soft_max_ext(ctx0, scores, nullptr, 1.0f/sqrtf(n_embd), 0.0f);
|
||||
|
||||
return ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, states)), scores);
|
||||
}
|
||||
|
||||
ggml_tensor * llama_model_inkling2::graph::convolve(ggml_tensor * cur, ggml_tensor * weight, ggml_tensor * dynamic,
|
||||
ggml_tensor * state, ggml_tensor * history, llm_graph_input_rs * inp, int64_t offset) {
|
||||
const int64_t kernel = hparams.n_shortconv_l_cache;
|
||||
const int64_t channels = cur->ne[0];
|
||||
const int64_t nt = ubatch.n_seq_tokens;
|
||||
const int64_t ns = ubatch.n_seqs;
|
||||
auto * past = ggml_view_3d(ctx0, history, kernel - 1, channels, ns, history->nb[1], history->nb[2], offset*history->nb[1]);
|
||||
auto * x = ggml_transpose(ctx0, ggml_reshape_3d(ctx0, cur, channels, nt, ns));
|
||||
// causal short convolution with a residual; dynamic adds input-dependent taps per head
|
||||
// the last n_conv - 1 inputs of each sequence live in the recurrent state at channel offset `offset`
|
||||
ggml_tensor * llama_model_inkling2::graph::convolve(
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * weight,
|
||||
ggml_tensor * dynamic,
|
||||
ggml_tensor * state,
|
||||
ggml_tensor * history,
|
||||
llm_graph_input_rs * inp,
|
||||
int64_t offset) {
|
||||
const int64_t n_conv = hparams.n_shortconv_l_cache;
|
||||
const int64_t n_channels = cur->ne[0];
|
||||
const int64_t n_seq_tokens = ubatch.n_seq_tokens;
|
||||
const int64_t n_seqs = ubatch.n_seqs;
|
||||
|
||||
ggml_tensor * past = ggml_view_3d(ctx0, history, n_conv - 1, n_channels, n_seqs, history->nb[1], history->nb[2], offset*history->nb[1]);
|
||||
|
||||
ggml_tensor * x = ggml_transpose(ctx0, ggml_reshape_3d(ctx0, cur, n_channels, n_seq_tokens, n_seqs));
|
||||
x = ggml_concat(ctx0, past, x, 0);
|
||||
build_conv_state_store(inp, state, x, kernel - 1, offset*(kernel - 1)*ggml_element_size(state));
|
||||
auto * out = ggml_ssm_conv(ctx0, x, weight);
|
||||
out = ggml_reshape_2d(ctx0, out, channels, n_tokens);
|
||||
|
||||
build_conv_state_store(inp, state, x, n_conv - 1, offset*(n_conv - 1)*ggml_element_size(state));
|
||||
|
||||
ggml_tensor * out = ggml_ssm_conv(ctx0, x, weight);
|
||||
out = ggml_reshape_2d(ctx0, out, n_channels, n_tokens);
|
||||
|
||||
if (dynamic) {
|
||||
const int64_t heads = dynamic->ne[0]/kernel;
|
||||
dynamic = ggml_reshape_3d(ctx0, dynamic, kernel, heads, n_tokens);
|
||||
for (int64_t lag = 0; lag < kernel; ++lag) {
|
||||
auto * tap = ggml_view_3d(ctx0, dynamic, 1, heads, n_tokens, dynamic->nb[1], dynamic->nb[2], lag*dynamic->nb[0]);
|
||||
auto * window = ggml_view_3d(ctx0, x, nt, channels, ns, x->nb[1], x->nb[2], (kernel - 1 - lag)*x->nb[0]);
|
||||
const int64_t n_heads = dynamic->ne[0]/n_conv;
|
||||
|
||||
dynamic = ggml_reshape_3d(ctx0, dynamic, n_conv, n_heads, n_tokens);
|
||||
|
||||
for (int64_t lag = 0; lag < n_conv; ++lag) {
|
||||
ggml_tensor * tap = ggml_view_3d(ctx0, dynamic, 1, n_heads, n_tokens, dynamic->nb[1], dynamic->nb[2], lag*dynamic->nb[0]);
|
||||
ggml_tensor * window = ggml_view_3d(ctx0, x, n_seq_tokens, n_channels, n_seqs, x->nb[1], x->nb[2], (n_conv - 1 - lag)*x->nb[0]);
|
||||
|
||||
window = ggml_cont(ctx0, ggml_transpose(ctx0, window));
|
||||
window = ggml_reshape_3d(ctx0, window, channels/heads, heads, n_tokens);
|
||||
auto * term = ggml_reshape_2d(ctx0, ggml_mul(ctx0, window, tap), channels, n_tokens);
|
||||
window = ggml_reshape_3d(ctx0, window, n_channels/n_heads, n_heads, n_tokens);
|
||||
|
||||
ggml_tensor * term = ggml_reshape_2d(ctx0, ggml_mul(ctx0, window, tap), n_channels, n_tokens);
|
||||
|
||||
out = ggml_add(ctx0, out, term);
|
||||
}
|
||||
}
|
||||
return ggml_scale(ctx0, ggml_add(ctx0, cur, out), 1.0f/sqrtf((dynamic ? 2 : 1)*kernel + 1));
|
||||
|
||||
return ggml_scale(ctx0, ggml_add(ctx0, cur, out), 1.0f/sqrtf((dynamic ? 2 : 1)*n_conv + 1));
|
||||
}
|
||||
|
||||
llama_model_inkling2::graph::graph(const llm_graph_params & params) : llm_graph_context(params) {
|
||||
auto * inp = build_inp_mem_hybrid_iswa();
|
||||
auto * inp = build_inp_mem_hybrid_iswa_rel(hparams.n_relative_extent, hparams.n_swa);
|
||||
auto * attn = inp->get_attn();
|
||||
auto * recr = inp->get_recr();
|
||||
auto * positions = build_inp_pos();
|
||||
auto * out_ids = build_inp_out_ids();
|
||||
auto rel = std::make_unique<llm_graph_input_inkling2_pos>();
|
||||
rel->cache[0] = attn->mctx->get_base();
|
||||
rel->cache[1] = attn->mctx->get_swa();
|
||||
rel->extent[0] = hparams.n_relative_extent;
|
||||
rel->extent[1] = hparams.n_swa;
|
||||
ggml_tensor * distances[2];
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
distances[i] = rel->distance[i] = ggml_new_tensor_2d(ctx0, GGML_TYPE_I32, rel->cache[i]->get_n_kv(), n_tokens);
|
||||
ggml_set_input(distances[i]);
|
||||
}
|
||||
res->add_input(std::move(rel));
|
||||
auto * tau = ggml_scale_bias(ctx0, ggml_cast(ctx0, positions, GGML_TYPE_F32), 1.0f, 1.0f);
|
||||
|
||||
ggml_tensor * inp_pos = build_inp_pos();
|
||||
ggml_tensor * inp_out_ids = build_inp_out_ids();
|
||||
|
||||
// log-scaled positions: 1 + alpha*log(max(1, (pos + 1)/floor)), multiplies q and r of the full-attention layers
|
||||
ggml_tensor * tau = ggml_scale_bias(ctx0, ggml_cast(ctx0, inp_pos, GGML_TYPE_F32), 1.0f, 1.0f);
|
||||
tau = ggml_log(ctx0, ggml_clamp(ctx0, ggml_scale(ctx0, tau, 1.0f/hparams.n_log_scale_floor), 1.0f, INFINITY));
|
||||
tau = ggml_reshape_3d(ctx0, ggml_scale_bias(ctx0, tau, hparams.f_log_scale_alpha, 1.0f), 1, 1, n_tokens);
|
||||
auto * cur = build_inp_embd(model.tok_embd);
|
||||
|
||||
ggml_tensor * cur = build_inp_embd(model.tok_embd);
|
||||
|
||||
// important: do not normalize raw embeddings input (i.e. encoded image embeddings)
|
||||
if (ubatch.token) {
|
||||
cur = build_norm(cur, model.tok_norm, nullptr, LLM_NORM_RMS, -1);
|
||||
}
|
||||
|
||||
std::vector<ggml_tensor *> blocks;
|
||||
|
||||
for (int il = 0; il < n_layer; ++il) {
|
||||
const auto & l = model.layers[il];
|
||||
const bool swa = hparams.is_swa(il);
|
||||
const int64_t nh = hparams.n_head(il);
|
||||
const int64_t nk = hparams.n_head_kv(il);
|
||||
const int64_t hd = hparams.n_embd_head_k(il);
|
||||
const int64_t kv = nk*hd;
|
||||
auto * state = recr->mctx->get_r_l(il);
|
||||
auto * history = build_rs(recr, state, hparams.n_embd_r(), ubatch.n_seqs);
|
||||
const auto & layer = model.layers[il];
|
||||
|
||||
const bool is_swa = hparams.is_swa(il);
|
||||
const int64_t n_head = hparams.n_head(il);
|
||||
const int64_t n_head_kv = hparams.n_head_kv(il);
|
||||
const int64_t n_embd_head = hparams.n_embd_head_k(il);
|
||||
const int64_t n_embd_k_gqa = n_embd_head*n_head_kv;
|
||||
|
||||
ggml_tensor * state = recr->mctx->get_r_l(il);
|
||||
ggml_tensor * history = build_rs(recr, state, hparams.n_embd_r(), ubatch.n_seqs);
|
||||
history = ggml_reshape_3d(ctx0, history, hparams.n_shortconv_l_cache - 1, hparams.n_sconv_channels, ubatch.n_seqs);
|
||||
auto * residual = cur;
|
||||
|
||||
ggml_tensor * inpSA = cur;
|
||||
|
||||
// attention inputs: SWA layers mix one, full-attention layers one each for Q, K and V
|
||||
ggml_tensor * inputs[3] = {cur, cur, cur};
|
||||
if (l.attn_res_query) {
|
||||
auto * mixed = mix(blocks, cur, l.attn_res_query);
|
||||
if (layer.attn_res_query) {
|
||||
ggml_tensor * mixed = mix(blocks, cur, layer.attn_res_query);
|
||||
for (int j = 0; j < 3; ++j) {
|
||||
inputs[j] = ggml_cont(ctx0, ggml_view_2d(ctx0, mixed, n_embd, n_tokens, mixed->nb[2], (swa ? 0 : j)*mixed->nb[1]));
|
||||
inputs[j] = ggml_cont(ctx0, ggml_view_2d(ctx0, mixed, n_embd, n_tokens, mixed->nb[2], (is_swa ? 0 : j)*mixed->nb[1]));
|
||||
}
|
||||
}
|
||||
if (swa) {
|
||||
inputs[0] = build_norm(inputs[0], l.attn_norm, nullptr, LLM_NORM_RMS, il);
|
||||
|
||||
if (is_swa) {
|
||||
inputs[0] = build_norm(inputs[0], layer.attn_norm, nullptr, LLM_NORM_RMS, il);
|
||||
inputs[1] = inputs[2] = inputs[0];
|
||||
} else {
|
||||
inputs[0] = build_norm(inputs[0], l.attn_q_inp_norm, nullptr, LLM_NORM_RMS, il);
|
||||
inputs[1] = build_norm(inputs[1], l.attn_k_inp_norm, nullptr, LLM_NORM_RMS, il);
|
||||
inputs[2] = build_norm(inputs[2], l.attn_v_inp_norm, nullptr, LLM_NORM_RMS, il);
|
||||
inputs[0] = build_norm(inputs[0], layer.attn_q_inp_norm, nullptr, LLM_NORM_RMS, il);
|
||||
inputs[1] = build_norm(inputs[1], layer.attn_k_inp_norm, nullptr, LLM_NORM_RMS, il);
|
||||
inputs[2] = build_norm(inputs[2], layer.attn_v_inp_norm, nullptr, LLM_NORM_RMS, il);
|
||||
}
|
||||
auto * q = build_lora_mm(l.wq, inputs[0]);
|
||||
auto * k = convolve(build_lora_mm(l.wk, inputs[1]), l.attn_k_conv,
|
||||
swa ? nullptr : build_lora_mm(l.attn_k_conv_proj, inputs[1]), state, history, recr, 0);
|
||||
auto * v = convolve(build_lora_mm(l.wv, inputs[2]), l.attn_v_conv,
|
||||
swa ? nullptr : build_lora_mm(l.attn_v_conv_proj, inputs[2]), state, history, recr, kv);
|
||||
auto * r = build_lora_mm(l.attn_rel, inputs[0]);
|
||||
q = clip(ggml_reshape_3d(ctx0, q, hd, nh, n_tokens), l.attn_q_clip, hparams.f_qk_clip);
|
||||
k = clip(ggml_reshape_3d(ctx0, k, hd, nk, n_tokens), nullptr, hparams.f_qk_clip);
|
||||
v = ggml_rms_norm(ctx0, ggml_reshape_3d(ctx0, v, hd, nk, n_tokens), hparams.f_norm_rms_eps);
|
||||
r = clip(ggml_reshape_3d(ctx0, r, hparams.n_relative, nh, n_tokens), l.attn_rel_clip, hparams.f_relative_clip);
|
||||
if (!swa) {
|
||||
q = ggml_mul(ctx0, q, tau);
|
||||
r = ggml_mul(ctx0, r, tau);
|
||||
|
||||
// self-attention
|
||||
{
|
||||
ggml_tensor * Qcur = build_lora_mm(layer.wq, inputs[0]);
|
||||
ggml_tensor * Kcur = convolve(build_lora_mm(layer.wk, inputs[1]), layer.attn_k_conv,
|
||||
is_swa ? nullptr : build_lora_mm(layer.attn_k_conv_proj, inputs[1]), state, history, recr, 0);
|
||||
ggml_tensor * Vcur = convolve(build_lora_mm(layer.wv, inputs[2]), layer.attn_v_conv,
|
||||
is_swa ? nullptr : build_lora_mm(layer.attn_v_conv_proj, inputs[2]), state, history, recr, n_embd_k_gqa);
|
||||
ggml_tensor * Rcur = build_lora_mm(layer.attn_rel, inputs[0]);
|
||||
|
||||
Qcur = clip(ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), layer.attn_q_clip, hparams.f_qk_clip);
|
||||
Kcur = clip(ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), nullptr, hparams.f_qk_clip);
|
||||
Vcur = ggml_rms_norm(ctx0, ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens), hparams.f_norm_rms_eps);
|
||||
Rcur = clip(ggml_reshape_3d(ctx0, Rcur, hparams.n_relative, n_head, n_tokens), layer.attn_rel_clip, hparams.f_relative_clip);
|
||||
|
||||
if (!is_swa) {
|
||||
Qcur = ggml_mul(ctx0, Qcur, tau);
|
||||
Rcur = ggml_mul(ctx0, Rcur, tau);
|
||||
}
|
||||
|
||||
cb(Qcur, "Qcur", il);
|
||||
cb(Kcur, "Kcur", il);
|
||||
cb(Vcur, "Vcur", il);
|
||||
cb(Rcur, "Rcur", il);
|
||||
|
||||
ggml_tensor * rel_pos = is_swa ? inp->self_rel_pos_swa : inp->self_rel_pos;
|
||||
|
||||
const int64_t n_stream = (is_swa ? attn->self_attn_mask_swa.raw : attn->self_attn_mask.raw)->ne[3];
|
||||
const int64_t n_kv = rel_pos->ne[0];
|
||||
|
||||
ggml_tensor * sinks = build_lora_mm(layer.attn_sink_proj, inputs[0]);
|
||||
|
||||
// project r on the bias table first, then gather by distance: only the gathered bias is n_kv x n_tokens x n_head
|
||||
ggml_tensor * table = ggml_mul_mat(ctx0, ggml_pad(ctx0, layer.attn_rel_bias, 0, 1, 0, 0), Rcur);
|
||||
|
||||
// the sink logit t is per query: exp(s - t)/(1 + sum(exp(s - t))) = exp(s)/(exp(t) + sum(exp(s))), with a zero sink
|
||||
table = ggml_sub(ctx0, table, ggml_reshape_3d(ctx0, sinks, 1, n_head, n_tokens));
|
||||
table = ggml_cont(ctx0, ggml_permute(ctx0, table, 1, 0, 2, 3));
|
||||
|
||||
ggml_tensor * kq_b = ggml_get_rows(ctx0, table, rel_pos);
|
||||
kq_b = ggml_reshape_4d(ctx0, kq_b, n_head, n_kv, n_tokens/n_stream, n_stream);
|
||||
kq_b = ggml_cont(ctx0, ggml_permute(ctx0, kq_b, 2, 0, 1, 3));
|
||||
cb(kq_b, "kq_b", il);
|
||||
|
||||
sinks = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_head), 0.0f);
|
||||
|
||||
cur = build_attn(inp,
|
||||
nullptr, nullptr, nullptr,
|
||||
Qcur, Kcur, Vcur, kq_b, sinks, 1.0f/sqrtf(n_embd_head), il);
|
||||
|
||||
cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, build_lora_mm(layer.wqkv_gate, inputs[0])));
|
||||
cur = convolve(build_lora_mm(layer.wo, cur), layer.attn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa);
|
||||
cb(cur, "attn_out", il);
|
||||
}
|
||||
auto * mask = swa ? attn->self_attn_mask_swa.raw : attn->self_attn_mask.raw;
|
||||
const int64_t streams = mask->ne[3];
|
||||
const int64_t n_kv = distances[swa]->ne[0];
|
||||
auto * sinks = build_lora_mm(l.attn_sink_proj, inputs[0]);
|
||||
// project r on the bias table first, then gather by distance: only the gathered bias is n_kv x n_tokens x nh
|
||||
auto * table = ggml_mul_mat(ctx0, ggml_pad(ctx0, l.attn_rel_bias, 0, 1, 0, 0), r);
|
||||
if (cparams.flash_attn) {
|
||||
// exp(s - t)/(1 + sum(exp(s - t))) = exp(s)/(exp(t) + sum(exp(s))).
|
||||
table = ggml_sub(ctx0, table, ggml_reshape_3d(ctx0, sinks, 1, nh, n_tokens));
|
||||
} else {
|
||||
// build_attn scales the bias along with QK; FA adds the mask after scaling QK.
|
||||
table = ggml_scale(ctx0, table, sqrtf(hd));
|
||||
}
|
||||
table = ggml_cont(ctx0, ggml_permute(ctx0, table, 1, 0, 2, 3));
|
||||
auto * bias = ggml_get_rows(ctx0, table, distances[swa]);
|
||||
bias = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_4d(ctx0, bias, nh, n_kv, n_tokens/streams, streams), 2, 0, 1, 3));
|
||||
ggml_tensor * fa_mask = nullptr;
|
||||
if (cparams.flash_attn) {
|
||||
fa_mask = ggml_cast(ctx0, ggml_add(ctx0, bias, ggml_cast(ctx0, mask, GGML_TYPE_F32)), GGML_TYPE_F16);
|
||||
sinks = ggml_fill(ctx0, ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, nh), 0.0f);
|
||||
bias = nullptr;
|
||||
} else {
|
||||
sinks = ggml_reshape_4d(ctx0, ggml_cont(ctx0, ggml_permute(ctx0,
|
||||
ggml_reshape_3d(ctx0, sinks, nh, n_tokens/streams, streams), 1, 0, 2, 3)), 1, n_tokens/streams, nh, streams);
|
||||
}
|
||||
cur = build_attn(attn, nullptr, nullptr, nullptr, q, k, v, bias, sinks, nullptr, 1.0f/sqrtf(hd), il, fa_mask);
|
||||
cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, build_lora_mm(l.wqkv_gate, inputs[0])));
|
||||
cur = convolve(build_lora_mm(l.wo, cur), l.attn_post_conv, nullptr, state, history, recr, 2*kv);
|
||||
|
||||
// the first layer of each block starts a new residual stream, the earlier one joins the mixed blocks
|
||||
if (il % hparams.attn_res_block_size == 0) {
|
||||
blocks.push_back(residual);
|
||||
blocks.push_back(inpSA);
|
||||
} else {
|
||||
cur = ggml_add(ctx0, residual, cur);
|
||||
cur = ggml_add(ctx0, inpSA, cur);
|
||||
}
|
||||
residual = cur;
|
||||
cur = ggml_reshape_2d(ctx0, mix(blocks, cur, l.ffn_res_query), n_embd, n_tokens);
|
||||
cur = build_norm(cur, l.ffn_norm, nullptr, LLM_NORM_RMS, il);
|
||||
cur = build_moe_ffn(cur, l.ffn_gate_inp, nullptr, nullptr, l.ffn_down_exps, l.ffn_exp_probs_b,
|
||||
hparams.n_expert, hparams.n_expert_used(il), LLM_FFN_SITU, true, hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID, il, nullptr, l.ffn_gate_up_exps);
|
||||
cur = convolve(cur, l.ffn_post_conv, nullptr, state, history, recr, 2*kv + n_embd);
|
||||
cur = ggml_add(ctx0, residual, cur);
|
||||
|
||||
ggml_tensor * ffn_inp = cur;
|
||||
cb(ffn_inp, "ffn_inp", il);
|
||||
|
||||
// MoE
|
||||
cur = ggml_reshape_2d(ctx0, mix(blocks, cur, layer.ffn_res_query), n_embd, n_tokens);
|
||||
cur = build_norm(cur, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);
|
||||
|
||||
cur = build_moe_ffn(cur,
|
||||
layer.ffn_gate_inp,
|
||||
nullptr,
|
||||
nullptr,
|
||||
layer.ffn_down_exps,
|
||||
layer.ffn_exp_probs_b,
|
||||
hparams.n_expert,
|
||||
hparams.n_expert_used(il),
|
||||
LLM_FFN_SITU, true,
|
||||
hparams.expert_weights_scale,
|
||||
LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,
|
||||
il,
|
||||
nullptr,
|
||||
layer.ffn_gate_up_exps);
|
||||
|
||||
cur = convolve(cur, layer.ffn_post_conv, nullptr, state, history, recr, 2*n_embd_k_gqa + n_embd);
|
||||
cb(cur, "ffn_out", il);
|
||||
|
||||
cur = ggml_add(ctx0, ffn_inp, cur);
|
||||
cb(cur, "l_out", il);
|
||||
}
|
||||
|
||||
cur = ggml_reshape_2d(ctx0, mix(blocks, cur, model.output_res_query), n_embd, n_tokens);
|
||||
cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);
|
||||
if (out_ids) {
|
||||
cur = ggml_get_rows(ctx0, cur, out_ids);
|
||||
|
||||
if (inp_out_ids) {
|
||||
cur = ggml_get_rows(ctx0, cur, inp_out_ids);
|
||||
}
|
||||
|
||||
cb(cur, "result_norm", -1);
|
||||
res->t_embd = cur;
|
||||
|
||||
// lm_head
|
||||
cur = build_lora_mm(model.output, cur);
|
||||
ggml_prec_set_acc(cur, GGML_PREC_F32);
|
||||
const float cap = hparams.f_final_logit_softcapping;
|
||||
cur = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, cur, 1.0f/cap)), cap);
|
||||
|
||||
if (hparams.f_final_logit_softcapping != 0.0f) {
|
||||
const float cap = hparams.f_final_logit_softcapping;
|
||||
|
||||
cur = ggml_scale(ctx0, ggml_tanh(ctx0, ggml_scale(ctx0, cur, 1.0f/cap)), cap);
|
||||
}
|
||||
|
||||
// drop the padded vocabulary
|
||||
cur = ggml_cont(ctx0, ggml_view_2d(ctx0, cur, hparams.n_vocab_unpadded, cur->ne[1], cur->nb[1], 0));
|
||||
res->t_logits = cur;
|
||||
|
||||
cb(cur, "result_output", -1);
|
||||
res->t_logits = cur;
|
||||
}
|
||||
|
||||
+8
-1
@@ -2480,7 +2480,14 @@ struct llama_model_inkling2 : public llama_model_base {
|
||||
graph(const llm_graph_params & params);
|
||||
ggml_tensor * mix(const std::vector<ggml_tensor *> & blocks, ggml_tensor * cur, ggml_tensor * query);
|
||||
ggml_tensor * clip(ggml_tensor * cur, ggml_tensor * scale, float threshold);
|
||||
ggml_tensor * convolve(ggml_tensor * cur, ggml_tensor * weight, ggml_tensor * dynamic, ggml_tensor * state, ggml_tensor * history, llm_graph_input_rs * inp, int64_t offset);
|
||||
ggml_tensor * convolve(
|
||||
ggml_tensor * cur,
|
||||
ggml_tensor * weight,
|
||||
ggml_tensor * dynamic,
|
||||
ggml_tensor * state,
|
||||
ggml_tensor * history,
|
||||
llm_graph_input_rs * inp,
|
||||
int64_t offset);
|
||||
};
|
||||
|
||||
std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
|
||||
|
||||
@@ -367,6 +367,8 @@ endif()
|
||||
llama_build(test-backend-ops.cpp LINK ggml Threads::Threads)
|
||||
|
||||
llama_build_and_test(test-backend-tensor.cpp LINK ggml)
|
||||
llama_build_and_test(test-sleep.cpp LINK ggml)
|
||||
llama_build_and_test(test-backend-sched.cpp LINK ggml)
|
||||
|
||||
if (BUILD_SHARED_LIBS)
|
||||
llama_build_and_test(test-llama-mmap.cpp ${PROJECT_SOURCE_DIR}/src/llama-impl.cpp ${PROJECT_SOURCE_DIR}/src/llama-mmap.cpp LABEL behavior)
|
||||
|
||||
@@ -9657,6 +9657,33 @@ struct test_fill : public test_case {
|
||||
}
|
||||
};
|
||||
|
||||
// GGML_OP_SLEEP (for scheduler testing only)
|
||||
struct test_sleep : public test_case {
|
||||
const ggml_type type;
|
||||
const std::array<int64_t, 4> ne;
|
||||
const int32_t us;
|
||||
|
||||
std::string vars() override { return VARS_TO_STR3(type, ne, us); }
|
||||
|
||||
// the op only delays, the output aliases the input and must match it exactly
|
||||
double max_nmse_err() override { return 0.0; }
|
||||
|
||||
test_sleep(int32_t us = 100, ggml_type type = GGML_TYPE_F32,
|
||||
std::array<int64_t, 4> ne = { 10, 10, 4, 3 })
|
||||
: type(type), ne(ne), us(us) {}
|
||||
|
||||
ggml_tensor * build_graph(ggml_context * ctx) override {
|
||||
ggml_tensor * a = ggml_new_tensor_4d(ctx, type, ne[0], ne[1], ne[2], ne[3]);
|
||||
ggml_set_name(a, "a");
|
||||
|
||||
ggml_tensor * out = ggml_sleep(ctx, a, us);
|
||||
|
||||
ggml_set_name(out, "out");
|
||||
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
// GGML_OP_SOLVE_TRI
|
||||
struct test_solve_tri : public test_case {
|
||||
const ggml_type type;
|
||||
@@ -12996,6 +13023,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_fill(-152.0f, GGML_TYPE_F16, { 800, 600, 4, 4 }));
|
||||
test_cases.emplace_back(new test_fill(3.5f, GGML_TYPE_F16, { 2048, 512, 2, 2 }));
|
||||
|
||||
test_cases.emplace_back(new test_sleep(0));
|
||||
test_cases.emplace_back(new test_sleep(10, GGML_TYPE_F16, { 128, 4, 2, 2 }));
|
||||
test_cases.emplace_back(new test_sleep(10, GGML_TYPE_F32, { 128, 4, 2, 2 }));
|
||||
|
||||
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
|
||||
test_cases.emplace_back(new test_diag(type));
|
||||
test_cases.emplace_back(new test_diag(type, { 79, 1, 19, 13 }));
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -170,6 +170,40 @@ static bool test_tensor_n(ggml_backend_buffer_type_t buft) {
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
std::vector<ggml_tensor *> batch;
|
||||
ggml_backend_buffer_ptr scatter_buf = alloc_tensors(ctx.get(), buft, { 64 }, batch);
|
||||
if (!scatter_buf) {
|
||||
return false;
|
||||
}
|
||||
batch.resize(128, batch[0]);
|
||||
uint8_t input[32], expected[256], out[256];
|
||||
for (size_t i = 0; i < sizeof(input); i++) {
|
||||
input[i] = (uint8_t) (i + 1);
|
||||
}
|
||||
std::vector<size_t> offsets(batch.size()), sizes(batch.size(), 1), data_offsets(batch.size());
|
||||
for (bool overlap : { false, true }) {
|
||||
memset(expected, 0, sizeof(expected));
|
||||
ggml_backend_tensor_set(batch[0], expected, 0, sizeof(expected));
|
||||
for (size_t i = 0; i < batch.size(); i++) {
|
||||
offsets[i] = 2*((i*5) % batch.size()) + 1;
|
||||
data_offsets[i] = i % sizeof(input);
|
||||
}
|
||||
// Repeated sources are valid; overlapping destinations must retain copy order.
|
||||
if (overlap) {
|
||||
offsets.back() = offsets.front();
|
||||
}
|
||||
for (size_t i = 0; i < batch.size(); i++) {
|
||||
expected[offsets[i]] = input[data_offsets[i]];
|
||||
}
|
||||
ggml_backend_tensor_set_n(batch.data(), input, offsets.data(), sizes.data(), data_offsets.data(), batch.size());
|
||||
ggml_backend_tensor_get(batch[0], out, 0, sizeof(out));
|
||||
if (memcmp(out, expected, sizeof(out)) != 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
// Tests for GGML_OP_SLEEP, on every backend that supports it. Three properties are checked:
|
||||
// - the op only delays: input = output
|
||||
// - the op does not return early: a chain of n sleeps must take at least n*us
|
||||
// Only a lower bound is asserted for the duration. A busy-wait cannot finish early, so that bound is
|
||||
// deterministic, whereas an upper bound would be flaky on a loaded machine.
|
||||
// Known limitation: Vulkan does not specify the tick rate of the shader clock, see ggml_vk_sleep. A driver
|
||||
// whose clock runs slower than assumed could undershoot the request and fail this bound.
|
||||
|
||||
#include "ggml.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "ggml-cpp.h"
|
||||
|
||||
#include <cinttypes>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <random>
|
||||
#include <vector>
|
||||
|
||||
enum test_result {
|
||||
TEST_OK,
|
||||
TEST_FAIL,
|
||||
TEST_SKIP,
|
||||
};
|
||||
|
||||
static test_result test_sleep(ggml_backend_t backend, ggml_type type, int64_t ne, int32_t us, int n_nodes) {
|
||||
const size_t graph_size = n_nodes + 16;
|
||||
|
||||
ggml_init_params params_static = {
|
||||
/*.mem_size =*/ ggml_tensor_overhead(),
|
||||
/*.mem_buffer =*/ nullptr,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ggml_context_ptr ctx_static(ggml_init(params_static));
|
||||
|
||||
ggml_tensor * x = ggml_new_tensor_1d(ctx_static.get(), type, ne);
|
||||
ggml_set_name(x, "x");
|
||||
ggml_set_input(x);
|
||||
|
||||
ggml_init_params params_compute = {
|
||||
/*.mem_size =*/ n_nodes*ggml_tensor_overhead() + ggml_graph_overhead_custom(graph_size, false),
|
||||
/*.mem_buffer =*/ nullptr,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
ggml_context_ptr ctx_compute(ggml_init(params_compute));
|
||||
ggml_cgraph * gf = ggml_new_graph_custom(ctx_compute.get(), graph_size, false);
|
||||
|
||||
ggml_tensor * out = x;
|
||||
for (int i = 0; i < n_nodes; i++) {
|
||||
out = ggml_sleep(ctx_compute.get(), out, us);
|
||||
}
|
||||
ggml_set_name(out, "out");
|
||||
ggml_build_forward_expand(gf, out);
|
||||
|
||||
// the backends log to stdout while computing, so the result line is only printed once it is complete
|
||||
char desc[128];
|
||||
snprintf(desc, sizeof(desc), " %-6s %-4s ne=%-9" PRId64 " us=%-7d n_nodes=%d ",
|
||||
ggml_backend_name(backend), ggml_type_name(type), ne, us, n_nodes);
|
||||
|
||||
test_result result = TEST_OK;
|
||||
|
||||
if (!ggml_backend_supports_op(backend, out)) {
|
||||
printf("%snot supported\n", desc);
|
||||
result = TEST_SKIP;
|
||||
} else {
|
||||
ggml_backend_buffer_ptr buf_static(ggml_backend_alloc_ctx_tensors(ctx_static.get(), backend));
|
||||
|
||||
// random bytes rather than random floats, so that bit patterns a value-wise copy could
|
||||
// mangle, such as NaNs and denormals, are covered as well
|
||||
std::vector<uint8_t> data_in(ggml_nbytes(x));
|
||||
std::mt19937 rng(1234);
|
||||
for (size_t i = 0; i < data_in.size(); i++) {
|
||||
data_in[i] = rng() & 0xFF;
|
||||
}
|
||||
ggml_backend_tensor_set(x, data_in.data(), 0, data_in.size());
|
||||
|
||||
ggml_gallocr_ptr galloc(ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)));
|
||||
ggml_gallocr_alloc_graph(galloc.get(), gf);
|
||||
|
||||
// warm up, so that one-time costs such as lazy CUDA module loading are not timed
|
||||
ggml_backend_graph_compute(backend, gf);
|
||||
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
const ggml_status status = ggml_backend_graph_compute(backend, gf);
|
||||
const int64_t t_us = ggml_time_us() - t_start_us;
|
||||
|
||||
std::vector<uint8_t> data_out(ggml_nbytes(out));
|
||||
ggml_backend_tensor_get(out, data_out.data(), 0, data_out.size());
|
||||
|
||||
const int64_t t_min_us = (int64_t) n_nodes * us;
|
||||
|
||||
const bool ok_status = status == GGML_STATUS_SUCCESS;
|
||||
const bool ok_alias = out->data == x->data;
|
||||
const bool ok_data = memcmp(data_in.data(), data_out.data(), data_in.size()) == 0;
|
||||
const bool ok_time = t_us >= t_min_us;
|
||||
|
||||
printf("%selapsed=%7" PRId64 " us (>= %6" PRId64 ") alias: %-4s data: %-4s time: %s\n",
|
||||
desc, t_us, t_min_us, ok_alias ? "OK" : "FAIL", ok_data ? "OK" : "FAIL", ok_time ? "OK" : "FAIL");
|
||||
|
||||
if (!ok_status) {
|
||||
printf(" compute failed: %s\n", ggml_status_to_string(status));
|
||||
}
|
||||
|
||||
result = ok_status && ok_alias && ok_data && ok_time ? TEST_OK : TEST_FAIL;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
int main() {
|
||||
ggml_time_init();
|
||||
ggml_backend_load_all();
|
||||
|
||||
int n_ok = 0;
|
||||
int n_fail = 0;
|
||||
int n_skip = 0;
|
||||
|
||||
for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
|
||||
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
|
||||
|
||||
ggml_backend_ptr backend(ggml_backend_dev_init(dev, nullptr));
|
||||
if (backend == nullptr) {
|
||||
printf(" failed to initialize %s\n", ggml_backend_dev_name(dev));
|
||||
n_fail++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// a zero duration must still be a valid no-op, longer ones are checked against the clock,
|
||||
// and the last case verifies that a chain of sleeps accumulates instead of collapsing into one
|
||||
const struct {
|
||||
ggml_type type;
|
||||
int64_t ne;
|
||||
int32_t us;
|
||||
int n_nodes;
|
||||
} cases[] = {
|
||||
{ GGML_TYPE_F32, 1, 0, 1 },
|
||||
{ GGML_TYPE_F32, 4096, 10000, 1 },
|
||||
{ GGML_TYPE_F16, 1024, 10000, 1 },
|
||||
{ GGML_TYPE_F32, 1 << 20, 5000, 4 },
|
||||
};
|
||||
|
||||
for (const auto & c : cases) {
|
||||
switch (test_sleep(backend.get(), c.type, c.ne, c.us, c.n_nodes)) {
|
||||
case TEST_OK: n_ok++; break;
|
||||
case TEST_FAIL: n_fail++; break;
|
||||
case TEST_SKIP: n_skip++; break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
printf("%d passed, %d failed, %d skipped\n", n_ok, n_fail, n_skip);
|
||||
|
||||
return n_fail > 0;
|
||||
}
|
||||
@@ -353,6 +353,8 @@ struct server_slot {
|
||||
return false;
|
||||
}
|
||||
|
||||
const int64_t t_start = ggml_time_us();
|
||||
|
||||
const size_t cur_size_tgt = llama_state_seq_get_size_ext(ctx_tgt, id, LLAMA_STATE_SEQ_FLAGS_NONE);
|
||||
const size_t cur_size_dft = ctx_dft ? llama_state_seq_get_size_ext(ctx_dft, id, LLAMA_STATE_SEQ_FLAGS_NONE) : 0;
|
||||
|
||||
@@ -374,6 +376,8 @@ struct server_slot {
|
||||
return false;
|
||||
}
|
||||
|
||||
SLT_TRC(*this, "prompt cache save took %.2f ms\n", (ggml_time_us() - t_start) / 1000.0);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -208,6 +208,8 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
|
||||
if (it_best != states.end()) {
|
||||
SRV_TRC(" - found better prompt with f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
|
||||
|
||||
const int64_t t_start = ggml_time_us();
|
||||
|
||||
{
|
||||
auto & data = it_best->data.main;
|
||||
|
||||
@@ -252,6 +254,8 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
|
||||
prompt = std::move(it_best->prompt);
|
||||
|
||||
states.erase(it_best);
|
||||
|
||||
SRV_TRC("slot %d: prompt cache restore took %.2f ms\n", id_slot, (ggml_time_us() - t_start) / 1000.0);
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
Reference in New Issue
Block a user