upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-04 09:00:48 -05:00
Code Issues Packages Projects Releases Wiki Activity

ggml-cuda: Allow transpose-free gemmv computation (#26171)

Browse Source 
When matrix's weights are shaped 1xK is leverage a transpose-free
computation to use mat_mul_vec_f.
This commit is contained in:
Robert Esclapez
2026-07-30 15:39:46 +02:00
committed by GitHub
parent 6b36c23056
commit e1a1abb787
2 changed files with 18 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
ggml/src/ggml-cuda/ggml-cuda.cu
View File

@@ -1836,6 +1836,20 @@ static void ggml_cuda_mul_mat(ggml_backend_cuda_context & ctx, const ggml_tensor
ggml_cuda_mul_mat_vec_f(ctx, src0, src1, nullptr, dst);
return;
}
// A transposed vector can still use MMVQ (i.e. ne01 == 1)
if (ne01 == 1 && ne11 > MMVF_MAX_BATCH_SIZE && ne2 == 1 && ne3 == 1
&& src0->type == GGML_TYPE_F32
&& ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)
&& ggml_cuda_should_use_mmvf(src1->type, cc, src1->ne, src1->nb, /*ne11 =*/ 1)) {
ggml_tensor dst_vec = *dst;
dst_vec.ne[0] = ne11;
dst_vec.ne[1] = 1;
dst_vec.nb[1] = dst_vec.nb[0]*ne11;
dst_vec.nb[2] = dst_vec.nb[1];
dst_vec.nb[3] = dst_vec.nb[1];
ggml_cuda_mul_mat_vec_f(ctx, src1, src0, nullptr, &dst_vec);
return;
}
if (ggml_cuda_should_use_mmf(src0->type, cc, warp_size, src0->ne, src0->nb, ne11, /*mul_mat_id =*/ false)) {
ggml_cuda_mul_mat_f(ctx, src0, src1, nullptr, dst);
return;

4
tests/test-backend-ops.cpp
View File

@@ -8838,6 +8838,10 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_MXFP4, GGML_TYPE_F32, 2880, 32, 2880, {1, 1}, {1, 1}));
// m == 1, with n on both sides of MMVF_MAX_BATCH_SIZE (8): mmvf below, operand swap above
for (int64_t n : {1, 7, 8, 9, 16, 128, 512}) {
test_cases.emplace_back(new test_mul_mat(GGML_TYPE_F32, GGML_TYPE_F32, 1, n, 2048, {1, 1}, {1, 1}));
}
#if 0
{