upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-01 18:37:28 -05:00
Code Issues Packages Projects Releases Wiki Activity

clean up mul_mat b-type selection

Browse Source
This commit is contained in:
Ruben Ortlam
2026-09-30 14:24:48 +02:00
parent 0e8f0c31b2
commit 944962df8d
1 changed files with 50 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-vulkan/ggml-vulkan.cpp
+50 -70
View File
@@ -6265,6 +6265,20 @@ static vk_pipeline ggml_vk_get_64b_indexing_pipeline(ggml_backend_vk_context * c
return pipeline;
}
// perf-heuristic for choosing mul_mat b type, based on shader availability
static ggml_type ggml_vk_mul_mat_b_type(
ggml_backend_vk_context * ctx, ggml_type src0_type, const ggml_tensor * src1,
ggml_type f16_type, ggml_prec prec, bool mul_mat_id, bool can_quantize_y, bool prefer_f16_b) {
auto available = [&](ggml_type b) {
return ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0_type, b, prec, mul_mat_id) != nullptr;
};
if (can_quantize_y && available(GGML_TYPE_Q8_1)) return GGML_TYPE_Q8_1;
if (prefer_f16_b && available(f16_type)) return f16_type;
if (ggml_vk_dim01_contiguous(src1) && available(src1->type)) return src1->type;
if (src1->type == GGML_TYPE_BF16 && available(GGML_TYPE_F32)) return GGML_TYPE_F32;
return f16_type;
}
static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& subctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, bool disable_split_k) {
VK_LOG_DEBUG("ggml_vk_mul_mat_q_f16((" << src0 << ", name=" << src0->name << ", type=" << ggml_type_name(src0->type) << ", ne0=" << src0->ne[0] << ", ne1=" << src0->ne[1] << ", ne2=" << src0->ne[2] << ", ne3=" << src0->ne[3] << ", nb0=" << src0->nb[0] << ", nb1=" << src0->nb[1] << ", nb2=" << src0->nb[2] << ", nb3=" << src0->nb[3];
std::cerr << "), (" << src1 << ", name=" << src1->name << ", type=" << ggml_type_name(src1->type) << ", ne0=" << src1->ne[0] << ", ne1=" << src1->ne[1] << ", ne2=" << src1->ne[2] << ", ne3=" << src1->ne[3] << ", nb0=" << src1->nb[0] << ", nb1=" << src1->nb[1] << ", nb2=" << src1->nb[2] << ", nb3=" << src1->nb[3];
@@ -6309,61 +6323,35 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
src1_uma = d_Qy != nullptr;
}
// TODO: Clean up this logic to pick src1 type by capability
// Reformat and convert to fp16 if non-contiguous, or for coopmat2 for better perf
const bool x_non_contig = (ctx->device->coopmat2 && src0->type == GGML_TYPE_F32) ||
!ggml_vk_dim01_contiguous(src0);
// If src0 is BF16, try to use a BF16 x BF16 multiply
ggml_type f16_type = src0->type == GGML_TYPE_BF16 ? GGML_TYPE_BF16 : GGML_TYPE_F16;
// BF16 src1 with a non-BF16 src0 has no matching kernel and must be converted.
const bool widen_bf16 = src1->type == GGML_TYPE_BF16 && f16_type != GGML_TYPE_BF16;
const bool is_coopmat = ctx->device->coopmat_support || ctx->device->coopmat2;
const bool can_quantize_y = (ctx->device->integer_dot_product || ctx->device->coopmat_int_support) &&
src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0;
// coopmat shaders are fast enough that memory bandwidth reduction from f16 activations helps
const bool prefer_f16_b = is_coopmat && ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32;
// Prefer the int8 MMQ path (quantize src1 to q8_1) whenever a matching pipeline exists.
// The pipeline lookup returns nullptr for types without a q8_1 pipeline (e.g. RDNA4-skipped
// quants), in which case coopmat1 falls back to the f16 B-type quant matmul below.
bool quantize_y = (ctx->device->integer_dot_product || ctx->device->coopmat_int_support) &&
src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0;
const ggml_type y_kernel_type = ggml_vk_mul_mat_b_type(ctx, src0->type, src1, f16_type,
(ggml_prec)dst->op_params[0], false, can_quantize_y, prefer_f16_b);
// Check for mmq first
const std::vector<vk_matmul_pipeline_pair>* mmp_map = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0]) : nullptr;
if (mmp_map == nullptr) {
quantize_y = false;
}
const bool quantize_y = y_kernel_type == GGML_TYPE_Q8_1;
const bool y_f32_kernel = y_kernel_type == GGML_TYPE_F32;
const bool y_needs_reformat = !quantize_y && (!ggml_vk_dim01_contiguous(src1) || y_kernel_type != src1->type);
const bool y_non_contig = (ctx->device->coopmat2 && src1->type == GGML_TYPE_F32) ||
// coopmat1: force f32->f16 conversion so the f16 B-type quant pipeline is
// used, but only when the int8 MMQ path above is not taken.
(ctx->device->coopmat_support && !ctx->device->coopmat2 && !quantize_y &&
ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32) ||
(src0->type == GGML_TYPE_BF16 && src1->type != GGML_TYPE_BF16) ||
widen_bf16 ||
!ggml_vk_dim01_contiguous(src1);
// coopmat2 has no F32-activation pipelines, so BF16 widens to F16 there, else F32
// this makes the same range assumption that fp32->fp16 already makes
const ggml_type y_kernel_type = !y_non_contig ? src1->type
: (widen_bf16 && !ctx->device->coopmat2) ? GGML_TYPE_F32
: f16_type;
const bool y_f32_kernel = y_kernel_type == GGML_TYPE_F32;
if (mmp_map == nullptr) {
// Fall back to f16 dequant mul mat
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, (ggml_prec)dst->op_params[0]);
}
const std::vector<vk_matmul_pipeline_pair>* mmp_map =
ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, (ggml_prec)dst->op_params[0]);
const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig;
const bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig);
const bool qy_needs_dequant = y_needs_reformat;
if (qx_needs_dequant) {
// Fall back to dequant + f16 mulmat
// dequant src0 to f16
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, (ggml_prec)dst->op_params[0]);
}
// Not implemented
GGML_ASSERT(y_non_contig || !qy_needs_dequant); // NOLINT
GGML_ASSERT(mmp_map != nullptr);
const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, ne11, false));
@@ -6399,7 +6387,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
} else {
to_fp16_vk_0 = ggml_vk_get_to_fp16(ctx, src0->type);
}
if (y_non_contig) {
if (y_needs_reformat) {
to_fp16_vk_1 = ggml_vk_get_cpy_pipeline(ctx, src1, nullptr, y_kernel_type);
} else {
to_fp16_vk_1 = ggml_vk_get_to_fp16(ctx, src1->type);
@@ -6498,7 +6486,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
ggml_vk_dispatch_pipeline(ctx, subctx, to_fp16_vk_0, { vk_subbuffer{ d_Qx, qx_buf_offset, qx_sz }, vk_subbuffer{ d_X, 0, x_sz } }, pc, { (uint32_t)(x_ne), 1, 1});
ggml_vk_sync_buffers(ctx, subctx);
}
if (y_non_contig) {
if (y_needs_reformat) {
if (ctx->prealloc_y_last_pipeline_used != to_fp16_vk_1.get() ||
ctx->prealloc_y_last_tensor_used != src1 ||
ctx->prealloc_y_last_k_padded) {
@@ -6554,7 +6542,7 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
if (x_non_contig || qx_needs_dequant) {
ctx->prealloc_x_need_sync = true;
}
if (y_non_contig || quantize_y) {
if (y_needs_reformat || quantize_y) {
ctx->prealloc_y_need_sync = true;
}
}
@@ -7399,46 +7387,38 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
#else
const bool y_decode_vector_staging = false;
#endif
const bool y_non_contig = y_decode_vector_staging ||
(ctx->device->coopmat2 && src1->type == GGML_TYPE_F32) ||
// Intel coopmat1: force f32->f16 conversion so the f16 B-type quant pipeline is used.
(ctx->device->coopmat_support && !ctx->device->coopmat2 &&
ctx->device->vendor_id == VK_VENDOR_ID_INTEL &&
ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32) ||
(src0->type == GGML_TYPE_BF16 && src1->type != GGML_TYPE_BF16) ||
!ggml_vk_dim01_contiguous(src1);
const bool can_quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 &&
ggml_is_contiguous(src1) && (ne11 * ne10) % 4 == 0;
// Intel coopmat1 matrix cores prefer f16 activations for quantized weights
const bool prefer_f16_b = ctx->device->coopmat_support && !ctx->device->coopmat2 &&
ctx->device->vendor_id == VK_VENDOR_ID_INTEL &&
ggml_is_quantized(src0->type) && src1->type == GGML_TYPE_F32;
const bool y_f32_kernel = src1->type == GGML_TYPE_F32 && !y_non_contig;
const ggml_type y_kernel_type = ggml_vk_mul_mat_b_type(ctx, src0->type, src1, f16_type,
(ggml_prec)dst->op_params[0], true, can_quantize_y, prefer_f16_b);
bool quantize_y = ctx->device->integer_dot_product && src1->type == GGML_TYPE_F32 && ggml_is_contiguous(src1) && !y_non_contig && (ne11 * ne10) % 4 == 0;
const bool quantize_y = y_kernel_type == GGML_TYPE_Q8_1;
const bool y_f32_kernel = y_kernel_type == GGML_TYPE_F32;
// Check for mmq first
const std::vector<vk_matmul_pipeline_pair>* mmp_map = quantize_y ? ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, GGML_TYPE_Q8_1, (ggml_prec)dst->op_params[0], true) : nullptr;
if (mmp_map == nullptr) {
// Fall back to f16 dequant mul mat
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_non_contig ? f16_type : src1->type, (ggml_prec)dst->op_params[0], true);
quantize_y = false;
}
const std::vector<vk_matmul_pipeline_pair>* mmp_map =
ggml_vk_get_mul_mat_mat_pipeline_map(ctx, src0->type, y_kernel_type, (ggml_prec)dst->op_params[0], true);
const bool qx_needs_dequant = mmp_map == nullptr || x_non_contig;
bool qy_needs_dequant = !quantize_y && ((src1->type != f16_type && !y_f32_kernel) || y_non_contig);
if (qx_needs_dequant) {
// Fall back to dequant + f16 mulmat
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_f32_kernel ? GGML_TYPE_F32 : f16_type, (ggml_prec)dst->op_params[0], true);
// dequant src0 to f16
mmp_map = ggml_vk_get_mul_mat_mat_pipeline_map(ctx, f16_type, y_kernel_type, (ggml_prec)dst->op_params[0], true);
}
GGML_ASSERT(mmp_map != nullptr);
// Coopmat2 MUL_MAT_ID BK specialization constants in ggml_vk_load_shaders are at most 64.
const uint32_t y_staged_row_stride = ctx->device->coopmat2 && !quantize_y ? ggml_vk_align_size(ne10, 64) : ne10;
const bool y_needs_k_padding = ne10 != y_staged_row_stride;
const bool y_needs_reformat = y_non_contig || y_needs_k_padding;
qy_needs_dequant = qy_needs_dequant || y_needs_k_padding;
// Not implemented
GGML_ASSERT(y_needs_reformat || !qy_needs_dequant); // NOLINT
GGML_ASSERT(mmp_map != nullptr);
const bool y_needs_reformat = !quantize_y && (!ggml_vk_dim01_contiguous(src1) || y_kernel_type != src1->type ||
y_decode_vector_staging || y_needs_k_padding);
const bool qy_needs_dequant = y_needs_reformat;
const uint32_t n_per_expert = (uint32_t)CEIL_DIV(nei0 * nei1, n_as);
const uint32_t kpad = quantize_y ? 0 : ggml_vk_align_size(ne10, ggml_vk_guess_matmul_pipeline_align_map(ctx, *mmp_map, ne01, n_per_expert, true));