From 57fb74fba34b92feb0d22f0c80b638e9802d13ea Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Fri, 15 May 2026 13:10:19 +0200 Subject: [PATCH] add q4_1, q8_0, iq4_nl repacking --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 514 ++++++++++++++++++ .../vulkan-shaders/dequant_funcs.glsl | 41 +- .../vulkan-shaders/dequant_funcs_cm2.glsl | 53 +- .../vulkan-shaders/mul_mat_vecq_funcs.glsl | 15 +- .../vulkan-shaders/mul_mm_cm2.comp | 4 + .../vulkan-shaders/mul_mm_funcs.glsl | 17 + .../vulkan-shaders/mul_mmq_funcs.glsl | 17 + .../vulkan-shaders/vulkan-shaders-gen.cpp | 4 +- 8 files changed, 655 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 4c8e1ebcde..fd165db6e3 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -7339,10 +7339,67 @@ static size_t ggml_vk_repack_mxfp4_size_tensor(const ggml_tensor * tensor) { return ggml_vk_repack_mxfp4_size(ggml_vk_get_num_blocks(tensor)); } +static size_t ggml_vk_repack_q4_1_delta_offset(size_t n_blocks) { + return GGML_PAD(n_blocks * 16, VULKAN_REPACK_ALIGNMENT); +} + +static size_t ggml_vk_repack_q4_1_size(size_t n_blocks) { + return ggml_vk_repack_q4_1_delta_offset(n_blocks) + n_blocks * 4; +} + +static size_t ggml_vk_repack_q4_1_delta_offset_tensor(const ggml_tensor * tensor) { + return ggml_vk_repack_q4_1_delta_offset(ggml_vk_get_num_blocks(tensor)); +} + +static size_t ggml_vk_repack_q4_1_size_tensor(const ggml_tensor * tensor) { + return ggml_vk_repack_q4_1_size(ggml_vk_get_num_blocks(tensor)); +} + +static size_t ggml_vk_repack_q8_0_delta_offset(size_t n_blocks) { + return GGML_PAD(n_blocks * 32, VULKAN_REPACK_ALIGNMENT); +} + +static size_t ggml_vk_repack_q8_0_size(size_t n_blocks) { + return ggml_vk_repack_q8_0_delta_offset(n_blocks) + n_blocks * 2; +} + +static size_t ggml_vk_repack_q8_0_delta_offset_tensor(const ggml_tensor * tensor) { + return ggml_vk_repack_q8_0_delta_offset(ggml_vk_get_num_blocks(tensor)); +} + +static size_t ggml_vk_repack_q8_0_size_tensor(const ggml_tensor * tensor) { + return ggml_vk_repack_q8_0_size(ggml_vk_get_num_blocks(tensor)); +} + +static size_t ggml_vk_repack_iq4_nl_delta_offset(size_t n_blocks) { + return GGML_PAD(n_blocks * 16, VULKAN_REPACK_ALIGNMENT); +} + +static size_t ggml_vk_repack_iq4_nl_size(size_t n_blocks) { + return ggml_vk_repack_iq4_nl_delta_offset(n_blocks) + n_blocks * 2; +} + +static size_t ggml_vk_repack_iq4_nl_delta_offset_tensor(const ggml_tensor * tensor) { + return ggml_vk_repack_iq4_nl_delta_offset(ggml_vk_get_num_blocks(tensor)); +} + +static size_t ggml_vk_repack_iq4_nl_size_tensor(const ggml_tensor * tensor) { + return ggml_vk_repack_iq4_nl_size(ggml_vk_get_num_blocks(tensor)); +} + static size_t ggml_vk_repack_size_tensor(const ggml_tensor * tensor) { if (tensor->type == GGML_TYPE_Q4_0) { return ggml_vk_repack_q4_0_size_tensor(tensor); } + if (tensor->type == GGML_TYPE_Q4_1) { + return ggml_vk_repack_q4_1_size_tensor(tensor); + } + if (tensor->type == GGML_TYPE_Q8_0) { + return ggml_vk_repack_q8_0_size_tensor(tensor); + } + if (tensor->type == GGML_TYPE_IQ4_NL) { + return ggml_vk_repack_iq4_nl_size_tensor(tensor); + } if (tensor->type == GGML_TYPE_MXFP4) { return ggml_vk_repack_mxfp4_size_tensor(tensor); } @@ -7993,6 +8050,12 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub uint32_t deltas_offset = 0; if (src0->type == GGML_TYPE_Q4_0) { deltas_offset = ggml_vk_repack_q4_0_delta_offset_tensor(src0) / 2; + } else if (src0->type == GGML_TYPE_Q4_1) { + deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(src0) / 2; + } else if (src0->type == GGML_TYPE_Q8_0) { + deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(src0) / 2; + } else if (src0->type == GGML_TYPE_IQ4_NL) { + deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(src0) / 2; } else if (src0->type == GGML_TYPE_MXFP4) { deltas_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(src0); } @@ -8300,6 +8363,12 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context& uint32_t deltas_offset = 0; if (src0->type == GGML_TYPE_Q4_0) { deltas_offset = ggml_vk_repack_q4_0_delta_offset_tensor(src0) / 2; + } else if (src0->type == GGML_TYPE_Q4_1) { + deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(src0) / 2; + } else if (src0->type == GGML_TYPE_Q8_0) { + deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(src0) / 2; + } else if (src0->type == GGML_TYPE_IQ4_NL) { + deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(src0) / 2; } else if (src0->type == GGML_TYPE_MXFP4) { deltas_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(src0); } @@ -8861,6 +8930,12 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context& uint32_t deltas_offset = 0; if (src0->type == GGML_TYPE_Q4_0) { deltas_offset = ggml_vk_repack_q4_0_delta_offset_tensor(src0) / 2; + } else if (src0->type == GGML_TYPE_Q4_1) { + deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(src0) / 2; + } else if (src0->type == GGML_TYPE_Q8_0) { + deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(src0) / 2; + } else if (src0->type == GGML_TYPE_IQ4_NL) { + deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(src0) / 2; } else if (src0->type == GGML_TYPE_MXFP4) { deltas_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(src0); } @@ -9091,6 +9166,12 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte uint32_t deltas_offset = 0; if (src0->type == GGML_TYPE_Q4_0) { deltas_offset = ggml_vk_repack_q4_0_delta_offset_tensor(src0) / 2; + } else if (src0->type == GGML_TYPE_Q4_1) { + deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(src0) / 2; + } else if (src0->type == GGML_TYPE_Q8_0) { + deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(src0) / 2; + } else if (src0->type == GGML_TYPE_IQ4_NL) { + deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(src0) / 2; } else if (src0->type == GGML_TYPE_MXFP4) { deltas_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(src0); } @@ -13911,6 +13992,66 @@ static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml return; } + if (tensor->type == GGML_TYPE_Q4_1) { + const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + const block_q4_1 * src = (const block_q4_1 *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(quants + 16 * i, src[i].qs, 16); + memcpy(&deltas[i * 2], &src[i].data.dm, sizeof(ggml_half2)); + } + + ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + return; + } + + if (tensor->type == GGML_TYPE_Q8_0) { + const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + const block_q8_0 * src = (const block_q8_0 *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(quants + 32 * i, src[i].qs, 32); + deltas[i] = src[i].d; + } + + ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + return; + } + + if (tensor->type == GGML_TYPE_IQ4_NL) { + const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + const block_iq4_nl * src = (const block_iq4_nl *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(quants + 16 * i, src[i].qs, 16); + deltas[i] = src[i].d; + } + + ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + return; + } + if (tensor->type == GGML_TYPE_MXFP4) { const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor); const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor); @@ -13965,6 +14106,66 @@ static void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, g return; } + if (tensor->type == GGML_TYPE_Q4_1) { + const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + const block_q4_1 * src = (const block_q4_1 *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(quants + 16 * i, src[i].qs, 16); + memcpy(&deltas[i * 2], &src[i].data.dm, sizeof(ggml_half2)); + } + + ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + return; + } + + if (tensor->type == GGML_TYPE_Q8_0) { + const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + const block_q8_0 * src = (const block_q8_0 *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(quants + 32 * i, src[i].qs, 32); + deltas[i] = src[i].d; + } + + ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + return; + } + + if (tensor->type == GGML_TYPE_IQ4_NL) { + const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + const block_iq4_nl * src = (const block_iq4_nl *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(quants + 16 * i, src[i].qs, 16); + deltas[i] = src[i].d; + } + + ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + return; + } + if (tensor->type == GGML_TYPE_MXFP4) { const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor); const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor); @@ -14018,6 +14219,66 @@ static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, cons return; } + if (tensor->type == GGML_TYPE_Q4_1) { + const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + block_q4_1 * dst = (block_q4_1 *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(dst[i].qs, quants + 16 * i, 16); + memcpy(&dst[i].data.dm, &deltas[i * 2], sizeof(ggml_half2)); + } + + return; + } + + if (tensor->type == GGML_TYPE_Q8_0) { + const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + block_q8_0 * dst = (block_q8_0 *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(dst[i].qs, quants + 32 * i, 32); + dst[i].d = deltas[i]; + } + + return; + } + + if (tensor->type == GGML_TYPE_IQ4_NL) { + const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + block_iq4_nl * dst = (block_iq4_nl *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(dst[i].qs, quants + 16 * i, 16); + dst[i].d = deltas[i]; + } + + return; + } + if (tensor->type == GGML_TYPE_MXFP4) { const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor); const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor); @@ -14073,6 +14334,66 @@ static void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, c return; } + if (tensor->type == GGML_TYPE_Q4_1) { + const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + block_q4_1 * dst = (block_q4_1 *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(dst[i].qs, quants + 16 * i, 16); + memcpy(&dst[i].data.dm, &deltas[i * 2], sizeof(ggml_half2)); + } + + return; + } + + if (tensor->type == GGML_TYPE_Q8_0) { + const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + block_q8_0 * dst = (block_q8_0 *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(dst[i].qs, quants + 32 * i, 32); + dst[i].d = deltas[i]; + } + + return; + } + + if (tensor->type == GGML_TYPE_IQ4_NL) { + const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + block_iq4_nl * dst = (block_iq4_nl *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(dst[i].qs, quants + 16 * i, 16); + dst[i].d = deltas[i]; + } + + return; + } + if (tensor->type == GGML_TYPE_MXFP4) { const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor); const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor); @@ -14177,6 +14498,24 @@ static size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_typ return ggml_vk_repack_q4_0_size(num_blocks_per_row * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]); } + if (tensor->type == GGML_TYPE_Q4_1) { + const size_t num_blocks_per_row = tensor->ne[0] / ggml_blck_size(tensor->type); + + return ggml_vk_repack_q4_1_size(num_blocks_per_row * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]); + } + + if (tensor->type == GGML_TYPE_Q8_0) { + const size_t num_blocks_per_row = tensor->ne[0] / ggml_blck_size(tensor->type); + + return ggml_vk_repack_q8_0_size(num_blocks_per_row * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]); + } + + if (tensor->type == GGML_TYPE_IQ4_NL) { + const size_t num_blocks_per_row = tensor->ne[0] / ggml_blck_size(tensor->type); + + return ggml_vk_repack_iq4_nl_size(num_blocks_per_row * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]); + } + if (tensor->type == GGML_TYPE_MXFP4) { const size_t num_blocks_per_row = tensor->ne[0] / ggml_blck_size(tensor->type); @@ -14332,6 +14671,66 @@ static void ggml_backend_vk_set_tensor_2d_async(ggml_backend_t backend, ggml_ten return; } + if (tensor->type == GGML_TYPE_Q4_1) { + const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + const block_q4_1 * src = (const block_q4_1 *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(quants + 16 * i, src[i].qs, 16); + memcpy(&deltas[i * 2], &src[i].data.dm, sizeof(ggml_half2)); + } + + ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + return; + } + + if (tensor->type == GGML_TYPE_Q8_0) { + const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + const block_q8_0 * src = (const block_q8_0 *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(quants + 32 * i, src[i].qs, 32); + deltas[i] = src[i].d; + } + + ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + return; + } + + if (tensor->type == GGML_TYPE_IQ4_NL) { + const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + const block_iq4_nl * src = (const block_iq4_nl *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(quants + 16 * i, src[i].qs, 16); + deltas[i] = src[i].d; + } + + ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + return; + } + if (tensor->type == GGML_TYPE_MXFP4) { const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor); const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor); @@ -14440,6 +14839,66 @@ static void ggml_backend_vk_get_tensor_2d_async(ggml_backend_t backend, const gg return; } + if (tensor->type == GGML_TYPE_Q4_1) { + const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + block_q4_1 * dst = (block_q4_1 *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(dst[i].qs, quants + 16 * i, 16); + memcpy(&dst[i].data.dm, &deltas[i * 2], sizeof(ggml_half2)); + } + + return; + } + + if (tensor->type == GGML_TYPE_Q8_0) { + const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + block_q8_0 * dst = (block_q8_0 *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(dst[i].qs, quants + 32 * i, 32); + dst[i].d = deltas[i]; + } + + return; + } + + if (tensor->type == GGML_TYPE_IQ4_NL) { + const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(tensor); + const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(tensor); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size); + + block_iq4_nl * dst = (block_iq4_nl *)data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) { + memcpy(dst[i].qs, quants + 16 * i, 16); + dst[i].d = deltas[i]; + } + + return; + } + if (tensor->type == GGML_TYPE_MXFP4) { const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor); const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor); @@ -17001,6 +17460,61 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph * dst[i].d = deltas[i]; } + memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS); + } else if (srci->type == GGML_TYPE_Q4_1) { + const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(srci); + const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(srci); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + ggml_vk_buffer_read(buffer_gpu, offset, data_repacked, repacked_size); + + block_q4_1 * dst = (block_q4_1 *)srci_clone->data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(srci); i++) { + memcpy(dst[i].qs, quants + 16 * i, 16); + dst[i].d = deltas[i * 2]; + dst[i].m = deltas[i * 2 + 1]; + } + + memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS); + } else if (srci->type == GGML_TYPE_Q8_0) { + const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(srci); + const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(srci); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + ggml_vk_buffer_read(buffer_gpu, offset, data_repacked, repacked_size); + + block_q8_0 * dst = (block_q8_0 *)srci_clone->data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(srci); i++) { + memcpy(dst[i].qs, quants + 32 * i, 32); + dst[i].d = deltas[i]; + } + + memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS); + } else if (srci->type == GGML_TYPE_IQ4_NL) { + const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(srci); + const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(srci); + + void * data_repacked = ggml_vk_repack_scratch(repacked_size); + uint8_t * quants = (uint8_t *)data_repacked; + ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset); + + ggml_vk_buffer_read(buffer_gpu, offset, data_repacked, repacked_size); + + block_iq4_nl * dst = (block_iq4_nl *)srci_clone->data; + + for (size_t i = 0; i < ggml_vk_get_num_blocks(srci); i++) { + memcpy(dst[i].qs, quants + 16 * i, 16); + dst[i].d = deltas[i]; + } + memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS); } else if (srci->type == GGML_TYPE_MXFP4) { const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(srci); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl index 954e56cc19..e4640cf05c 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl @@ -45,6 +45,16 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { #endif #if defined(DATA_A_Q4_1) +#if defined(A_TYPE_REPACKED) +vec2 dequantize(uint ib, uint iqs, uint a_offset) { + const uint vui = uint(data_a_quants[(a_offset + ib) * 16 + iqs]); + return vec2(vui & 0xF, vui >> 4); +} +vec4 dequantize4(uint ib, uint iqs, uint a_offset) { + const uint vui = uint(data_a_quants16[(a_offset + ib) * 8 + iqs/2]); + return vec4(vui & 0xF, (vui >> 4) & 0xF, (vui >> 8) & 0xF, vui >> 12); +} +#else vec2 dequantize(uint ib, uint iqs, uint a_offset) { const uint vui = uint(data_a[a_offset + ib].qs[iqs]); return vec2(vui & 0xF, vui >> 4); @@ -54,6 +64,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { return vec4(vui & 0xF, (vui >> 4) & 0xF, (vui >> 8) & 0xF, vui >> 12); } #endif +#endif #if defined(DATA_A_Q5_0) vec2 dequantize(uint ib, uint iqs, uint a_offset) { @@ -88,6 +99,17 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { #endif #if defined(DATA_A_Q8_0) +#if defined(A_TYPE_REPACKED) +vec2 dequantize(uint ib, uint iqs, uint a_offset) { + return vec2(int(int8_t(data_a_quants[(a_offset + ib) * 32 + iqs])), + int(int8_t(data_a_quants[(a_offset + ib) * 32 + iqs + 1]))); +} +vec4 dequantize4(uint ib, uint iqs, uint a_offset) { + const i8vec2 v0 = unpack8(int32_t(data_a_quants16[(a_offset + ib) * 16 + iqs/2])).xy; + const i8vec2 v1 = unpack8(int32_t(data_a_quants16[(a_offset + ib) * 16 + iqs/2 + 1])).xy; + return vec4(v0.x, v0.y, v1.x, v1.y); +} +#else vec2 dequantize(uint ib, uint iqs, uint a_offset) { return vec2(int(data_a[a_offset + ib].qs[iqs]), int(data_a[a_offset + ib].qs[iqs + 1])); } @@ -97,6 +119,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { return vec4(v0.x, v0.y, v1.x, v1.y); } #endif +#endif #if defined(DATA_A_Q1_0) vec2 dequantize(uint ib, uint iqs, uint a_offset) { @@ -439,6 +462,16 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { #endif #if defined(DATA_A_IQ4_NL) +#if defined(A_TYPE_REPACKED) +vec2 dequantize(uint ib, uint iqs, uint a_offset) { + const uint vui = uint(data_a_quants[(a_offset + ib) * 16 + iqs]); + return vec2(kvalues_iq4nl[vui & 0xF], kvalues_iq4nl[vui >> 4]); +} +vec4 dequantize4(uint ib, uint iqs, uint a_offset) { + const uint vui = uint(data_a_quants16[(a_offset + ib) * 8 + iqs/2]); + return vec4(kvalues_iq4nl[vui & 0xF], kvalues_iq4nl[(vui >> 4) & 0xF], kvalues_iq4nl[(vui >> 8) & 0xF], kvalues_iq4nl[vui >> 12]); +} +#else vec2 dequantize(uint ib, uint iqs, uint a_offset) { const uint vui = uint(data_a[a_offset + ib].qs[iqs]); return vec2(kvalues_iq4nl[vui & 0xF], kvalues_iq4nl[vui >> 4]); @@ -448,6 +481,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { return vec4(kvalues_iq4nl[vui & 0xF], kvalues_iq4nl[(vui >> 4) & 0xF], kvalues_iq4nl[(vui >> 8) & 0xF], kvalues_iq4nl[vui >> 12]); } #endif +#endif #if defined(DATA_A_MXFP4) #if defined(A_TYPE_REPACKED) @@ -509,7 +543,7 @@ vec2 get_dm(uint ib, uint a_offset) { #if defined(DATA_A_Q4_0) || defined(DATA_A_Q5_0) || defined(DATA_A_Q8_0) || defined(DATA_A_IQ1_S) || defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) || defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) || defined(DATA_A_IQ4_XS) || defined(DATA_A_IQ4_NL) vec2 get_dm(uint ib, uint a_offset) { -#if defined(DATA_A_Q4_0) && defined(A_TYPE_REPACKED) +#if (defined(DATA_A_Q4_0) || defined(DATA_A_Q8_0) || defined(DATA_A_IQ4_NL)) && defined(A_TYPE_REPACKED) return vec2(float(data_a_deltas[a_offset + p.deltas_offset + ib]), 0); #else return vec2(float(data_a[a_offset + ib].d), 0); @@ -542,8 +576,13 @@ vec2 get_dm(uint ib, uint a_offset) { #if defined(DATA_A_Q4_1) || defined(DATA_A_Q5_1) vec2 get_dm(uint ib, uint a_offset) { +#if defined(DATA_A_Q4_1) && defined(A_TYPE_REPACKED) + return vec2(float(data_a_deltas[p.deltas_offset + (a_offset + ib) * 2]), + float(data_a_deltas[p.deltas_offset + (a_offset + ib) * 2 + 1])); +#else const vec2 dm = vec2(data_a_packed32[a_offset + ib].dm); return dm; +#endif } #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl index 2f0c5ad1d7..bf5891f4b7 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl @@ -54,19 +54,36 @@ float16_t dequantFuncQ4_0(const in decodeBufQ4_0 bl, const in uint blockCoords[2 return ret; } +#ifdef A_TYPE_REPACKED +layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4_1 { + uint16_t qs[8]; +}; +#else layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ4_1 { block_q4_1 block; }; +#endif float16_t dequantFuncQ4_1(const in decodeBufQ4_1 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) { + const uint idx = coordInBlock[1]; +#ifdef A_TYPE_REPACKED + const uint ib = pos_a + blockCoords[0] * (p.stride_a / QUANT_K) + blockCoords[1]; + const float16_t d = data_a_deltas[p.deltas_offset + ib * 2]; + const float16_t m = data_a_deltas[p.deltas_offset + ib * 2 + 1]; + uint32_t qs = uint32_t(bl.qs[(idx & 0xE) >> 1]); +#else const float16_t d = bl.block.d; const float16_t m = bl.block.m; - const uint idx = coordInBlock[1]; + uint32_t qs = bl.block.qs[idx & 0xF]; +#endif const uint iqs = idx & 0xF; const uint shift = (idx & 0x10) >> 2; - uint32_t qs = bl.block.qs[iqs]; +#ifdef A_TYPE_REPACKED + qs >>= ((iqs & 1) * 8 + shift); +#else qs >>= shift; +#endif qs &= 0xF; float16_t ret = float16_t(qs) * d + m; return ret; @@ -117,18 +134,28 @@ float16_t dequantFuncQ5_1(const in decodeBufQ5_1 bl, const in uint blockCoords[2 return ret; } +#ifdef A_TYPE_REPACKED +layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ8_0 { + int16_t qs[16]; +}; +#else layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ8_0 { block_q8_0_packed16 block; }; +#endif float16_t dequantFuncQ8_0(const in decodeBufQ8_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2]) { - const float16_t d = bl.block.d; const uint idx = coordInBlock[1]; const uint iqs = idx; - - // Load 16b and select the byte for this element +#ifdef A_TYPE_REPACKED + const uint ib = pos_a + blockCoords[0] * (p.stride_a / QUANT_K) + blockCoords[1]; + const float16_t d = data_a_deltas[p.deltas_offset + ib]; + int32_t qs = unpack8(bl.qs[(iqs & 0x1E) >> 1])[iqs & 1]; +#else + const float16_t d = bl.block.d; int32_t qs = unpack8(bl.block.qs[(iqs & 0x1E) >> 1])[iqs & 1]; +#endif float16_t ret = float16_t(qs) * d; return ret; } @@ -672,18 +699,32 @@ float16_t dequantFuncIQ4_XS(const in decodeBufIQ4_XS bl, const in uint blockCoor #endif #if defined(DATA_A_IQ4_NL) +#ifdef A_TYPE_REPACKED +layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufIQ4_NL { + uint16_t qs[8]; +}; +#else layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4_NL { block_iq4_nl block; }; +#endif float16_t dequantFuncIQ4_NL(const in decodeBufIQ4_NL bl, const in uint blockCoords[2], const in uint coordInBlock[2]) { - const float16_t d = bl.block.d; const uint idx = coordInBlock[1]; +#ifdef A_TYPE_REPACKED + const uint ib = pos_a + blockCoords[0] * (p.stride_a / QUANT_K) + blockCoords[1]; + const float16_t d = data_a_deltas[p.deltas_offset + ib]; + uint32_t qs = uint32_t(bl.qs[(idx & 0xE) >> 1]); + const uint shift = (idx & 0x10) >> 2; + qs >>= ((idx & 1) * 8 + shift); +#else + const float16_t d = bl.block.d; const uint iqs = idx & 0xF; const uint shift = (idx & 0x10) >> 2; uint32_t qs = bl.block.qs[iqs]; qs >>= shift; +#endif qs &= 0xF; float16_t ret = float16_t(kvalues_iq4nl[qs]) * d; return ret; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl index 390967c3bd..1001b35ef9 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl @@ -6,7 +6,7 @@ #if defined(DATA_A_Q4_0) || defined(DATA_A_Q5_0) || defined(DATA_A_Q8_0) || defined(DATA_A_IQ1_S) || defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) || defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) || defined(DATA_A_IQ4_XS) || defined(DATA_A_IQ4_NL) FLOAT_TYPE get_dm(uint ib) { -#if defined(DATA_A_Q4_0) && defined(A_TYPE_REPACKED) +#if (defined(DATA_A_Q4_0) || defined(DATA_A_Q8_0) || defined(DATA_A_IQ4_NL)) && defined(A_TYPE_REPACKED) return FLOAT_TYPE(data_a_deltas[p.deltas_offset + ib]); #else return FLOAT_TYPE(data_a[ib].d); @@ -16,7 +16,12 @@ FLOAT_TYPE get_dm(uint ib) { #if defined(DATA_A_Q4_1) || defined(DATA_A_Q5_1) FLOAT_TYPEV2 get_dm(uint ib) { +#if defined(DATA_A_Q4_1) && defined(A_TYPE_REPACKED) + return FLOAT_TYPEV2(data_a_deltas[p.deltas_offset + ib * 2], + data_a_deltas[p.deltas_offset + ib * 2 + 1]); +#else return FLOAT_TYPEV2(data_a_packed32[ib].dm); +#endif } #endif @@ -60,7 +65,11 @@ FLOAT_TYPE mul_q8_1(const int32_t q_sum, const float da, const vec2 dsb, const i #if defined(DATA_A_Q4_1) // 4-byte loads for Q4_1 blocks (20 bytes) i32vec2 repack(uint ib, uint iqs) { +#if defined(A_TYPE_REPACKED) + const uint32_t vui = data_a_quants32[ib * 4 + iqs]; +#else const uint32_t vui = data_a_packed32[ib].qs[iqs]; +#endif return i32vec2( vui & 0x0F0F0F0F, (vui >> 4) & 0x0F0F0F0F); } @@ -115,8 +124,12 @@ FLOAT_TYPE mul_q8_1(const int32_t q_sum, const vec2 dma, const vec2 dsb, const i #if defined(DATA_A_Q8_0) // 2-byte loads for Q8_0 blocks (34 bytes) int32_t repack(uint ib, uint iqs) { +#if defined(A_TYPE_REPACKED) + return int32_t(data_a_quants32[ib * 8 + iqs]); +#else return pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2 ], data_a_packed16[ib].qs[iqs * 2 + 1])); +#endif } FLOAT_TYPE mul_q8_1(const int32_t q_sum, const float da, const vec2 dsb, const int32_t sum_divisor) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp index 096538a6af..c18636cdf7 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp @@ -68,7 +68,11 @@ layout (push_constant) uniform parameter #ifdef A_TYPE_REPACKED +#if defined(DATA_A_Q8_0) +struct block_repacked_quants { uint16_t qs[16]; }; +#else struct block_repacked_quants { uint16_t qs[8]; }; +#endif layout (binding = 0) readonly buffer A {block_repacked_quants data_a[];}; layout (binding = 0) readonly buffer A_DELTAS {float16_t data_a_deltas[];}; layout (binding = 0) readonly buffer A_SCALES {uint8_t data_a_scales[];}; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl index cc7d526654..fa512acf85 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl @@ -73,8 +73,14 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const uint ib = idx / 4; const uint iqs = idx & 0x03; +#if defined(A_TYPE_REPACKED) + const vec2 dm = vec2(data_a_deltas[p.deltas_offset + ib * 2], + data_a_deltas[p.deltas_offset + ib * 2 + 1]); + const uint vui = data_a_quants32[ib * 4 + iqs]; +#else const vec2 dm = vec2(data_a_packed32[ib].dm); const uint vui = data_a_packed32[ib].qs[iqs]; +#endif const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y; const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y; @@ -128,9 +134,15 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const uint ib = idx / 8; const uint iqs = idx & 0x07; +#if defined(A_TYPE_REPACKED) + const float d = float(data_a_deltas[p.deltas_offset + ib]); + const i8vec2 v0 = unpack8(int32_t(data_a_quants16[ib * 16 + 2*iqs])).xy; + const i8vec2 v1 = unpack8(int32_t(data_a_quants16[ib * 16 + 2*iqs + 1])).xy; +#else const float d = float(data_a_packed16[ib].d); const i8vec2 v0 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs])).xy; // vec4 used due to #12147 const i8vec2 v1 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs + 1])).xy; +#endif const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d; buf_a[buf_idx ] = FLOAT_TYPEV2(v.xy); @@ -486,8 +498,13 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const uint ib = idx / 8; const uint iqs = idx & 0x07; +#if defined(A_TYPE_REPACKED) + const FLOAT_TYPE d = FLOAT_TYPE(data_a_deltas[p.deltas_offset + ib]); + const uint vui = uint(data_a_quants16[ib * 8 + iqs]); +#else const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d); const uint vui = uint(data_a_packed16[ib].qs[iqs]); +#endif buf_a[buf_idx ] = d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF], kvalues_iq4nl[bitfieldExtract(vui, 8, 4)]); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl index 0e9e3d6912..dcfd51519b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl @@ -26,12 +26,21 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { #endif } #else // DATA_A_Q4_1 +#if defined(A_TYPE_REPACKED) + buf_a[buf_ib].qs[iqs] = data_a_quants32[ib * 4 + iqs]; + + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_deltas[p.deltas_offset + ib * 2], + data_a_deltas[p.deltas_offset + ib * 2 + 1]); + } +#else buf_a[buf_ib].qs[iqs] = data_a_packed32[ib].qs[iqs]; if (iqs == 0) { buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib].dm); } #endif +#endif } void block_a_to_registers(const uint reg_ib, const uint buf_ib) { @@ -123,12 +132,20 @@ ACC_TYPE mmq_dot_product(const uint ib_a) { #if defined(DATA_A_Q8_0) // 2-byte loads for Q8_0 blocks (34 bytes) void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) { +#if defined(A_TYPE_REPACKED) + buf_a[buf_ib].qs[iqs] = int32_t(data_a_quants32[ib * 8 + iqs]); + + if (iqs == 0) { + buf_a[buf_ib].dm = FLOAT_TYPE(data_a_deltas[p.deltas_offset + ib]); + } +#else buf_a[buf_ib].qs[iqs] = pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2], data_a_packed16[ib].qs[iqs * 2 + 1])); if (iqs == 0) { buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d); } +#endif } void block_a_to_registers(const uint reg_ib, const uint buf_ib) { diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 48240bdda5..379c683da1 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -565,7 +565,7 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c } std::map mm_base_dict = base_dict; - if (tname == "q4_0" || tname == "mxfp4") { + if (tname == "q4_0" || tname == "q4_1" || tname == "q8_0" || tname == "iq4_nl" || tname == "mxfp4") { mm_base_dict["A_TYPE_REPACKED"] = "1"; } @@ -671,7 +671,7 @@ void process_shaders() { for (const auto& tname : type_names) { std::map mmv_base_dict = base_dict; - if (tname == "q4_0" || tname == "mxfp4") { + if (tname == "q4_0" || tname == "q4_1" || tname == "q8_0" || tname == "iq4_nl" || tname == "mxfp4") { mmv_base_dict["A_TYPE_REPACKED"] = "1"; }