upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-08-04 17:10:48 -05:00
Code Issues Packages Projects Releases Wiki Activity

add missing repacking functions

Browse Source
This commit is contained in:
Ruben Ortlam
2026-05-15 12:04:08 +02:00
parent b4e2621de8
commit b64f294cbf
1 changed files with 84 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
ggml/src/ggml-vulkan/ggml-vulkan.cpp
View File

@@ -13939,6 +13939,48 @@ static void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, g
return;
}
if (tensor->type == GGML_TYPE_Q4_0) {
const size_t repacked_size = ggml_vk_repack_q4_0_size_tensor(tensor);
const size_t deltas_offset = ggml_vk_repack_q4_0_delta_offset_tensor(tensor);
void * data_repacked = malloc(repacked_size);
uint8_t * quants = (uint8_t *)data_repacked;
ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
const block_q4_0 * src = (const block_q4_0 *)data;
for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
memcpy(quants + 16 * i, src[i].qs, 16);
deltas[i] = src[i].d;
}
ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
free(data_repacked);
return;
}
if (tensor->type == GGML_TYPE_MXFP4) {
const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor);
const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor);
void * data_repacked = malloc(repacked_size);
uint8_t * quants = (uint8_t *)data_repacked;
uint8_t * scales = (uint8_t *)data_repacked + scales_offset;
const block_mxfp4 * src = (const block_mxfp4 *)data;
for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
memcpy(quants + 16 * i, src[i].qs, 16);
scales[i] = src[i].e;
}
ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
free(data_repacked);
return;
}
ggml_vk_buffer_write_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_data, stride_tensor, size, n_copies);
}
@@ -14009,6 +14051,48 @@ static void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, c
vk_buffer buf = buf_ctx->dev_buffer;
if (tensor->type == GGML_TYPE_Q4_0) {
const size_t repacked_size = ggml_vk_repack_q4_0_size_tensor(tensor);
const size_t deltas_offset = ggml_vk_repack_q4_0_delta_offset_tensor(tensor);
void * data_repacked = malloc(repacked_size);
uint8_t * quants = (uint8_t *)data_repacked;
ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
block_q4_0 * dst = (block_q4_0 *)data;
for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
memcpy(dst[i].qs, quants + 16 * i, 16);
dst[i].d = deltas[i];
}
free(data_repacked);
return;
}
if (tensor->type == GGML_TYPE_MXFP4) {
const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor);
const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor);
void * data_repacked = malloc(repacked_size);
uint8_t * quants = (uint8_t *)data_repacked;
uint8_t * scales = (uint8_t *)data_repacked + scales_offset;
ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
block_mxfp4 * dst = (block_mxfp4 *)data;
for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
memcpy(dst[i].qs, quants + 16 * i, 16);
dst[i].e = scales[i];
}
free(data_repacked);
return;
}
ggml_vk_buffer_read_2d(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data, stride_tensor, stride_data, size, n_copies);
}