add q4_1, q8_0, iq4_nl repacking

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -7339,10 +7339,67 @@ static size_t ggml_vk_repack_mxfp4_size_tensor(const ggml_tensor * tensor) {
     return ggml_vk_repack_mxfp4_size(ggml_vk_get_num_blocks(tensor));
 }
 
+static size_t ggml_vk_repack_q4_1_delta_offset(size_t n_blocks) {
+    return GGML_PAD(n_blocks * 16, VULKAN_REPACK_ALIGNMENT);
+}
+
+static size_t ggml_vk_repack_q4_1_size(size_t n_blocks) {
+    return ggml_vk_repack_q4_1_delta_offset(n_blocks) + n_blocks * 4;
+}
+
+static size_t ggml_vk_repack_q4_1_delta_offset_tensor(const ggml_tensor * tensor) {
+    return ggml_vk_repack_q4_1_delta_offset(ggml_vk_get_num_blocks(tensor));
+}
+
+static size_t ggml_vk_repack_q4_1_size_tensor(const ggml_tensor * tensor) {
+    return ggml_vk_repack_q4_1_size(ggml_vk_get_num_blocks(tensor));
+}
+
+static size_t ggml_vk_repack_q8_0_delta_offset(size_t n_blocks) {
+    return GGML_PAD(n_blocks * 32, VULKAN_REPACK_ALIGNMENT);
+}
+
+static size_t ggml_vk_repack_q8_0_size(size_t n_blocks) {
+    return ggml_vk_repack_q8_0_delta_offset(n_blocks) + n_blocks * 2;
+}
+
+static size_t ggml_vk_repack_q8_0_delta_offset_tensor(const ggml_tensor * tensor) {
+    return ggml_vk_repack_q8_0_delta_offset(ggml_vk_get_num_blocks(tensor));
+}
+
+static size_t ggml_vk_repack_q8_0_size_tensor(const ggml_tensor * tensor) {
+    return ggml_vk_repack_q8_0_size(ggml_vk_get_num_blocks(tensor));
+}
+
+static size_t ggml_vk_repack_iq4_nl_delta_offset(size_t n_blocks) {
+    return GGML_PAD(n_blocks * 16, VULKAN_REPACK_ALIGNMENT);
+}
+
+static size_t ggml_vk_repack_iq4_nl_size(size_t n_blocks) {
+    return ggml_vk_repack_iq4_nl_delta_offset(n_blocks) + n_blocks * 2;
+}
+
+static size_t ggml_vk_repack_iq4_nl_delta_offset_tensor(const ggml_tensor * tensor) {
+    return ggml_vk_repack_iq4_nl_delta_offset(ggml_vk_get_num_blocks(tensor));
+}
+
+static size_t ggml_vk_repack_iq4_nl_size_tensor(const ggml_tensor * tensor) {
+    return ggml_vk_repack_iq4_nl_size(ggml_vk_get_num_blocks(tensor));
+}
+
 static size_t ggml_vk_repack_size_tensor(const ggml_tensor * tensor) {
     if (tensor->type == GGML_TYPE_Q4_0) {
         return ggml_vk_repack_q4_0_size_tensor(tensor);
     }
+    if (tensor->type == GGML_TYPE_Q4_1) {
+        return ggml_vk_repack_q4_1_size_tensor(tensor);
+    }
+    if (tensor->type == GGML_TYPE_Q8_0) {
+        return ggml_vk_repack_q8_0_size_tensor(tensor);
+    }
+    if (tensor->type == GGML_TYPE_IQ4_NL) {
+        return ggml_vk_repack_iq4_nl_size_tensor(tensor);
+    }
     if (tensor->type == GGML_TYPE_MXFP4) {
         return ggml_vk_repack_mxfp4_size_tensor(tensor);
     }
@@ -7993,6 +8050,12 @@ static void ggml_vk_mul_mat_q_f16(ggml_backend_vk_context * ctx, vk_context& sub
     uint32_t deltas_offset = 0;
     if (src0->type == GGML_TYPE_Q4_0) {
         deltas_offset = ggml_vk_repack_q4_0_delta_offset_tensor(src0) / 2;
+    } else if (src0->type == GGML_TYPE_Q4_1) {
+        deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(src0) / 2;
+    } else if (src0->type == GGML_TYPE_Q8_0) {
+        deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(src0) / 2;
+    } else if (src0->type == GGML_TYPE_IQ4_NL) {
+        deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(src0) / 2;
     } else if (src0->type == GGML_TYPE_MXFP4) {
         deltas_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(src0);
     }
@@ -8300,6 +8363,12 @@ static void ggml_vk_mul_mat_vec_q_f16(ggml_backend_vk_context * ctx, vk_context&
     uint32_t deltas_offset = 0;
     if (src0->type == GGML_TYPE_Q4_0) {
         deltas_offset = ggml_vk_repack_q4_0_delta_offset_tensor(src0) / 2;
+    } else if (src0->type == GGML_TYPE_Q4_1) {
+        deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(src0) / 2;
+    } else if (src0->type == GGML_TYPE_Q8_0) {
+        deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(src0) / 2;
+    } else if (src0->type == GGML_TYPE_IQ4_NL) {
+        deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(src0) / 2;
     } else if (src0->type == GGML_TYPE_MXFP4) {
         deltas_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(src0);
     }
@@ -8861,6 +8930,12 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
     uint32_t deltas_offset = 0;
     if (src0->type == GGML_TYPE_Q4_0) {
         deltas_offset = ggml_vk_repack_q4_0_delta_offset_tensor(src0) / 2;
+    } else if (src0->type == GGML_TYPE_Q4_1) {
+        deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(src0) / 2;
+    } else if (src0->type == GGML_TYPE_Q8_0) {
+        deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(src0) / 2;
+    } else if (src0->type == GGML_TYPE_IQ4_NL) {
+        deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(src0) / 2;
     } else if (src0->type == GGML_TYPE_MXFP4) {
         deltas_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(src0);
     }
@@ -9091,6 +9166,12 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte
     uint32_t deltas_offset = 0;
     if (src0->type == GGML_TYPE_Q4_0) {
         deltas_offset = ggml_vk_repack_q4_0_delta_offset_tensor(src0) / 2;
+    } else if (src0->type == GGML_TYPE_Q4_1) {
+        deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(src0) / 2;
+    } else if (src0->type == GGML_TYPE_Q8_0) {
+        deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(src0) / 2;
+    } else if (src0->type == GGML_TYPE_IQ4_NL) {
+        deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(src0) / 2;
     } else if (src0->type == GGML_TYPE_MXFP4) {
         deltas_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(src0);
     }
@@ -13911,6 +13992,66 @@ static void ggml_backend_vk_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml
         return;
     }
 
+    if (tensor->type == GGML_TYPE_Q4_1) {
+        const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        const block_q4_1 * src = (const block_q4_1 *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(quants + 16 * i, src[i].qs, 16);
+            memcpy(&deltas[i * 2], &src[i].data.dm, sizeof(ggml_half2));
+        }
+
+        ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        return;
+    }
+
+    if (tensor->type == GGML_TYPE_Q8_0) {
+        const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        const block_q8_0 * src = (const block_q8_0 *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(quants + 32 * i, src[i].qs, 32);
+            deltas[i] = src[i].d;
+        }
+
+        ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        return;
+    }
+
+    if (tensor->type == GGML_TYPE_IQ4_NL) {
+        const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        const block_iq4_nl * src = (const block_iq4_nl *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(quants + 16 * i, src[i].qs, 16);
+            deltas[i] = src[i].d;
+        }
+
+        ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        return;
+    }
+
     if (tensor->type == GGML_TYPE_MXFP4) {
         const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor);
         const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor);
@@ -13965,6 +14106,66 @@ static void ggml_backend_vk_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, g
         return;
     }
 
+    if (tensor->type == GGML_TYPE_Q4_1) {
+        const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        const block_q4_1 * src = (const block_q4_1 *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(quants + 16 * i, src[i].qs, 16);
+            memcpy(&deltas[i * 2], &src[i].data.dm, sizeof(ggml_half2));
+        }
+
+        ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        return;
+    }
+
+    if (tensor->type == GGML_TYPE_Q8_0) {
+        const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        const block_q8_0 * src = (const block_q8_0 *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(quants + 32 * i, src[i].qs, 32);
+            deltas[i] = src[i].d;
+        }
+
+        ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        return;
+    }
+
+    if (tensor->type == GGML_TYPE_IQ4_NL) {
+        const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        const block_iq4_nl * src = (const block_iq4_nl *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(quants + 16 * i, src[i].qs, 16);
+            deltas[i] = src[i].d;
+        }
+
+        ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        return;
+    }
+
     if (tensor->type == GGML_TYPE_MXFP4) {
         const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor);
         const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor);
@@ -14018,6 +14219,66 @@ static void ggml_backend_vk_buffer_get_tensor(ggml_backend_buffer_t buffer, cons
         return;
     }
 
+    if (tensor->type == GGML_TYPE_Q4_1) {
+        const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        block_q4_1 * dst = (block_q4_1 *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(dst[i].qs, quants + 16 * i, 16);
+            memcpy(&dst[i].data.dm, &deltas[i * 2], sizeof(ggml_half2));
+        }
+
+        return;
+    }
+
+    if (tensor->type == GGML_TYPE_Q8_0) {
+        const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        block_q8_0 * dst = (block_q8_0 *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(dst[i].qs, quants + 32 * i, 32);
+            dst[i].d = deltas[i];
+        }
+
+        return;
+    }
+
+    if (tensor->type == GGML_TYPE_IQ4_NL) {
+        const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        block_iq4_nl * dst = (block_iq4_nl *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(dst[i].qs, quants + 16 * i, 16);
+            dst[i].d = deltas[i];
+        }
+
+        return;
+    }
+
     if (tensor->type == GGML_TYPE_MXFP4) {
         const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor);
         const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor);
@@ -14073,6 +14334,66 @@ static void ggml_backend_vk_buffer_get_tensor_2d(ggml_backend_buffer_t buffer, c
         return;
     }
 
+    if (tensor->type == GGML_TYPE_Q4_1) {
+        const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        block_q4_1 * dst = (block_q4_1 *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(dst[i].qs, quants + 16 * i, 16);
+            memcpy(&dst[i].data.dm, &deltas[i * 2], sizeof(ggml_half2));
+        }
+
+        return;
+    }
+
+    if (tensor->type == GGML_TYPE_Q8_0) {
+        const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        block_q8_0 * dst = (block_q8_0 *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(dst[i].qs, quants + 32 * i, 32);
+            dst[i].d = deltas[i];
+        }
+
+        return;
+    }
+
+    if (tensor->type == GGML_TYPE_IQ4_NL) {
+        const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        block_iq4_nl * dst = (block_iq4_nl *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(dst[i].qs, quants + 16 * i, 16);
+            dst[i].d = deltas[i];
+        }
+
+        return;
+    }
+
     if (tensor->type == GGML_TYPE_MXFP4) {
         const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor);
         const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor);
@@ -14177,6 +14498,24 @@ static size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_typ
         return ggml_vk_repack_q4_0_size(num_blocks_per_row * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]);
     }
 
+    if (tensor->type == GGML_TYPE_Q4_1) {
+        const size_t num_blocks_per_row = tensor->ne[0] / ggml_blck_size(tensor->type);
+
+        return ggml_vk_repack_q4_1_size(num_blocks_per_row * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]);
+    }
+
+    if (tensor->type == GGML_TYPE_Q8_0) {
+        const size_t num_blocks_per_row = tensor->ne[0] / ggml_blck_size(tensor->type);
+
+        return ggml_vk_repack_q8_0_size(num_blocks_per_row * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]);
+    }
+
+    if (tensor->type == GGML_TYPE_IQ4_NL) {
+        const size_t num_blocks_per_row = tensor->ne[0] / ggml_blck_size(tensor->type);
+
+        return ggml_vk_repack_iq4_nl_size(num_blocks_per_row * tensor->ne[1] * tensor->ne[2] * tensor->ne[3]);
+    }
+
     if (tensor->type == GGML_TYPE_MXFP4) {
         const size_t num_blocks_per_row = tensor->ne[0] / ggml_blck_size(tensor->type);
 
@@ -14332,6 +14671,66 @@ static void ggml_backend_vk_set_tensor_2d_async(ggml_backend_t backend, ggml_ten
         return;
     }
 
+    if (tensor->type == GGML_TYPE_Q4_1) {
+        const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        const block_q4_1 * src = (const block_q4_1 *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(quants + 16 * i, src[i].qs, 16);
+            memcpy(&deltas[i * 2], &src[i].data.dm, sizeof(ggml_half2));
+        }
+
+        ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        return;
+    }
+
+    if (tensor->type == GGML_TYPE_Q8_0) {
+        const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        const block_q8_0 * src = (const block_q8_0 *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(quants + 32 * i, src[i].qs, 32);
+            deltas[i] = src[i].d;
+        }
+
+        ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        return;
+    }
+
+    if (tensor->type == GGML_TYPE_IQ4_NL) {
+        const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        const block_iq4_nl * src = (const block_iq4_nl *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(quants + 16 * i, src[i].qs, 16);
+            deltas[i] = src[i].d;
+        }
+
+        ggml_vk_buffer_write(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        return;
+    }
+
     if (tensor->type == GGML_TYPE_MXFP4) {
         const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor);
         const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor);
@@ -14440,6 +14839,66 @@ static void ggml_backend_vk_get_tensor_2d_async(ggml_backend_t backend, const gg
         return;
     }
 
+    if (tensor->type == GGML_TYPE_Q4_1) {
+        const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        block_q4_1 * dst = (block_q4_1 *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(dst[i].qs, quants + 16 * i, 16);
+            memcpy(&dst[i].data.dm, &deltas[i * 2], sizeof(ggml_half2));
+        }
+
+        return;
+    }
+
+    if (tensor->type == GGML_TYPE_Q8_0) {
+        const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        block_q8_0 * dst = (block_q8_0 *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(dst[i].qs, quants + 32 * i, 32);
+            dst[i].d = deltas[i];
+        }
+
+        return;
+    }
+
+    if (tensor->type == GGML_TYPE_IQ4_NL) {
+        const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(tensor);
+        const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(tensor);
+
+        void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+        uint8_t * quants = (uint8_t *)data_repacked;
+        ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+        ggml_vk_buffer_read(buf, vk_tensor_offset(tensor) + tensor->view_offs + offset, data_repacked, repacked_size);
+
+        block_iq4_nl * dst = (block_iq4_nl *)data;
+
+        for (size_t i = 0; i < ggml_vk_get_num_blocks(tensor); i++) {
+            memcpy(dst[i].qs, quants + 16 * i, 16);
+            dst[i].d = deltas[i];
+        }
+
+        return;
+    }
+
     if (tensor->type == GGML_TYPE_MXFP4) {
         const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(tensor);
         const size_t scales_offset = ggml_vk_repack_mxfp4_scale_offset_tensor(tensor);
@@ -17001,6 +17460,61 @@ static void ggml_vk_check_results_0(ggml_backend_vk_context * ctx, ggml_cgraph *
                         dst[i].d = deltas[i];
                     }
 
+                    memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS);
+                } else if (srci->type == GGML_TYPE_Q4_1) {
+                    const size_t repacked_size = ggml_vk_repack_q4_1_size_tensor(srci);
+                    const size_t deltas_offset = ggml_vk_repack_q4_1_delta_offset_tensor(srci);
+
+                    void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+                    uint8_t * quants = (uint8_t *)data_repacked;
+                    ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+                    ggml_vk_buffer_read(buffer_gpu, offset, data_repacked, repacked_size);
+
+                    block_q4_1 * dst = (block_q4_1 *)srci_clone->data;
+
+                    for (size_t i = 0; i < ggml_vk_get_num_blocks(srci); i++) {
+                        memcpy(dst[i].qs, quants + 16 * i, 16);
+                        dst[i].d = deltas[i * 2];
+                        dst[i].m = deltas[i * 2 + 1];
+                    }
+
+                    memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS);
+                } else if (srci->type == GGML_TYPE_Q8_0) {
+                    const size_t repacked_size = ggml_vk_repack_q8_0_size_tensor(srci);
+                    const size_t deltas_offset = ggml_vk_repack_q8_0_delta_offset_tensor(srci);
+
+                    void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+                    uint8_t * quants = (uint8_t *)data_repacked;
+                    ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+                    ggml_vk_buffer_read(buffer_gpu, offset, data_repacked, repacked_size);
+
+                    block_q8_0 * dst = (block_q8_0 *)srci_clone->data;
+
+                    for (size_t i = 0; i < ggml_vk_get_num_blocks(srci); i++) {
+                        memcpy(dst[i].qs, quants + 32 * i, 32);
+                        dst[i].d = deltas[i];
+                    }
+
+                    memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS);
+                } else if (srci->type == GGML_TYPE_IQ4_NL) {
+                    const size_t repacked_size = ggml_vk_repack_iq4_nl_size_tensor(srci);
+                    const size_t deltas_offset = ggml_vk_repack_iq4_nl_delta_offset_tensor(srci);
+
+                    void * data_repacked = ggml_vk_repack_scratch(repacked_size);
+                    uint8_t * quants = (uint8_t *)data_repacked;
+                    ggml_fp16_t * deltas = (ggml_fp16_t *)((uint8_t *)data_repacked + deltas_offset);
+
+                    ggml_vk_buffer_read(buffer_gpu, offset, data_repacked, repacked_size);
+
+                    block_iq4_nl * dst = (block_iq4_nl *)srci_clone->data;
+
+                    for (size_t i = 0; i < ggml_vk_get_num_blocks(srci); i++) {
+                        memcpy(dst[i].qs, quants + 16 * i, 16);
+                        dst[i].d = deltas[i];
+                    }
+
                     memcpy(srci_clone->nb, srci->nb, sizeof(size_t) * GGML_MAX_DIMS);
                 } else if (srci->type == GGML_TYPE_MXFP4) {
                     const size_t repacked_size = ggml_vk_repack_mxfp4_size_tensor(srci);

ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl

@@ -45,6 +45,16 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
 #endif
 
 #if defined(DATA_A_Q4_1)
+#if defined(A_TYPE_REPACKED)
+vec2 dequantize(uint ib, uint iqs, uint a_offset) {
+    const uint vui = uint(data_a_quants[(a_offset + ib) * 16 + iqs]);
+    return vec2(vui & 0xF, vui >> 4);
+}
+vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
+    const uint vui = uint(data_a_quants16[(a_offset + ib) * 8 + iqs/2]);
+    return vec4(vui & 0xF, (vui >> 4) & 0xF, (vui >> 8) & 0xF, vui >> 12);
+}
+#else
 vec2 dequantize(uint ib, uint iqs, uint a_offset) {
     const uint vui = uint(data_a[a_offset + ib].qs[iqs]);
     return vec2(vui & 0xF, vui >> 4);
@@ -54,6 +64,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
     return vec4(vui & 0xF, (vui >> 4) & 0xF, (vui >> 8) & 0xF, vui >> 12);
 }
 #endif
+#endif
 
 #if defined(DATA_A_Q5_0)
 vec2 dequantize(uint ib, uint iqs, uint a_offset) {
@@ -88,6 +99,17 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
 #endif
 
 #if defined(DATA_A_Q8_0)
+#if defined(A_TYPE_REPACKED)
+vec2 dequantize(uint ib, uint iqs, uint a_offset) {
+    return vec2(int(int8_t(data_a_quants[(a_offset + ib) * 32 + iqs])),
+                int(int8_t(data_a_quants[(a_offset + ib) * 32 + iqs + 1])));
+}
+vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
+    const i8vec2 v0 = unpack8(int32_t(data_a_quants16[(a_offset + ib) * 16 + iqs/2])).xy;
+    const i8vec2 v1 = unpack8(int32_t(data_a_quants16[(a_offset + ib) * 16 + iqs/2 + 1])).xy;
+    return vec4(v0.x, v0.y, v1.x, v1.y);
+}
+#else
 vec2 dequantize(uint ib, uint iqs, uint a_offset) {
     return vec2(int(data_a[a_offset + ib].qs[iqs]), int(data_a[a_offset + ib].qs[iqs + 1]));
 }
@@ -97,6 +119,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
     return vec4(v0.x, v0.y, v1.x, v1.y);
 }
 #endif
+#endif
 
 #if defined(DATA_A_Q1_0)
 vec2 dequantize(uint ib, uint iqs, uint a_offset) {
@@ -439,6 +462,16 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
 #endif
 
 #if defined(DATA_A_IQ4_NL)
+#if defined(A_TYPE_REPACKED)
+vec2 dequantize(uint ib, uint iqs, uint a_offset) {
+    const uint vui = uint(data_a_quants[(a_offset + ib) * 16 + iqs]);
+    return vec2(kvalues_iq4nl[vui & 0xF], kvalues_iq4nl[vui >> 4]);
+}
+vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
+    const uint vui = uint(data_a_quants16[(a_offset + ib) * 8 + iqs/2]);
+    return vec4(kvalues_iq4nl[vui & 0xF], kvalues_iq4nl[(vui >> 4) & 0xF], kvalues_iq4nl[(vui >> 8) & 0xF], kvalues_iq4nl[vui >> 12]);
+}
+#else
 vec2 dequantize(uint ib, uint iqs, uint a_offset) {
     const uint vui = uint(data_a[a_offset + ib].qs[iqs]);
     return vec2(kvalues_iq4nl[vui & 0xF], kvalues_iq4nl[vui >> 4]);
@@ -448,6 +481,7 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) {
     return vec4(kvalues_iq4nl[vui & 0xF], kvalues_iq4nl[(vui >> 4) & 0xF], kvalues_iq4nl[(vui >> 8) & 0xF], kvalues_iq4nl[vui >> 12]);
 }
 #endif
+#endif
 
 #if defined(DATA_A_MXFP4)
 #if defined(A_TYPE_REPACKED)
@@ -509,7 +543,7 @@ vec2 get_dm(uint ib, uint a_offset) {
 
 #if defined(DATA_A_Q4_0) || defined(DATA_A_Q5_0) || defined(DATA_A_Q8_0) || defined(DATA_A_IQ1_S) || defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) || defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) || defined(DATA_A_IQ4_XS) || defined(DATA_A_IQ4_NL)
 vec2 get_dm(uint ib, uint a_offset) {
-#if defined(DATA_A_Q4_0) && defined(A_TYPE_REPACKED)
+#if (defined(DATA_A_Q4_0) || defined(DATA_A_Q8_0) || defined(DATA_A_IQ4_NL)) && defined(A_TYPE_REPACKED)
     return vec2(float(data_a_deltas[a_offset + p.deltas_offset + ib]), 0);
 #else
     return vec2(float(data_a[a_offset + ib].d), 0);
@@ -542,8 +576,13 @@ vec2 get_dm(uint ib, uint a_offset) {
 
 #if defined(DATA_A_Q4_1) || defined(DATA_A_Q5_1)
 vec2 get_dm(uint ib, uint a_offset) {
+#if defined(DATA_A_Q4_1) && defined(A_TYPE_REPACKED)
+    return vec2(float(data_a_deltas[p.deltas_offset + (a_offset + ib) * 2]),
+                float(data_a_deltas[p.deltas_offset + (a_offset + ib) * 2 + 1]));
+#else
     const vec2 dm = vec2(data_a_packed32[a_offset + ib].dm);
     return dm;
+#endif
 }
 #endif
 

ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs_cm2.glsl

@@ -54,19 +54,36 @@ float16_t dequantFuncQ4_0(const in decodeBufQ4_0 bl, const in uint blockCoords[2
     return ret;
 }
 
+#ifdef A_TYPE_REPACKED
+layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ4_1 {
+   uint16_t qs[8];
+};
+#else
 layout(buffer_reference, std430, buffer_reference_align = 4) buffer decodeBufQ4_1 {
    block_q4_1 block;
 };
+#endif
 
 float16_t dequantFuncQ4_1(const in decodeBufQ4_1 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
 {
+    const uint idx = coordInBlock[1];
+#ifdef A_TYPE_REPACKED
+    const uint ib = pos_a + blockCoords[0] * (p.stride_a / QUANT_K) + blockCoords[1];
+    const float16_t d = data_a_deltas[p.deltas_offset + ib * 2];
+    const float16_t m = data_a_deltas[p.deltas_offset + ib * 2 + 1];
+    uint32_t qs = uint32_t(bl.qs[(idx & 0xE) >> 1]);
+#else
     const float16_t d = bl.block.d;
     const float16_t m = bl.block.m;
-    const uint idx = coordInBlock[1];
+    uint32_t qs = bl.block.qs[idx & 0xF];
+#endif
     const uint iqs = idx & 0xF;
     const uint shift = (idx & 0x10) >> 2;
-    uint32_t qs = bl.block.qs[iqs];
+#ifdef A_TYPE_REPACKED
+    qs >>= ((iqs & 1) * 8 + shift);
+#else
     qs >>= shift;
+#endif
     qs &= 0xF;
     float16_t ret = float16_t(qs) * d + m;
     return ret;
@@ -117,18 +134,28 @@ float16_t dequantFuncQ5_1(const in decodeBufQ5_1 bl, const in uint blockCoords[2
     return ret;
 }
 
+#ifdef A_TYPE_REPACKED
+layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufQ8_0 {
+   int16_t qs[16];
+};
+#else
 layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufQ8_0 {
    block_q8_0_packed16 block;
 };
+#endif
 
 float16_t dequantFuncQ8_0(const in decodeBufQ8_0 bl, const in uint blockCoords[2], const in uint coordInBlock[2])
 {
-    const float16_t d = bl.block.d;
     const uint idx = coordInBlock[1];
     const uint iqs = idx;
-
-    // Load 16b and select the byte for this element
+#ifdef A_TYPE_REPACKED
+    const uint ib = pos_a + blockCoords[0] * (p.stride_a / QUANT_K) + blockCoords[1];
+    const float16_t d = data_a_deltas[p.deltas_offset + ib];
+    int32_t qs = unpack8(bl.qs[(iqs & 0x1E) >> 1])[iqs & 1];
+#else
+    const float16_t d = bl.block.d;
     int32_t qs = unpack8(bl.block.qs[(iqs & 0x1E) >> 1])[iqs & 1];
+#endif
     float16_t ret = float16_t(qs) * d;
     return ret;
 }
@@ -672,18 +699,32 @@ float16_t dequantFuncIQ4_XS(const in decodeBufIQ4_XS bl, const in uint blockCoor
 #endif
 
 #if defined(DATA_A_IQ4_NL)
+#ifdef A_TYPE_REPACKED
+layout(buffer_reference, std430, buffer_reference_align = 16) buffer decodeBufIQ4_NL {
+   uint16_t qs[8];
+};
+#else
 layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufIQ4_NL {
    block_iq4_nl block;
 };
+#endif
 
 float16_t dequantFuncIQ4_NL(const in decodeBufIQ4_NL bl, const in uint blockCoords[2], const in uint coordInBlock[2])
 {
-    const float16_t d = bl.block.d;
     const uint idx = coordInBlock[1];
+#ifdef A_TYPE_REPACKED
+    const uint ib = pos_a + blockCoords[0] * (p.stride_a / QUANT_K) + blockCoords[1];
+    const float16_t d = data_a_deltas[p.deltas_offset + ib];
+    uint32_t qs = uint32_t(bl.qs[(idx & 0xE) >> 1]);
+    const uint shift = (idx & 0x10) >> 2;
+    qs >>= ((idx & 1) * 8 + shift);
+#else
+    const float16_t d = bl.block.d;
     const uint iqs = idx & 0xF;
     const uint shift = (idx & 0x10) >> 2;
     uint32_t qs = bl.block.qs[iqs];
     qs >>= shift;
+#endif
     qs &= 0xF;
     float16_t ret = float16_t(kvalues_iq4nl[qs]) * d;
     return ret;

ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl

@@ -6,7 +6,7 @@
 
 #if defined(DATA_A_Q4_0) || defined(DATA_A_Q5_0) || defined(DATA_A_Q8_0) || defined(DATA_A_IQ1_S) || defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) || defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) || defined(DATA_A_IQ4_XS) || defined(DATA_A_IQ4_NL)
 FLOAT_TYPE get_dm(uint ib) {
-#if defined(DATA_A_Q4_0) && defined(A_TYPE_REPACKED)
+#if (defined(DATA_A_Q4_0) || defined(DATA_A_Q8_0) || defined(DATA_A_IQ4_NL)) && defined(A_TYPE_REPACKED)
     return FLOAT_TYPE(data_a_deltas[p.deltas_offset + ib]);
 #else
     return FLOAT_TYPE(data_a[ib].d);
@@ -16,7 +16,12 @@ FLOAT_TYPE get_dm(uint ib) {
 
 #if defined(DATA_A_Q4_1) || defined(DATA_A_Q5_1)
 FLOAT_TYPEV2 get_dm(uint ib) {
+#if defined(DATA_A_Q4_1) && defined(A_TYPE_REPACKED)
+    return FLOAT_TYPEV2(data_a_deltas[p.deltas_offset + ib * 2],
+                        data_a_deltas[p.deltas_offset + ib * 2 + 1]);
+#else
     return FLOAT_TYPEV2(data_a_packed32[ib].dm);
+#endif
 }
 #endif
 
@@ -60,7 +65,11 @@ FLOAT_TYPE mul_q8_1(const int32_t q_sum, const float da, const vec2 dsb, const i
 #if defined(DATA_A_Q4_1)
 // 4-byte loads for Q4_1 blocks (20 bytes)
 i32vec2 repack(uint ib, uint iqs) {
+#if defined(A_TYPE_REPACKED)
+    const uint32_t vui = data_a_quants32[ib * 4 + iqs];
+#else
     const uint32_t vui = data_a_packed32[ib].qs[iqs];
+#endif
     return i32vec2( vui       & 0x0F0F0F0F,
                    (vui >> 4) & 0x0F0F0F0F);
 }
@@ -115,8 +124,12 @@ FLOAT_TYPE mul_q8_1(const int32_t q_sum, const vec2 dma, const vec2 dsb, const i
 #if defined(DATA_A_Q8_0)
 // 2-byte loads for Q8_0 blocks (34 bytes)
 int32_t repack(uint ib, uint iqs) {
+#if defined(A_TYPE_REPACKED)
+    return int32_t(data_a_quants32[ib * 8 + iqs]);
+#else
     return pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2    ],
                           data_a_packed16[ib].qs[iqs * 2 + 1]));
+#endif
 }
 
 FLOAT_TYPE mul_q8_1(const int32_t q_sum, const float da, const vec2 dsb, const int32_t sum_divisor) {

ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp

@@ -68,7 +68,11 @@ layout (push_constant) uniform parameter
 
 
 #ifdef A_TYPE_REPACKED
+#if defined(DATA_A_Q8_0)
+struct block_repacked_quants { uint16_t qs[16]; };
+#else
 struct block_repacked_quants { uint16_t qs[8]; };
+#endif
 layout (binding = 0) readonly buffer A {block_repacked_quants data_a[];};
 layout (binding = 0) readonly buffer A_DELTAS {float16_t data_a_deltas[];};
 layout (binding = 0) readonly buffer A_SCALES {uint8_t data_a_scales[];};

ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl

@@ -73,8 +73,14 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin
             const uint ib = idx / 4;
             const uint iqs = idx & 0x03;
 
+#if defined(A_TYPE_REPACKED)
+            const vec2 dm = vec2(data_a_deltas[p.deltas_offset + ib * 2],
+                                data_a_deltas[p.deltas_offset + ib * 2 + 1]);
+            const uint vui = data_a_quants32[ib * 4 + iqs];
+#else
             const vec2 dm = vec2(data_a_packed32[ib].dm);
             const uint vui = data_a_packed32[ib].qs[iqs];
+#endif
             const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y;
             const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y;
 
@@ -128,9 +134,15 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin
             const uint ib = idx / 8;
             const uint iqs = idx & 0x07;
 
+#if defined(A_TYPE_REPACKED)
+            const float d = float(data_a_deltas[p.deltas_offset + ib]);
+            const i8vec2 v0 = unpack8(int32_t(data_a_quants16[ib * 16 + 2*iqs])).xy;
+            const i8vec2 v1 = unpack8(int32_t(data_a_quants16[ib * 16 + 2*iqs + 1])).xy;
+#else
             const float d = float(data_a_packed16[ib].d);
             const i8vec2 v0 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs])).xy; // vec4 used due to #12147
             const i8vec2 v1 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs + 1])).xy;
+#endif
             const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d;
 
             buf_a[buf_idx    ] = FLOAT_TYPEV2(v.xy);
@@ -486,8 +498,13 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin
             const uint ib = idx / 8;
             const uint iqs = idx & 0x07;
 
+#if defined(A_TYPE_REPACKED)
+            const FLOAT_TYPE d = FLOAT_TYPE(data_a_deltas[p.deltas_offset + ib]);
+            const uint vui = uint(data_a_quants16[ib * 8 + iqs]);
+#else
             const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d);
             const uint vui = uint(data_a_packed16[ib].qs[iqs]);
+#endif
 
             buf_a[buf_idx    ] = d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF],
                                                   kvalues_iq4nl[bitfieldExtract(vui, 8, 4)]);

ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl

@@ -26,12 +26,21 @@ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
 #endif
     }
 #else // DATA_A_Q4_1
+#if defined(A_TYPE_REPACKED)
+    buf_a[buf_ib].qs[iqs] = data_a_quants32[ib * 4 + iqs];
+
+    if (iqs == 0) {
+        buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_deltas[p.deltas_offset + ib * 2],
+                                         data_a_deltas[p.deltas_offset + ib * 2 + 1]);
+    }
+#else
     buf_a[buf_ib].qs[iqs] = data_a_packed32[ib].qs[iqs];
 
     if (iqs == 0) {
         buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib].dm);
     }
 #endif
+#endif
 }
 
 void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
@@ -123,12 +132,20 @@ ACC_TYPE mmq_dot_product(const uint ib_a) {
 #if defined(DATA_A_Q8_0)
 // 2-byte loads for Q8_0 blocks (34 bytes)
 void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
+#if defined(A_TYPE_REPACKED)
+    buf_a[buf_ib].qs[iqs] = int32_t(data_a_quants32[ib * 8 + iqs]);
+
+    if (iqs == 0) {
+        buf_a[buf_ib].dm = FLOAT_TYPE(data_a_deltas[p.deltas_offset + ib]);
+    }
+#else
     buf_a[buf_ib].qs[iqs] = pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2],
                                            data_a_packed16[ib].qs[iqs * 2 + 1]));
 
     if (iqs == 0) {
         buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d);
     }
+#endif
 }
 
 void block_a_to_registers(const uint reg_ib, const uint buf_ib) {

ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp

@@ -565,7 +565,7 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c
         }
 
         std::map<std::string, std::string> mm_base_dict = base_dict;
-        if (tname == "q4_0" || tname == "mxfp4") {
+        if (tname == "q4_0" || tname == "q4_1" || tname == "q8_0" || tname == "iq4_nl" || tname == "mxfp4") {
             mm_base_dict["A_TYPE_REPACKED"] = "1";
         }
 
@@ -671,7 +671,7 @@ void process_shaders() {
 
     for (const auto& tname : type_names) {
         std::map<std::string, std::string> mmv_base_dict = base_dict;
-        if (tname == "q4_0" || tname == "mxfp4") {
+        if (tname == "q4_0" || tname == "q4_1" || tname == "q8_0" || tname == "iq4_nl" || tname == "mxfp4") {
             mmv_base_dict["A_TYPE_REPACKED"] = "1";
         }