Compare commits

...
5 Commits
Author SHA1 Message Date
Johannes Gäßler b94959a64d CUDA: implement support for duplicate expert IDs (#32505)
implement support for duplicate expert IDs
2026-11-19 23:21:24 +01:00
Imdadullah Mohammed 50518b64c6 model : hrm : do not compute slot count as uint32_t (#32601)
load_arch_hparams multiplies n_hrm_layers_per_stack, n_hrm_h_cycles and
(n_hrm_l_cycles + 1) into a uint32_t, then asserts the product equals
block_count.

Any factors whose true product is block_count + k*2^32 still pass the
assert; 1073741825 * 2 * 2 = 2^32 + 4 compares equal to block_count = 4.

load_arch_tensors iterates the same three values in int and writes past
the end of the block_count-sized layers vector.
2026-11-19 23:21:03 +01:00
c406f4367e opencl: support dp4a for KQ mm in decomposed FA (#32577)
* opencl: support int8 KQ for DK = 512

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
Assisted-by: Codex

* opencl: allow query chunk up to 96 for int8 KQ

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
Assisted-by: Codex

* opencl: extend `build_program_from_source` to allow non-fatal failure

Assisted-by: Codex

* opencl: add int8 KQ for decomposed FA with DK <= 256

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
Assisted-by: Codex

---------

Co-authored-by: Li He <lih@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
2026-11-19 11:30:19 -08:00
anujj 7fce5439fe ggml : add BF16 support to ROPE and SET_ROWS (CPU, CUDA) (#32387)
* ggml : add BF16 support to ROPE and SET_ROWS (CPU, CUDA)

* cuda : SET_ROWS from BF16 into quantized types and F8_E4M3

* cuda : fuse BF16 rope into a BF16 cache

* tests : trim the BF16 rope cases to one per code path
2026-11-19 23:43:55 +05:30
JamePeng 53ecea05a0 mtmd : fix YuE2 projector loading (#32606)
- Allow cls.output.weight to be absent from the mmproj. It is an optional
Qwen3 reranking head that the YuE2 NAR decoder never uses.

- Without this fix, mtmd_init_from_file() fails with
"tensor cls.output.weight is missing from the mmproj", preventing
audio generation from starting.
2026-11-19 19:11:11 +01:00
26 changed files with 1130 additions and 383 deletions
-6
View File
@@ -1316,12 +1316,6 @@ extern "C" {
// indirect matrix multiplication
// an id of -1 skips the slot and zeroes the matching row of the result
//
// [TAG_MUL_MAT_ID_DUPLICATE_ID]
// note: this op has undefined behaviour if a given `a` is referenced more than once for a given `b`
// (i.e. duplicate experts per token are not guaranteed to be supported in all backends)
// TODO: add backend support: https://github.com/ggml-org/llama.cpp/pull/29998#issuecomment-6153742533
// the CPU backend skips negative ids, their rows of the result are zero
GGML_API struct ggml_tensor * ggml_mul_mat_id(
struct ggml_context * ctx,
struct ggml_tensor * as,
+6
View File
@@ -2501,6 +2501,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
break;
case GGML_OP_SET_ROWS:
{
if (op->src[0]->type == GGML_TYPE_BF16) {
return false;
}
switch (op->type) {
case GGML_TYPE_F32:
case GGML_TYPE_F16:
@@ -2553,6 +2556,9 @@ static bool ggml_backend_cann_supports_op(ggml_backend_dev_t dev, const ggml_ten
if (((const int32_t *) op->op_params)[15] != 0) {
return false; // FIXME: support ggml_rope_set_offset
}
if (op->src[0]->type != GGML_TYPE_F32 && op->src[0]->type != GGML_TYPE_F16) {
return false;
}
if (op->src[0]->ne[0] > 896) {
return false;
}
+2 -1
View File
@@ -2933,7 +2933,8 @@ struct ggml_cplan ggml_graph_plan(
} break;
case GGML_OP_SET_ROWS:
{
if (node->src[0]->type == GGML_TYPE_F16 && node->type != GGML_TYPE_F16) {
if ((node->src[0]->type == GGML_TYPE_F16 && node->type != GGML_TYPE_F16) ||
(node->src[0]->type == GGML_TYPE_BF16 && node->type != GGML_TYPE_BF16)) {
cur = ggml_type_size(GGML_TYPE_F32) * node->src[0]->ne[0] * n_tasks;
}
} break;
+35 -3
View File
@@ -5531,7 +5531,7 @@ static void ggml_compute_forward_set_rows_impl(
assert(ne0 == nc);
assert(ne2 == ne02);
assert(ne3 == ne03);
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);
assert(ne02 % ne11 == 0);
assert(ne03 % ne12 == 0);
@@ -5578,6 +5578,20 @@ static void ggml_compute_forward_set_rows_impl(
from_float(wdata,
((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc);
}
} else if constexpr (std::is_same_v<src_t, ggml_bf16_t>) {
if (dst->type == GGML_TYPE_BF16) {
memcpy(
((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3),
((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03),
rs);
} else {
float * wdata = (float *) params->wdata + (nc + CACHE_LINE_SIZE_F32) * ith;
ggml_bf16_to_fp32_row(
(const ggml_bf16_t *) ((char *) src0->data + i*nb01 + i02*nb02 + i03*nb03),
wdata, nc);
from_float(wdata,
((char *) dst->data + i1*nb1 + i02*nb2 + i03*nb3), nc);
}
} else {
GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type));
}
@@ -5614,6 +5628,16 @@ void ggml_compute_forward_set_rows(
GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type));
}
} break;
case GGML_TYPE_BF16:
{
if (src1->type == GGML_TYPE_I64) {
ggml_compute_forward_set_rows_impl<ggml_bf16_t, int64_t>(params, dst);
} else if (src1->type == GGML_TYPE_I32) {
ggml_compute_forward_set_rows_impl<ggml_bf16_t, int32_t>(params, dst);
} else {
GGML_ABORT("src1->type = %d (%s) not supported", src1->type, ggml_type_name(src1->type));
}
} break;
default:
{
GGML_ABORT("src0->type = %d (%s) not supported", src0->type, ggml_type_name(src0->type));
@@ -6302,7 +6326,7 @@ static void rotate_pairs(const int64_t n, const int64_t n_offset, const float *
}
}
template<typename T> //float or ggml_fp16_t
template<typename T> //float, ggml_fp16_t or ggml_bf16_t
static void ggml_compute_forward_rope_flt(
const ggml_compute_params * params,
ggml_tensor * dst,
@@ -6312,7 +6336,7 @@ static void ggml_compute_forward_rope_flt(
const ggml_tensor * src1 = dst->src[1];
const ggml_tensor * src2 = dst->src[2];
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);
GGML_ASSERT(src1->type == GGML_TYPE_I32);
float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
@@ -6472,6 +6496,10 @@ void ggml_compute_forward_rope(
{
ggml_compute_forward_rope_flt<ggml_fp16_t>(params, dst, true);
} break;
case GGML_TYPE_BF16:
{
ggml_compute_forward_rope_flt<ggml_bf16_t>(params, dst, true);
} break;
case GGML_TYPE_F32:
{
ggml_compute_forward_rope_flt<float>(params, dst, true);
@@ -6496,6 +6524,10 @@ void ggml_compute_forward_rope_back(
{
ggml_compute_forward_rope_flt<ggml_fp16_t>(params, dst, false);
} break;
case GGML_TYPE_BF16:
{
ggml_compute_forward_rope_flt<ggml_bf16_t>(params, dst, false);
} break;
case GGML_TYPE_F32:
{
ggml_compute_forward_rope_flt<float>(params, dst, false);
+10
View File
@@ -452,6 +452,16 @@ static __device__ __forceinline__ ggml_cuda_thread_mask_t ggml_cuda_ballot(int p
return __ballot_sync(mask, predicate);
}
static __device__ __forceinline__ int ggml_cuda_popc(ggml_cuda_thread_mask_t x) {
static_assert(sizeof(ggml_cuda_thread_mask_t) == 4 || sizeof(ggml_cuda_thread_mask_t) == 8, "unsupported thread mask type");
if constexpr (sizeof(ggml_cuda_thread_mask_t) == 4) {
return __popc(x);
} else {
return __popcll(x);
}
}
template<typename T> static __device__ __forceinline__ int ggml_cuda_popc(T x) = delete; // Makes implicit type casts result in an error.
static __device__ __forceinline__ int ggml_cuda_all(int predicate, ggml_cuda_thread_mask_t mask = GGML_CUDA_FULL_THREAD_MASK) {
return __all_sync(mask, predicate);
}
+7 -16
View File
@@ -17,15 +17,6 @@ static __global__ void flash_attn_mask_to_sparse_indices(
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
constexpr int values_per_lane = 8;
using lane_mask_t = std::conditional_t<warp_size == 64, uint64_t, uint32_t>;
constexpr uint64_t ballot_mask = ~(lane_mask_t)0;
const auto popc = [](const lane_mask_t mask) {
if constexpr (warp_size == 64) {
return __popcll(mask);
} else {
return __popc(mask);
}
};
const int tid = threadIdx.x;
const int warp = tid / warp_size;
const int lane = tid % warp_size;
@@ -48,7 +39,7 @@ static __global__ void flash_attn_mask_to_sparse_indices(
__syncthreads();
for (int i0 = 0; i0 < ne30; i0 += blockDim.x*values_per_lane) {
lane_mask_t selected_warp[values_per_lane];
ggml_cuda_thread_mask_t selected_warp[values_per_lane];
int warp_count = 0;
#pragma unroll
for (int item = 0; item < values_per_lane; ++item) {
@@ -60,8 +51,8 @@ static __global__ void flash_attn_mask_to_sparse_indices(
selected |= (!oob || q < q1 - q0) && isfinite(__half2float(mask[q*s31 + i]));
}
}
selected_warp[item] = (lane_mask_t)ggml_cuda_ballot((int) selected, ballot_mask);
warp_count += popc(selected_warp[item]);
selected_warp[item] = ggml_cuda_ballot((int) selected);
warp_count += ggml_cuda_popc(selected_warp[item]);
}
if (lane == 0) {
@@ -81,16 +72,16 @@ static __global__ void flash_attn_mask_to_sparse_indices(
}
__syncthreads();
const lane_mask_t lane_mask = lane == 0 ? 0 : ((lane_mask_t)1 << lane) - 1;
const ggml_cuda_thread_mask_t lane_mask = lane == 0 ? 0 : ((ggml_cuda_thread_mask_t)1 << lane) - 1;
int warp_item_offset = 0;
#pragma unroll
for (int item = 0; item < values_per_lane; ++item) {
const int i = i0 + (warp*values_per_lane + item)*warp_size + lane;
const int dst = row_count + warp_offsets[warp] + warp_item_offset + popc(selected_warp[item] & lane_mask);
if ((selected_warp[item] & ((lane_mask_t)1 << lane)) && dst < n_kv_max) {
const int dst = row_count + warp_offsets[warp] + warp_item_offset + ggml_cuda_popc(selected_warp[item] & lane_mask);
if ((selected_warp[item] & ((ggml_cuda_thread_mask_t)1 << lane)) && dst < n_kv_max) {
indices[dst] = i;
}
warp_item_offset += popc(selected_warp[item]);
warp_item_offset += ggml_cuda_popc(selected_warp[item]);
}
__syncthreads();
+12 -4
View File
@@ -2730,7 +2730,8 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
ids_from_sorted_host[i12*n_expert_used + iex] = ids_to_sorted_host.size();
ids_to_sorted_host.push_back(i12*ne11 + iex % ne11);
tokens_per_expert[i02]++;
break;
// TODO: removing the break here was done to make duplicate IDs work but this caused a ~10% performance regression, better solution needed.
// break;
}
}
}
@@ -3504,7 +3505,12 @@ static bool ggml_cuda_should_fuse_rope_set_rows(const ggml_tensor * rope,
return false;
}
if (set_rows->type != GGML_TYPE_F32 && set_rows->type != GGML_TYPE_F16) {
if (set_rows->type != GGML_TYPE_F32 && set_rows->type != GGML_TYPE_F16 && set_rows->type != GGML_TYPE_BF16) {
return false;
}
// BF16 rope fuses into an F16 or BF16 cache, other ropes not into a BF16 cache
if (rope->type == GGML_TYPE_BF16 ? set_rows->type == GGML_TYPE_F32 : set_rows->type == GGML_TYPE_BF16) {
return false;
}
@@ -6510,7 +6516,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || op->type == GGML_TYPE_F8_E4M3 ||
op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 ||
op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) &&
op->src[0]->type == GGML_TYPE_F32
(op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_BF16)
) || (
op->type == GGML_TYPE_F16 && op->src[0]->type == GGML_TYPE_F16
)
@@ -6731,7 +6737,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
return false;
case GGML_OP_ROPE:
case GGML_OP_ROPE_BACK: {
return op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && ggml_is_contiguous_2(op->src[0]);
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 || op->src[0]->type == GGML_TYPE_BF16) &&
op->type == op->src[0]->type &&
op->src[0]->nb[0] == ggml_type_size(op->src[0]->type) && ggml_is_contiguous_2(op->src[0]);
}
case GGML_OP_IM2COL:
case GGML_OP_IM2COL_3D:
+4 -5
View File
@@ -43,7 +43,6 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
const int64_t s13 = src1->nb[3] / ts_src1;
const int64_t s3 = dst->nb[3] / ts_dst;
const int64_t ids_s0 = ids ? ids->nb[0] / ggml_type_size(ids->type) : 0;
const int64_t ids_s1 = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;
mmf_ids_data ids_info{};
@@ -70,7 +69,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
nchannels_y = ids->ne[0];
}
if (ids && ncols_dst > 16) {
if (ids) {
const int64_t n_expert_used = ids->ne[0];
const int64_t n_experts = ne02;
const int64_t n_tokens = ne12;
@@ -110,7 +109,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
constexpr int vals_per_T = 1;
mul_mat_f_switch_rows_per_block<float>(
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
} break;
case GGML_TYPE_F16: {
@@ -118,7 +117,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
constexpr int vals_per_T = 2;
mul_mat_f_switch_rows_per_block<half2>(
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
} break;
case GGML_TYPE_BF16: {
@@ -126,7 +125,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
constexpr int vals_per_T = 2;
mul_mat_f_switch_rows_per_block<nv_bfloat162>(
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
} break;
default:
+69 -176
View File
@@ -47,12 +47,11 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
bool ggml_cuda_should_use_mmf(enum ggml_type type, int cc, int warp_size, const int64_t * scr0_ne, const size_t * src0_nb, const int src1_ncols, bool mul_mat_id);
template <typename T, int rows_per_block, int cols_per_block, int nwarps, bool has_ids>
template <typename T, int rows_per_block, int cols_per_block, int nwarps>
__launch_bounds__(ggml_cuda_get_physical_warp_size()*nwarps, 1)
static __global__ void mul_mat_f(
const T * __restrict__ x, const float * __restrict__ y, const int32_t * __restrict__ ids, float * __restrict__ dst,
const int ncols, const int ncols_dst_total, const int nchannels_dst, const int stride_row, const int stride_col_y, const int stride_col_dst,
const int stride_col_id, const int stride_row_id,
const T * __restrict__ x, const float * __restrict__ y, float * __restrict__ dst,
const int ncols, const int stride_row, const int stride_col_y, const int stride_col_dst,
const int channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
const int sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst) {
// TODO: handle this in a consistent and simpler way after AMD MFMA support has been added
@@ -90,85 +89,26 @@ static __global__ void mul_mat_f(
constexpr int ntA = rows_per_block / tile_A::I;
constexpr int ntB = (cols_per_block + tile_B::I - 1) / tile_B::I;
const int row0 = blockIdx.x * rows_per_block;
const int row0 = blockIdx.x * rows_per_block;
int expert_idx = 0;
[[maybe_unused]] int col_base = 0;
const int channel_dst = has_ids ? 0 : blockIdx.y;
if constexpr (has_ids) {
// experts + tiles of ncols_dst are packed in the y dimension
int col_tiles = (ncols_dst_total + cols_per_block - 1) / cols_per_block;
const int nchannels_x = gridDim.y / col_tiles;
const int tile_idx = blockIdx.y / nchannels_x;
expert_idx = blockIdx.y - tile_idx * nchannels_x;
col_base = tile_idx * cols_per_block;
}
const int channel_x = has_ids ? expert_idx : (channel_dst / channel_ratio);
const int channel_dst = blockIdx.y;
const int channel_x = channel_dst / channel_ratio;
const int channel_y = channel_dst;
const int sample_dst = blockIdx.z;
const int sample_x = sample_dst / sample_ratio;
const int sample_y = sample_dst;
x += int64_t(sample_x) *stride_sample_x + channel_x *stride_channel_x + row0*stride_row ;
y += int64_t(sample_y) *stride_sample_y + (has_ids ? 0 : channel_y *stride_channel_y);
dst += int64_t(sample_dst)*stride_sample_dst + (has_ids ? 0 : channel_dst*stride_channel_dst);
if constexpr (has_ids) {
constexpr int y_stride_scale = std::is_same_v<T, float> ? 1 : 2;
const int64_t col_offset = col_base;
y += col_offset * stride_col_y * y_stride_scale;
dst += col_offset * stride_col_dst;
ids += col_offset * stride_row_id;
}
y += int64_t(sample_y) *stride_sample_y + channel_y *stride_channel_y;
dst += int64_t(sample_dst)*stride_sample_dst + channel_dst*stride_channel_dst;
[[maybe_unused]] const float2 * y2 = (const float2 *) y;
extern __shared__ char data_mmv[];
char * shmem_base = data_mmv;
[[maybe_unused]] int * slot_map = (int *) shmem_base;
char * compute_base = has_ids ? (shmem_base + GGML_PAD(cols_per_block, 16) * sizeof(int)) : shmem_base;
tile_C C[ntA][ntB];
T * tile_xy = (T *) compute_base + threadIdx.y*(tile_A::I * tile_k_padded);
if constexpr (has_ids) {
int found = 0;
for (int j0 = 0; j0 < cols_per_block; j0 += nwarps) {
const int j = j0 + threadIdx.y;
if (threadIdx.x == 0) {
slot_map[j] = -1;
}
ggml_cuda_syncwarp();
if (col_base + j >= ncols_dst_total) {
continue;
}
const int32_t * __restrict__ id_row = ids + j*stride_row_id;
for (int k = threadIdx.x; k < nchannels_dst; k += warp_size) {
int match = id_row[k*stride_col_id] == expert_idx;
if (match) {
slot_map[j] = k;
found = 1;
break;
}
}
}
if (!__syncthreads_or(found)) {
return;
}
}
T * tile_xy = (T *) data_mmv + threadIdx.y*(tile_A::I * tile_k_padded);
for (int col = threadIdx.y*warp_size + threadIdx.x; col < ncols; col += nwarps*warp_size) {
tile_A A[ntA][warp_size / tile_A::J];
@@ -194,26 +134,15 @@ static __global__ void mul_mat_f(
for (int j0 = 0; j0 < tile_B::I; ++j0) {
const int j = j0 + itB*tile_B::I;
if constexpr (!has_ids) {
tile_xy[j0*tile_k_padded + threadIdx.x] = j < cols_per_block ? y[j*stride_col_y + col] : 0.0f;
} else {
const bool valid = j < cols_per_block && (col_base + j) < ncols_dst_total && slot_map[j] >= 0;
tile_xy[j0*tile_k_padded + threadIdx.x] = valid ? y[slot_map[j]*stride_channel_y + j*stride_col_y + col] : 0.0f;
}
tile_xy[j0*tile_k_padded + threadIdx.x] = j < cols_per_block ? y[j*stride_col_y + col] : 0.0f;
}
} else if constexpr (std::is_same_v<T, half2> || std::is_same_v<T, nv_bfloat162>) {
#pragma unroll
for (int j0 = 0; j0 < tile_B::I; ++j0) {
const int j = j0 + itB*tile_B::I;
if constexpr (!has_ids) {
const float2 tmp = j < cols_per_block ? y2[j*stride_col_y + col] : make_float2(0.0f, 0.0f);
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
} else {
const bool valid = j < cols_per_block && (col_base + j) < ncols_dst_total && slot_map[j] >= 0;
float2 tmp = valid ? *(const float2*) &y[slot_map[j]*stride_channel_y + 2*(j*stride_col_y + col)] : make_float2(0.0f, 0.0f);
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
}
const float2 tmp = j < cols_per_block ? y2[j*stride_col_y + col] : make_float2(0.0f, 0.0f);
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
}
} else {
static_assert(std::is_same_v<T, void>, "unsupported type");
@@ -231,7 +160,7 @@ static __global__ void mul_mat_f(
}
}
float * buf_iw = (float *) compute_base;
float * buf_iw = (float *) data_mmv;
constexpr int kiw = nwarps*rows_per_block + mmf_get_padding();
if (nwarps > 1) {
@@ -278,26 +207,15 @@ static __global__ void mul_mat_f(
}
}
if constexpr (!has_ids) {
#pragma unroll
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
dst[j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
}
} else {
const int slot = (j < cols_per_block) ? slot_map[j] : -1;
if (slot >= 0 && (col_base + j) < ncols_dst_total) {
#pragma unroll
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
dst[slot*stride_channel_dst + j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
}
}
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
dst[j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
}
}
}
#else
GGML_UNUSED_VARS(x, y, ids, dst,
ncols, ncols_dst_total, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id,
GGML_UNUSED_VARS(x, y, dst,
ncols, stride_row, stride_col_y, stride_col_dst,
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
NO_DEVICE_CODE;
@@ -588,22 +506,21 @@ static __global__ void mul_mat_f_ids(
template<typename T, int rows_per_block, int cols_per_block, int nwarps>
static inline void mul_mat_f_switch_ids(
const T * x, const float * y, const int32_t * ids, float * dst,
const T * x, const float * y, float * dst,
const int64_t ncols_x, const int64_t ncols_dst, const int64_t nchannels_dst,
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
const int64_t stride_col_id, const int64_t stride_row_id,
const int64_t channel_ratio, const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst,
const int64_t sample_ratio, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
const dim3 & block_nums, const dim3 & block_dims, const int nbytes_shared_total, cudaStream_t stream,
const mmf_ids_data * ids_data) {
const bool has_ids_data = ids_data && ids_data->ids_src_compact;
// Use the compact-ids kernel only for larger tiles; for small ncols_dst (< 16)
// we prefer the normal mul_mat_f path with has_ids=true.
if (has_ids_data && ncols_dst > 16) {
if (has_ids_data) {
if (ids_data->max_tiles == 0) {
return;
}
GGML_ASSERT(block_nums.z == 1);
dim3 block_nums_ids(block_nums.x, ids_data->max_tiles, 1);
const uint3 sis1_fd = ids_data->sis1 > 0 ? init_fastdiv_values((uint32_t) ids_data->sis1) : make_uint3(0, 0, 1);
@@ -615,19 +532,10 @@ static inline void mul_mat_f_switch_ids(
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
sis1_fd, nch_fd);
} else if (ids) {
const int64_t col_tiles = (ncols_dst + cols_per_block - 1) / cols_per_block;
dim3 block_nums_ids = block_nums;
block_nums_ids.y *= col_tiles;
mul_mat_f<T, rows_per_block, cols_per_block, nwarps, true><<<block_nums_ids, block_dims, nbytes_shared_total, stream>>>
(x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
} else {
mul_mat_f<T, rows_per_block, cols_per_block, nwarps, false><<<block_nums, block_dims, nbytes_shared_total, stream>>>
(x, y, ids, dst, ncols_x, cols_per_block, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
mul_mat_f<T, rows_per_block, cols_per_block, nwarps><<<block_nums, block_dims, nbytes_shared_total, stream>>>
(x, y, dst, ncols_x, stride_row, stride_col_y, stride_col_dst,
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
}
}
@@ -637,7 +545,6 @@ void mul_mat_f_cuda(
const T * x, const float * y, const int32_t * ids, float * dst,
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
const int64_t stride_col_id, const int64_t stride_row_id,
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
@@ -673,70 +580,59 @@ void mul_mat_f_cuda(
const int nbytes_shared_iter = nwarps_best * (volta_mma_available(cc) ? tile_A_32::I : tile_A_16::I) * (warp_size + mmf_get_padding(cc)) * 4;
const int nbytes_cols_per_block_pad = (amd_wmma_available(cc) || amd_mfma_available(cc)) ? tile_B_16::I : tile_B_8::I;
const int nbytes_shared_combine = GGML_PAD(cols_per_block, nbytes_cols_per_block_pad) * (nwarps_best*rows_per_block + mmf_get_padding(cc)) * 4;
const int nbytes_shared = std::max(nbytes_shared_iter, nbytes_shared_combine);
const int nbytes_slotmap = ids ? GGML_PAD(cols_per_block, 16) * sizeof(int) : 0;
const int nbytes_shared_total = nbytes_shared + nbytes_slotmap;
const int64_t grid_y = ids ? nchannels_x : nchannels_dst;
const int nbytes_shared_total = std::max(nbytes_shared_iter, nbytes_shared_combine);
const dim3 block_nums(nrows_x/rows_per_block, grid_y, nsamples_dst);
const dim3 block_nums(nrows_x/rows_per_block, nchannels_dst, nsamples_dst);
const dim3 block_dims(warp_size, nwarps_best, 1);
switch (nwarps_best) {
case 1: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 1>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 2: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 2>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 3: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 3>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 4: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 4>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 5: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 5>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 6: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 6>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 7: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 7>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 8: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 8>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
default:
GGML_ABORT("fatal error");
@@ -750,95 +646,94 @@ static void mul_mat_f_switch_cols_per_block(
const T * x, const float * y, const int32_t * ids, float * dst,
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
const int64_t stride_col_id, const int stride_row_id,
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
cudaStream_t stream, const mmf_ids_data * ids_data) {
const int ncols_case = (ids && ncols_dst > 16) ? 16 : ncols_dst;
const int ncols_case = ids ? 16 : ncols_dst;
GGML_ASSERT(ids || ncols_dst <= 16);
switch (ncols_case) {
case 1: {
mul_mat_f_cuda<T, rows_per_block, 1>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 2: {
mul_mat_f_cuda<T, rows_per_block, 2>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 3: {
mul_mat_f_cuda<T, rows_per_block, 3>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 4: {
mul_mat_f_cuda<T, rows_per_block, 4>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 5: {
mul_mat_f_cuda<T, rows_per_block, 5>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 6: {
mul_mat_f_cuda<T, rows_per_block, 6>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 7: {
mul_mat_f_cuda<T, rows_per_block, 7>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 8: {
mul_mat_f_cuda<T, rows_per_block, 8>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 9: {
mul_mat_f_cuda<T, rows_per_block, 9>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 10: {
mul_mat_f_cuda<T, rows_per_block, 10>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 11: {
mul_mat_f_cuda<T, rows_per_block, 11>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 12: {
mul_mat_f_cuda<T, rows_per_block, 12>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 13: {
mul_mat_f_cuda<T, rows_per_block, 13>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 14: {
mul_mat_f_cuda<T, rows_per_block, 14>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 15: {
mul_mat_f_cuda<T, rows_per_block, 15>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 16: {
mul_mat_f_cuda<T, rows_per_block, 16>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
default:
@@ -851,7 +746,6 @@ static void mul_mat_f_switch_rows_per_block(
const int rows_per_block, const T * x, const float * y, const int32_t * ids, float * dst,
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
const int64_t stride_col_id, const int stride_row_id,
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
@@ -860,13 +754,13 @@ static void mul_mat_f_switch_rows_per_block(
case MMF_ROWS_PER_BLOCK: {
mul_mat_f_switch_cols_per_block<T, MMF_ROWS_PER_BLOCK>(
x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case MMF_ROWS_PER_BLOCK_CDNA: {
mul_mat_f_switch_cols_per_block<T, MMF_ROWS_PER_BLOCK_CDNA>(
x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
default:
@@ -878,7 +772,6 @@ static void mul_mat_f_switch_rows_per_block(
template void mul_mat_f_cuda<T, nrows_dst, ncols_dst>( \
const T * x, const float * y, const int32_t * ids, float * dst, \
const int64_t ncols_x, const int64_t nrows_x, int64_t ncols_dst_total, const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst, \
const int64_t stride_col_id, const int64_t stride_row_id, \
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst, \
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,\
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst, \
+159 -60
View File
@@ -24,6 +24,50 @@ template <int n> struct mm_ids_pow2 { static constexpr int value = 2*mm_ids_pow2
template <> struct mm_ids_pow2<1> { static constexpr int value = 1; };
template <> struct mm_ids_pow2<0> { static constexpr int value = 1; };
template <bool increment_nex_prev, bool write_to_store>
static __device__ __forceinline__ void check_experts(
mm_ids_helper_store * store, int & nex_prev, int & it_compact,
const int expert, const int it, const int iex, const int expert_found) {
if (increment_nex_prev) {
nex_prev += expert_found < expert;
}
if (write_to_store) {
const ggml_cuda_thread_mask_t iex_used = ggml_cuda_ballot(expert_found == expert);
const ggml_cuda_thread_mask_t mask = (ggml_cuda_thread_mask_t(1) << threadIdx.x) - 1;
const int it_compact_offset = ggml_cuda_popc(iex_used & mask);
if (expert_found == expert) {
store[it_compact + it_compact_offset] = mm_ids_helper_store(it, iex);
}
it_compact += ggml_cuda_popc(iex_used);
}
}
static __device__ __forceinline__ void dump_store(
int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst,
const int n_expert_used, const int nchannels_y, const int sis1, const bool write_inverse,
const mm_ids_helper_store * __restrict__ store, const int nex_prev, int & n_dumped, const int it_compact) {
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {
const mm_ids_helper_store store_it = store[itc];
const int it = store_it.it();
const int iex_used = store_it.iex_used();
ids_dst[nex_prev + n_dumped + itc] = it*n_expert_used + iex_used;
// ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup
if (write_inverse) {
ids_src1[it*n_expert_used + iex_used] = nex_prev + n_dumped + itc;
} else {
ids_src1[nex_prev + n_dumped + itc] = it*sis1 + iex_used % nchannels_y;
}
}
n_dumped += it_compact;
}
constexpr int n_unroll = 4; // Manually unroll this many loop iterations for better performance.
// Helper function for mul_mat_id, converts ids to a more convenient format.
// ids_src1 describes how to permute the flattened column indices of src1 in order to get a compact src1 tensor sorted by expert.
// ids_dst describes the same mapping but for the dst tensor.
@@ -36,85 +80,138 @@ static __global__ void mm_ids_helper(
const int32_t * __restrict__ ids,
int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec,
const int n_tokens, const int n_expert_used_var, const int nchannels_y, const uint3 tile_width,
const int si1, const int sis1, const bool write_inverse) {
const int si1, const int sis1, const bool write_inverse, const int ne_store) {
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
const int n_expert_used = n_expert_used_template == 0 ? n_expert_used_var : n_expert_used_template;
const int expert = blockIdx.x;
const int expert = blockIdx.x; // The expert this CUDA block handles.
extern __shared__ char data_mm_ids_helper[];
mm_ids_helper_store * store = (mm_ids_helper_store *) data_mm_ids_helper;
int nex_prev = 0; // Number of columns for experts with a lower index.
int n_dumped = 0; // Number of data values that have been dumped to output buffers.
int it_compact = 0; // Running index for the compact slice of this expert.
// Step 1: Iterate over n_tokens, increment nex_prev to determine the output indices, fill up store.
// Step 1.5: If the store does not have enough space to hold all values, iterate over the rest of the tokens just to increment nex_prev.
// Step 2: Dump the stored values using nex_prev to get the correct output indices.
// Step 3: If we were not able to process all values, resume at the index up to which we were able to dump the output and re-do.
if constexpr (n_expert_used_template == 0) {
// Generic implementation:
for (int it = 0; it < n_tokens; ++it) {
int iex_used = -1; // The index at which the expert is used, if any.
for (int iex = threadIdx.x; iex < n_expert_used; iex += warp_size) {
const int expert_used = ids[it*si1 + iex];
nex_prev += expert_used < expert;
if (expert_used == expert) {
iex_used = iex;
int it = 0;
for (; it < n_tokens && it_compact + max(warp_size, n_expert_used) <= ne_store; ++it) {
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
const int iex = iex0 + threadIdx.x;
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
constexpr bool increment_nex_prev = true;
constexpr bool write_to_store = true;
check_experts<increment_nex_prev, write_to_store>
(store, nex_prev, it_compact, expert, it, iex, expert_found);
}
}
int it_dumped = it;
for (; it < n_tokens; ++it) {
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
const int iex = iex0 + threadIdx.x;
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
constexpr bool increment_nex_prev = true;
constexpr bool write_to_store = false;
check_experts<increment_nex_prev, write_to_store>
(store, nex_prev, it_compact, expert, it, iex, expert_found);
}
}
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
ggml_cuda_syncwarp();
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
it_compact = 0;
for (int it = it_dumped; it < n_tokens; ++it) {
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
const int iex = iex0 + threadIdx.x;
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
constexpr bool increment_nex_prev = false;
constexpr bool write_to_store = true;
check_experts<increment_nex_prev, write_to_store>
(store, nex_prev, it_compact, expert, it, iex, expert_found);
if (it_compact + max(warp_size, n_expert_used) >= ne_store) {
ggml_cuda_syncwarp();
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
it_compact = 0;
}
}
if (iex_used != -1) {
store[it_compact] = mm_ids_helper_store(it, iex_used);
}
if (warp_reduce_any<warp_size>(iex_used != -1)) {
it_compact++;
}
}
if (it_compact > 0) {
ggml_cuda_syncwarp();
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
}
} else {
// Implementation optimized for specific numbers of experts used:
// a warp holds a whole number of token slots, so the slot count is padded to a power of 2
constexpr int neu_padded = mm_ids_pow2<n_expert_used_template>::value;
static_assert(neu_padded <= warp_size && warp_size % neu_padded == 0, "bad n_expert_used");
for (int it0 = 0; it0 < n_tokens; it0 += warp_size/neu_padded) {
const int it = it0 + threadIdx.x / neu_padded;
const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any.
const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any.
const int expert_used = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
ids[it*si1 + iex] : INT_MAX;
const int iex_used = expert_used == expert ? iex : -1;
nex_prev += expert_used < expert;
// Whether the threads at this token position have used the expert:
const int it_compact_add_self = warp_reduce_any<neu_padded>(iex_used != -1);
// Do a scan over threads at lower token positions in warp to get the correct index for writing data:
int it_compact_add_lower = 0;
int it0 = 0;
for (; it0 < n_tokens && it_compact + n_unroll*warp_size <= ne_store; it0 += n_unroll*warp_size/neu_padded) {
#pragma unroll
for (int offset = neu_padded; offset < warp_size; offset += neu_padded) {
const int tmp = ggml_cuda_shfl_up(it_compact_add_self, offset, warp_size);
if (threadIdx.x >= static_cast<unsigned int>(offset)) {
it_compact_add_lower += tmp;
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
const int it = it0 + it1 + threadIdx.x / neu_padded;
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
ids[it*si1 + iex] : INT_MAX;
constexpr bool increment_nex_prev = true;
constexpr bool write_to_store = true;
check_experts<increment_nex_prev, write_to_store>
(store, nex_prev, it_compact, expert, it, iex, expert_found);
}
}
int it0_dumped = it0;
for (; it0 < n_tokens; it0 += n_unroll*warp_size/neu_padded) {
#pragma unroll
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
const int it = it0 + it1 + threadIdx.x / neu_padded;
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
ids[it*si1 + iex] : INT_MAX;
constexpr bool increment_nex_prev = true;
constexpr bool write_to_store = false;
check_experts<increment_nex_prev, write_to_store>
(store, nex_prev, it_compact, expert, it, iex, expert_found);
}
}
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
ggml_cuda_syncwarp();
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
it_compact = 0;
for (it0 = it0_dumped; it0 < n_tokens; it0 += n_unroll*warp_size/neu_padded) {
#pragma unroll
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
const int it = it0 + it1 + threadIdx.x / neu_padded;
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
ids[it*si1 + iex] : INT_MAX;
constexpr bool increment_nex_prev = false;
constexpr bool write_to_store = true;
check_experts<increment_nex_prev, write_to_store>
(store, nex_prev, it_compact, expert, it, iex, expert_found);
if (it_compact + warp_size >= ne_store) {
ggml_cuda_syncwarp();
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
it_compact = 0;
}
}
if (iex_used != -1) {
store[it_compact + it_compact_add_lower] = mm_ids_helper_store(it, iex_used);
}
// The thread with the highest index in the warp always has the sum over the whole warp, use it to increment all threads:
it_compact += ggml_cuda_shfl(it_compact_add_lower + it_compact_add_self, warp_size - 1, warp_size);
}
}
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
ggml_cuda_syncwarp();
for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {
const mm_ids_helper_store store_it = store[itc];
const int it = store_it.it();
const int iex_used = store_it.iex_used();
ids_dst[nex_prev + itc] = it*n_expert_used + iex_used;
// ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup
if (write_inverse) {
ids_src1[it*n_expert_used + iex_used] = nex_prev + itc;
} else {
ids_src1[nex_prev + itc] = it*sis1 + iex_used % nchannels_y;
if (it_compact > 0) {
ggml_cuda_syncwarp();
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
}
}
@@ -123,7 +220,7 @@ static __global__ void mm_ids_helper(
}
int col_low = nex_prev;
int col_high = nex_prev + min(int(tile_width.z), it_compact);
int col_high = nex_prev + min(int(tile_width.z), n_dumped);
// The while loop covers all tiles at the seams containing the column with % tile_width == 0.
while (col_high > fastdiv(col_low + tile_width.z - 1, tile_width) * tile_width.z) {
@@ -131,7 +228,7 @@ static __global__ void mm_ids_helper(
= {expert, col_low, col_high, /*padding =*/ 0};
col_low = col_high;
col_high = min(col_high + int(tile_width.z), nex_prev + it_compact);
col_high = min(col_high + int(tile_width.z), nex_prev + n_dumped);
}
// Finally, at most one extra tile_spec between the seams per expert.
@@ -157,10 +254,12 @@ static void launch_mm_ids_helper(
const dim3 num_blocks(n_experts, 1, 1);
const dim3 block_size(warp_size, 1, 1);
const size_t nbytes_shared = n_tokens*sizeof(mm_ids_helper_store);
GGML_ASSERT(nbytes_shared <= smpbo);
const size_t ne_shared_pad = n_expert_used_template == 0 ? std::max(n_expert_used_var, warp_size) : n_unroll*warp_size;
const size_t nbytes_store_core = std::min(smpbo - ne_shared_pad * sizeof(mm_ids_helper_store), n_tokens * sizeof(mm_ids_helper_store));
const size_t nbytes_shared = nbytes_store_core + ne_shared_pad * sizeof(mm_ids_helper_store);
const size_t ne_store = nbytes_shared / sizeof(mm_ids_helper_store);
mm_ids_helper<n_expert_used_template><<<num_blocks, block_size, nbytes_shared, stream>>>
(ids, ids_src1, ids_dst, tile_spec, n_tokens, n_expert_used_var, nchannels_y, tile_width_fd, si1, sis1, write_inverse);
(ids, ids_src1, ids_dst, tile_spec, n_tokens, n_expert_used_var, nchannels_y, tile_width_fd, si1, sis1, write_inverse, ne_store);
}
void ggml_cuda_launch_mm_ids_helper(
+50 -13
View File
@@ -92,13 +92,16 @@ static __global__ void rope_norm(const T * x,
} else if constexpr (std::is_same_v<half, D>) {
half2 v = make_half2(x0, x1);
ggml_cuda_memcpy_1<4>(dst + idst, &v);
} else if constexpr (std::is_same_v<nv_bfloat16, D>) {
const __align__(4) nv_bfloat16 v[2] = { ggml_cuda_cast<nv_bfloat16>(x0), ggml_cuda_cast<nv_bfloat16>(x1) };
ggml_cuda_memcpy_1<sizeof(v)>(dst + idst, v);
}
};
if (i0 < n_offs || i0 >= n_offs + n_dims) {
if (inplace) {
return;
}
store_coaelsced(x[ix + 0], x[ix + 1]);
store_coaelsced(ggml_cuda_cast<float>(x[ix + 0]), ggml_cuda_cast<float>(x[ix + 1]));
return;
}
@@ -113,8 +116,8 @@ static __global__ void rope_norm(const T * x,
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta);
const float x0 = x[ix + 0];
const float x1 = x[ix + 1];
const float x0 = ggml_cuda_cast<float>(x[ix + 0]);
const float x1 = ggml_cuda_cast<float>(x[ix + 1]);
store_coaelsced(x0 * cos_theta - x1 * sin_theta, x0 * sin_theta + x1 * cos_theta);
}
@@ -189,8 +192,8 @@ static __global__ void rope_neox(const T * x,
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta);
// idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
const float x0 = x[ix + n_offs/2 + 0];
const float x1 = x[ix + n_offs/2 + n_dims/2];
const float x0 = ggml_cuda_cast<float>(x[ix + n_offs/2 + 0]);
const float x1 = ggml_cuda_cast<float>(x[ix + n_offs/2 + n_dims/2]);
dst[idst + n_offs/2 + 0] = ggml_cuda_cast<D>(x0 * cos_theta - x1 * sin_theta);
dst[idst + n_offs/2 + n_dims / 2] = ggml_cuda_cast<D>(x0 * sin_theta + x1 * cos_theta);
@@ -293,8 +296,8 @@ static __global__ void rope_multi(const T * x,
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, iw, ext_factor, attn_factor, cos_theta, sin_theta);
// idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
const float x0 = x[ix + n_offs/2 + 0];
const float x1 = x[ix + n_offs/2 + n_dims/2];
const float x0 = ggml_cuda_cast<float>(x[ix + n_offs/2 + 0]);
const float x1 = ggml_cuda_cast<float>(x[ix + n_offs/2 + n_dims/2]);
dst[idst + n_offs/2 + 0] = ggml_cuda_cast<D>(x0*cos_theta - x1*sin_theta);
dst[idst + n_offs/2 + n_dims/2] = ggml_cuda_cast<D>(x0*sin_theta + x1*cos_theta);
@@ -357,11 +360,11 @@ static __global__ void rope_vision(const T * x,
rope_yarn<forward>(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta);
const float x0 = x[ix + 0];
const float x1 = x[ix + n_dims];
const float x0 = ggml_cuda_cast<float>(x[ix + 0]);
const float x1 = ggml_cuda_cast<float>(x[ix + n_dims]);
dst[idst + 0] = x0*cos_theta - x1*sin_theta;
dst[idst + n_dims] = x0*sin_theta + x1*cos_theta;
dst[idst + 0] = ggml_cuda_cast<T>(x0*cos_theta - x1*sin_theta);
dst[idst + n_dims] = ggml_cuda_cast<T>(x0*sin_theta + x1*cos_theta);
}
template <bool forward, typename T, typename D>
@@ -569,8 +572,8 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
}
cudaStream_t stream = ctx.stream();
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_BF16);
GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_BF16);
// When not fused, src0 and dst types must match
// When fused (ROPE+VIEW+SET_ROWS), src0 may be F32 and dst may be F16
GGML_ASSERT(src0->type == dst->type || (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16));
@@ -655,6 +658,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
set_rows_stride, inplace, stream);
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) {
rope_neox_cuda<forward, nv_bfloat16, nv_bfloat16>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02,
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
set_rows_stride, inplace, stream);
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) {
rope_neox_cuda<forward, nv_bfloat16, half>((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
set_rows_stride, inplace, stream);
} else {
GGML_ABORT("fatal error");
}
@@ -674,6 +687,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
row_indices, set_rows_stride, inplace, stream);
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) {
rope_multi_cuda<forward, nv_bfloat16, nv_bfloat16>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02,
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
row_indices, set_rows_stride, inplace, stream);
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) {
rope_multi_cuda<forward, nv_bfloat16, half>((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
ext_factor, attn_factor, corr_dims, freq_factors, sections, is_imrope,
row_indices, set_rows_stride, inplace, stream);
} else {
GGML_ABORT("fatal error");
}
@@ -686,6 +709,10 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
rope_vision_cuda<forward>((const half *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,
s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,
corr_dims, freq_factors, sections, stream);
} else if (src0->type == GGML_TYPE_BF16) {
rope_vision_cuda<forward>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02, s03, s1,
s2, s3, n_dims, nr, pos, freq_scale, freq_base, ext_factor, attn_factor,
corr_dims, freq_factors, sections, stream);
} else {
GGML_ABORT("fatal error");
}
@@ -705,6 +732,16 @@ void ggml_cuda_op_rope_impl(ggml_backend_cuda_context & ctx,
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
set_rows_stride, inplace, stream);
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_BF16) {
rope_norm_cuda<forward, nv_bfloat16, nv_bfloat16>((const nv_bfloat16 *) src0_d, (nv_bfloat16 *) dst_d, ne00, ne01, ne02, s01, s02,
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
set_rows_stride, inplace, stream);
} else if (src0->type == GGML_TYPE_BF16 && dst_type == GGML_TYPE_F16) {
rope_norm_cuda<forward, nv_bfloat16, half>((const nv_bfloat16 *) src0_d, (half *) dst_d, ne00, ne01, ne02, s01, s02,
s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
set_rows_stride, inplace, stream);
} else {
GGML_ABORT("fatal error");
}
+34 -19
View File
@@ -4,8 +4,8 @@
typedef void (*set_rows_kernel_t)(const char * src, char * dst);
// Generic quantized set_rows kernel template
template <typename idx_t, typename block_type, int qk, void (*quantize_func)(const float *, block_type *)>
static __global__ void k_set_rows_quant(const float * __restrict__ src0,
template <typename src_t, typename idx_t, typename block_type, int qk, void (*quantize_func)(const float *, block_type *)>
static __global__ void k_set_rows_quant(const src_t * __restrict__ src0,
const idx_t * __restrict__ src1,
block_type * __restrict__ dst,
const int64_t ne_total,
@@ -56,13 +56,23 @@ static __global__ void k_set_rows_quant(const float * __restrict__ src0,
ggml_cuda_pdl_sync();
const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12);
const float * src0_row = src0 + i01*s01 + i02*s02 + i03*s03;
const src_t * src0_row = src0 + i01*s01 + i02*s02 + i03*s03;
block_type * dst_row_ptr = dst + (dst_row*s1 + i02*s2 + i03*s3) / sizeof(block_type);
const float * src_block = src0_row + i00;
const src_t * src_block = src0_row + i00;
block_type * dst_block = dst_row_ptr + i00 / qk;
quantize_func(src_block, dst_block);
if constexpr (std::is_same_v<src_t, float>) {
quantize_func(src_block, dst_block);
} else {
// the quantize functions take F32 input: widen the block first
float src_block_f32[qk];
#pragma unroll
for (int j = 0; j < qk; ++j) {
src_block_f32[j] = ggml_cuda_cast<float>(src_block[j]);
}
quantize_func(src_block_f32, dst_block);
}
GGML_UNUSED(ne10);
GGML_UNUSED(ne11);
@@ -71,9 +81,9 @@ static __global__ void k_set_rows_quant(const float * __restrict__ src0,
}
// Template dispatch function for quantized set_rows
template<typename idx_t, typename block_type, int qk, void (*quantize_func)(const float*, block_type*)>
template<typename src_t, typename idx_t, typename block_type, int qk, void (*quantize_func)(const float*, block_type*)>
static void set_rows_cuda_quant(
const float * src0_d, const idx_t * src1_d, block_type * dst_d,
const src_t * src0_d, const idx_t * src1_d, block_type * dst_d,
const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,
const int64_t ne10, const int64_t ne11, const int64_t ne12, const int64_t ne13,
const size_t nb01, const size_t nb02, const size_t nb03,
@@ -87,9 +97,9 @@ static void set_rows_cuda_quant(
const dim3 block_size(CUDA_SET_ROWS_BLOCK_SIZE);
const dim3 grid_size(num_blocks);
const int64_t s01 = nb01/sizeof(float);
const int64_t s02 = nb02/sizeof(float);
const int64_t s03 = nb03/sizeof(float);
const int64_t s01 = nb01/sizeof(src_t);
const int64_t s02 = nb02/sizeof(src_t);
const int64_t s03 = nb03/sizeof(src_t);
const int64_t s10 = nb10/sizeof(idx_t);
const int64_t s11 = nb11/sizeof(idx_t);
const int64_t s12 = nb12/sizeof(idx_t);
@@ -104,7 +114,7 @@ static void set_rows_cuda_quant(
const uint3 ne11_fd = init_fastdiv_values((uint32_t) ne11);
const uint3 ne12_fd = init_fastdiv_values((uint32_t) ne12);
k_set_rows_quant<idx_t, block_type, qk, quantize_func><<<grid_size, block_size, 0, stream>>>(
k_set_rows_quant<src_t, idx_t, block_type, qk, quantize_func><<<grid_size, block_size, 0, stream>>>(
src0_d, src1_d, dst_d, ne_total, ne10, ne11, ne12, ne13, s01, s02, s03, s10, s11, s12, s1, s2, s3, ne00_fd,
ne01_fd, ne02_fd, ne11_fd, ne12_fd);
}
@@ -268,7 +278,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
stream
);
} else if (dst->type == GGML_TYPE_Q4_0) {
set_rows_cuda_quant<idx_t, block_q4_0, QK4_0, quantize_f32_q4_0_block>(
set_rows_cuda_quant<src_t, idx_t, block_q4_0, QK4_0, quantize_f32_q4_0_block>(
src0_d, src1_d, (block_q4_0*)dst->data,
ne00, ne01, ne02, ne03,
ne10, ne11, ne12, ne13,
@@ -278,7 +288,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
stream
);
} else if (dst->type == GGML_TYPE_Q4_1) {
set_rows_cuda_quant<idx_t, block_q4_1, QK4_1, quantize_f32_q4_1_block>(
set_rows_cuda_quant<src_t, idx_t, block_q4_1, QK4_1, quantize_f32_q4_1_block>(
src0_d, src1_d, (block_q4_1*)dst->data,
ne00, ne01, ne02, ne03,
ne10, ne11, ne12, ne13,
@@ -288,7 +298,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
stream
);
} else if (dst->type == GGML_TYPE_Q5_0) {
set_rows_cuda_quant<idx_t, block_q5_0, QK5_0, quantize_f32_q5_0_block>(
set_rows_cuda_quant<src_t, idx_t, block_q5_0, QK5_0, quantize_f32_q5_0_block>(
src0_d, src1_d, (block_q5_0*)dst->data,
ne00, ne01, ne02, ne03,
ne10, ne11, ne12, ne13,
@@ -298,7 +308,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
stream
);
} else if (dst->type == GGML_TYPE_Q5_1) {
set_rows_cuda_quant<idx_t, block_q5_1, QK5_1, quantize_f32_q5_1_block>(
set_rows_cuda_quant<src_t, idx_t, block_q5_1, QK5_1, quantize_f32_q5_1_block>(
src0_d, src1_d, (block_q5_1*)dst->data,
ne00, ne01, ne02, ne03,
ne10, ne11, ne12, ne13,
@@ -308,7 +318,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
stream
);
} else if (dst->type == GGML_TYPE_Q8_0) {
set_rows_cuda_quant<idx_t, block_q8_0, QK8_0, quantize_f32_q8_0_block>(
set_rows_cuda_quant<src_t, idx_t, block_q8_0, QK8_0, quantize_f32_q8_0_block>(
src0_d, src1_d, (block_q8_0*)dst->data,
ne00, ne01, ne02, ne03,
ne10, ne11, ne12, ne13,
@@ -318,7 +328,7 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
stream
);
} else if (dst->type == GGML_TYPE_IQ4_NL) {
set_rows_cuda_quant<idx_t, block_iq4_nl, QK4_NL, quantize_f32_iq4_nl_block>(
set_rows_cuda_quant<src_t, idx_t, block_iq4_nl, QK4_NL, quantize_f32_iq4_nl_block>(
src0_d, src1_d, (block_iq4_nl*)dst->data,
ne00, ne01, ne02, ne03,
ne10, ne11, ne12, ne13,
@@ -382,12 +392,11 @@ void set_rows_cuda<half, int64_t>(ggml_backend_cuda_context & ctx, const ggml_te
}
}
void ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
const ggml_tensor * src0 = dst->src[0];
const ggml_tensor * src1 = dst->src[1];
GGML_ASSERT(src0->type == GGML_TYPE_F32 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16));
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_BF16 || (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16));
GGML_ASSERT(src1->type == GGML_TYPE_I64 || src1->type == GGML_TYPE_I32);
if (src0->type == GGML_TYPE_F32) {
@@ -402,6 +411,12 @@ void ggml_cuda_op_set_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
} else {
set_rows_cuda<half, int32_t>(ctx, src0, src1, dst);
}
} else if (src0->type == GGML_TYPE_BF16) {
if (src1->type == GGML_TYPE_I64) {
set_rows_cuda<nv_bfloat16, int64_t>(ctx, src0, src1, dst);
} else {
set_rows_cuda<nv_bfloat16, int32_t>(ctx, src0, src1, dst);
}
} else {
GGML_ABORT("unsupported type %s", ggml_type_name(src0->type));
}
+2 -9
View File
@@ -184,13 +184,6 @@ static __global__ void top_k_radix_gather(
constexpr int NBINS = 1 << RADIX_BITS;
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
constexpr int nwarps = BLOCK_SIZE / warp_size;
const auto popc = [](const ggml_cuda_thread_mask_t mask) {
if constexpr (warp_size == 64) {
return __popcll(mask);
} else {
return __popc(mask);
}
};
__shared__ int warp_ties[nwarps];
const int row = blockIdx.x / blocks_per_row;
@@ -247,11 +240,11 @@ static __global__ void top_k_radix_gather(
const bool is_tie = col < warp_end && top_k_float_to_ordered(row_src[col]) == threshold;
const ggml_cuda_thread_mask_t ties = ggml_cuda_ballot(is_tie);
const int tie_pos = pos + popc(ties & (((ggml_cuda_thread_mask_t) 1 << lane) - 1));
const int tie_pos = pos + ggml_cuda_popc(ties & (((ggml_cuda_thread_mask_t) 1 << lane) - 1));
if (is_tie && tie_pos < pos_end) {
row_dst[k - rank + tie_pos] = col;
}
pos += popc(ties);
pos += ggml_cuda_popc(ties);
}
}
+1 -1
View File
@@ -1713,7 +1713,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
return has_simdgroup_reduction && op->src[0]->type == GGML_TYPE_F32 && (ggml_is_contiguous_rows(op->src[0]));
case GGML_OP_ROPE:
case GGML_OP_ROPE_BACK:
return ggml_rope_get_truncate(op);
return ggml_rope_get_truncate(op) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16);
case GGML_OP_IM2COL:
return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32);
case GGML_OP_CONV_2D:
+3
View File
@@ -323,6 +323,9 @@ set(GGML_OPENCL_KERNELS
mul_mm_f16_f32_kq_kqv
mul_mm_f16_f32_kq_p8
mul_mm_q8_f32_kqv
mul_mm_q8_kq_p8
mul_mm_q8_kq_p8_dk512
flash_attn_q8_rows
conv2d
conv2d_f16_f32
conv2_dw
+235 -55
View File
@@ -781,6 +781,10 @@ struct ggml_backend_opencl_context {
ggml_cl_buffer prealloc_fa_pd;
ggml_cl_buffer prealloc_fa_bmax;
ggml_cl_buffer prealloc_fa_bsum;
ggml_cl_buffer prealloc_fa_k8;
ggml_cl_buffer prealloc_fa_k8_scale;
ggml_cl_buffer prealloc_fa_q8;
ggml_cl_buffer prealloc_fa_q8_scale;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
ggml_cl_buffer prealloc_adreno_xmem_const;
@@ -899,6 +903,10 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_fa_kqv_direct_f16 = nullptr;
cl_kernel kernel_fa_kqv_direct_reduce = nullptr;
cl_kernel kernel_fa_v_transpose_q8 = nullptr;
cl_kernel kernel_fa_q8_rows_f16 = nullptr;
cl_kernel kernel_fa_q8_rows_f32 = nullptr;
cl_kernel kernel_mul_mm_q8_kq_p8 = nullptr;
cl_kernel kernel_mul_mm_q8_kq_p8_dk512 = nullptr;
ggml_opencl_fa_kernels fa;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
ggml_cl_adreno_xmem_attn_state adreno_xmem_attn;
@@ -1506,7 +1514,18 @@ struct ggml_backend_opencl_context {
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
~ggml_backend_opencl_context() {
for (cl_kernel kernel : {kernel_mul_mm_f16_f32_kq_p8, kernel_mul_mm_q8_f32_kqv, kernel_fa_p8_fixup, kernel_fa_v_transpose_q8, kernel_fa_kqv_direct_f16, kernel_fa_kqv_direct_reduce}) {
for (cl_kernel kernel : {
kernel_mul_mm_f16_f32_kq_p8,
kernel_mul_mm_q8_f32_kqv,
kernel_fa_p8_fixup,
kernel_fa_v_transpose_q8,
kernel_fa_kqv_direct_f16,
kernel_fa_kqv_direct_reduce,
kernel_fa_q8_rows_f16,
kernel_fa_q8_rows_f32,
kernel_mul_mm_q8_kq_p8,
kernel_mul_mm_q8_kq_p8_dk512
}) {
if (kernel != nullptr) {
CL_CHECK(clReleaseKernel(kernel));
}
@@ -1648,7 +1667,7 @@ static cl_program build_program_from_source_ex(cl_context ctx, cl_device_id dev,
return NULL;
}
static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts) {
static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts, bool fatal = true) {
cl_context ctx = backend_ctx->context;
cl_device_id dev = backend_ctx->device;
@@ -1660,7 +1679,7 @@ static cl_program build_program_from_source(ggml_backend_opencl_context * backen
return p_cached;
}
cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, /*fatal=*/true);
cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, fatal);
// Best-effort save of the freshly-built binary (no-op if cache disabled).
if (p != nullptr) {
@@ -3995,6 +4014,52 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
}
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string rows_src {
#include "flash_attn_q8_rows.cl.h"
};
const std::string kq_src {
#include "mul_mm_q8_kq_p8.cl.h"
};
#else
const std::string rows_src = read_file("flash_attn_q8_rows.cl");
const std::string kq_src = read_file("mul_mm_q8_kq_p8.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, rows_src.c_str(), compile_opts, /*fatal=*/false);
if (prog != nullptr) {
CL_CHECK((backend_ctx->kernel_fa_q8_rows_f16 = clCreateKernel(prog, "kernel_fa_q8_rows_f16", &err), err));
CL_CHECK((backend_ctx->kernel_fa_q8_rows_f32 = clCreateKernel(prog, "kernel_fa_q8_rows_f32", &err), err));
CL_CHECK(clReleaseProgram(prog));
}
prog = build_program_from_source(backend_ctx, kq_src.c_str(), compile_opts, /*fatal=*/false);
if (prog != nullptr) {
CL_CHECK((backend_ctx->kernel_mul_mm_q8_kq_p8 = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8", &err), err));
CL_CHECK(clReleaseProgram(prog));
}
GGML_LOG_CONT(".");
}
// mul_mm_q8_kq_p8_dk512
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "mul_mm_q8_kq_p8_dk512.cl.h"
};
#else
const std::string kernel_src = read_file("mul_mm_q8_kq_p8_dk512.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts, /*fatal=*/false);
if (prog != nullptr) {
cl_kernel kernel = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8_dk512", &err);
CL_CHECK(clReleaseProgram(prog));
if (err == CL_SUCCESS) {
backend_ctx->kernel_mul_mm_q8_kq_p8_dk512 = kernel;
}
}
GGML_LOG_CONT(".");
}
// mul_mm_f16_f32_kq_p8
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -12305,7 +12370,7 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
}
return false;
}
return true;
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
}
case GGML_OP_SOLVE_TRI:
return op->src[0]->type == GGML_TYPE_F32 && ggml_is_contiguous(op->src[0]);
@@ -22922,12 +22987,58 @@ static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_te
}
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
static void ggml_cl_fa_q8_rows(
ggml_backend_t backend, const ggml_tensor * src, const ggml_tensor * dst,
cl_mem values, cl_mem scales, int64_t row0, int nrow
) {
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
auto * extra = (ggml_tensor_extra_cl *) (src->view_src ? src->view_src->extra : src->extra);
const cl_ulong offset = extra->offset + src->view_offs + row0 * src->nb[1];
const cl_ulong nb1 = src->nb[1];
const cl_ulong nb2 = src->nb[2];
const int dk = src->ne[0];
const int nhead = src->ne[2];
cl_kernel kernel;
switch (src->type) {
case GGML_TYPE_F16: kernel = backend_ctx->kernel_fa_q8_rows_f16; break;
case GGML_TYPE_F32: kernel = backend_ctx->kernel_fa_q8_rows_f32; break;
default: GGML_ABORT("unsupported int8 KQ input type");
}
cl_uint a = 0;
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra->data_device));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &offset));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb2));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &values));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &scales));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dk));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nrow));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nhead));
const size_t blocks = (size_t) (dk / 32) * nrow * nhead;
size_t gws[3] = {(blocks + 63) / 64 * 64, 1, 1};
size_t lws[3] = {64, 1, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
}
static void ggml_cl_fa_kq_int8_prepare(ggml_backend_t backend, const ggml_tensor * dst, int64_t n_q_chunk) {
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
const ggml_tensor * q = dst->src[0];
const ggml_tensor * k = dst->src[1];
const size_t k_bytes = (size_t) k->ne[0] * k->ne[1] * k->ne[2];
const size_t q_bytes = (size_t) q->ne[0] * n_q_chunk * q->ne[2];
backend_ctx->prealloc_fa_k8.allocate(backend_ctx->context, k_bytes);
backend_ctx->prealloc_fa_k8_scale.allocate(backend_ctx->context, k_bytes / 32 * sizeof(cl_half));
backend_ctx->prealloc_fa_q8.allocate(backend_ctx->context, q_bytes);
backend_ctx->prealloc_fa_q8_scale.allocate(backend_ctx->context, q_bytes / 32 * sizeof(cl_half));
ggml_cl_fa_q8_rows(backend, k, dst, backend_ctx->prealloc_fa_k8.buffer, backend_ctx->prealloc_fa_k8_scale.buffer, 0, k->ne[1]);
}
static void ggml_cl_fa_kq_p8_dispatch(
ggml_backend_t backend,
const ggml_tensor * q, const ggml_tensor * k,
const ggml_tensor * mask, const ggml_tensor * sinks,
const ggml_tensor * dst,
int64_t q0, int nqc, float scale, float max_bias, int p_pitch
int64_t q0, int nqc, float scale, float max_bias, int p_pitch, bool kq_int8
) {
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
auto * extra_k = (ggml_tensor_extra_cl *) (k->view_src ? k->view_src->extra : k->extra);
@@ -22940,64 +23051,94 @@ static void ggml_cl_fa_kq_p8_dispatch(
const int D_A = k->ne[2];
const int D_B = q->ne[2];
cl_int err;
cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
cl_image_format fmt = {CL_RGBA, CL_FLOAT};
cl_image_desc desc = {};
desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
desc.image_width = region.size / 16;
desc.buffer = k_sub;
cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
CL_CHECK(err);
region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
region.size = (size_t) K * N * D_B * sizeof(float);
cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
const cl_ulong off_mask = extra_m->offset + mask->view_offs + q0 * mask->nb[1];
const cl_ulong mask_nb1 = mask->nb[1];
const int n_head_log2 = 1u << (unsigned) floorf(log2f((float) D_B));
const float m0 = powf(2.0f, -max_bias / n_head_log2);
const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
cl_kernel kernel;
cl_uint a = 0;
cl_kernel kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
if (kq_int8) {
ggml_cl_fa_q8_rows(backend, q, dst, backend_ctx->prealloc_fa_q8.buffer, backend_ctx->prealloc_fa_q8_scale.buffer, q0, nqc);
kernel = K <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8_scale.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8_scale.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
size_t gws[3] = {64 * (((size_t) N + 31) / 32) * (D_B / D_A), ((size_t) M + 511) / 512, (size_t) D_A};
size_t lws[3] = {64, 1, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
} else {
cl_int err;
const size_t n_tiles = (N + 31) / 32;
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
cl_image_format fmt = {CL_RGBA, CL_FLOAT};
cl_image_desc desc = {};
desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
desc.image_width = region.size / 16;
desc.buffer = k_sub;
cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
CL_CHECK(err);
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
CL_CHECK(clReleaseMemObject(k_img));
CL_CHECK(clReleaseMemObject(k_sub));
CL_CHECK(clReleaseMemObject(q_sub));
region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
region.size = (size_t) K * N * D_B * sizeof(float);
cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
const size_t n_tiles = (N + 31) / 32;
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
CL_CHECK(clReleaseMemObject(k_img));
CL_CHECK(clReleaseMemObject(k_sub));
CL_CHECK(clReleaseMemObject(q_sub));
}
cl_mem sinks_buf = extra_q->data_device;
cl_ulong sinks_off = 0;
@@ -23261,7 +23402,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
ggml_backend_t backend,
const ggml_tensor * q, const ggml_tensor * k,
ggml_tensor * dst,
int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct
int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct, bool kq_int8
) {
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
const ggml_tensor * v = dst->src[2];
@@ -23345,13 +23486,17 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
}
if (kq_int8) {
ggml_cl_fa_kq_int8_prepare(backend, dst, n_q_chunk);
}
ggml_tensor_extra_cl extra_kqv;
ggml_tensor kqv;
for (int64_t q0 = 0; q0 < n_q; q0 += n_q_chunk) {
const int64_t nqc = MIN(n_q_chunk, n_q - q0);
ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch);
ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch, kq_int8);
ggml_cl_fa_decompose_scratch_tensor(kqv, extra_kqv, backend_ctx->prealloc_fa_kqv.buffer, GGML_TYPE_F32, dv, nqc, n_head);
@@ -23408,6 +23553,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
// head size. GGML_OPENCL_FA_DECOMPOSE_MB sets the KQ scratch budget (default 256).
// GGML_OPENCL_FA_KQ_P8=0 disables U8 probabilities and int8 KQV.
// GGML_OPENCL_FA_KQV_DIRECT=0 disables direct PV for small query batches.
// GGML_OPENCL_FA_KQ_INT8=0 keeps F16 KQ; =1 enables int8 KQ for supported shapes.
static bool ggml_cl_flash_attn_decompose_dispatch(
ggml_backend_t backend,
const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst
@@ -23537,12 +23683,46 @@ static bool ggml_cl_flash_attn_decompose_dispatch(
return false;
}
static const int kq_int8_env = [] {
const char * e = getenv("GGML_OPENCL_FA_KQ_INT8");
return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
}();
static const int dk512_env = [] {
const char * e = getenv("GGML_OPENCL_FA_KQ_DK512_F16");
return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
}();
const bool dk512 = dk > 256 && dk <= 512 && n_q >= 32 &&
(dk512_env == 1 || (dk512_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E));
const bool kq_int8_possible = kq_p8 && (dk <= 256 || dk512) && dk % 32 == 0 &&
backend_ctx->kernel_fa_q8_rows_f16 != nullptr && backend_ctx->kernel_fa_q8_rows_f32 != nullptr &&
n_head <= INT_MAX && n_head_kv <= INT_MAX &&
n_q <= INT_MAX / n_head / (dk / 32) && n_kv <= INT_MAX / n_head_kv / (dk / 32) &&
(dst->src[4] == nullptr || dk > 64) &&
(kq_int8_env == 1 || (kq_int8_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && n_kv >= 2048)) &&
backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f16) >= 64 &&
backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f32) >= 64;
cl_kernel kq_int8_kernel = nullptr;
if (kq_int8_possible) {
kq_int8_kernel = dk <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
}
const bool kq_int8 = kq_int8_kernel != nullptr && backend_ctx->get_kernel_workgroup_size(kq_int8_kernel) >= 64;
// Fewer query tiles per KV block reduce cache pressure in the int8 KQ kernel.
static const int64_t kq_int8_max_chunk = [] {
const char * e = getenv("GGML_OPENCL_FA_KQ_INT8_MAX_CHUNK");
const int v = (e && e[0]) ? atoi(e) : 96;
return (int64_t) (v >= 32 ? (v / 32) * 32 : 96);
}();
if (kq_int8) {
n_q_chunk = MIN(n_q_chunk, kq_int8_max_chunk);
}
// even out the chunks, then align to the 32-wide n-tile
const int64_t n_chunks = (n_q + n_q_chunk - 1) / n_q_chunk;
n_q_chunk = MIN(n_q_chunk, ((n_q + n_chunks - 1) / n_chunks + 31) & ~(int64_t) 31);
if (kq_p8) {
ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct);
ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct, kq_int8);
} else {
ggml_cl_flash_attn_decompose(backend, q, k, dst, n_q_chunk, scale, max_bias);
}
@@ -0,0 +1,77 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
// Store 32-element blocks as separate int8 values and half scales.
kernel void kernel_fa_q8_rows_f16(
global const char * src, ulong off_src, ulong nb1, ulong nb2,
global char * dq, global half * dd,
int ne0, int nrow, int nhead
) {
const int nblk = ne0 / 32;
const int gid = get_global_id(0);
if (gid >= nblk*nrow*nhead) {
return;
}
const int b = gid % nblk;
const int r = (gid / nblk) % nrow;
const int h = gid / (nblk*nrow);
global const half * p = (global const half *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
float v[32];
float amax = 0.0f;
#pragma unroll
for (int i = 0; i < 8; ++i) {
const float4 x = convert_float4(vload4(i, p));
v[i*4+0] = x.s0;
v[i*4+1] = x.s1;
v[i*4+2] = x.s2;
v[i*4+3] = x.s3;
amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
}
const float d = amax / 127.0f;
const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
const size_t row = (size_t)h*nrow + r;
dd[row*nblk + b] = (half)d;
global char * q = dq + row*ne0 + b*32;
#pragma unroll
for (int i = 0; i < 32; ++i) {
q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
}
}
kernel void kernel_fa_q8_rows_f32(
global const char * src, ulong off_src, ulong nb1, ulong nb2,
global char * dq, global half * dd,
int ne0, int nrow, int nhead
) {
const int nblk = ne0 / 32;
const int gid = get_global_id(0);
if (gid >= nblk*nrow*nhead) {
return;
}
const int b = gid % nblk;
const int r = (gid / nblk) % nrow;
const int h = gid / (nblk*nrow);
global const float * p = (global const float *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
float v[32];
float amax = 0.0f;
#pragma unroll
for (int i = 0; i < 8; ++i) {
const float4 x = vload4(i, p);
v[i*4+0] = x.s0;
v[i*4+1] = x.s1;
v[i*4+2] = x.s2;
v[i*4+3] = x.s3;
amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
}
const float d = amax / 127.0f;
const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
const size_t row = (size_t)h*nrow + r;
dd[row*nblk + b] = (half)d;
global char * q = dq + row*ne0 + b*32;
#pragma unroll
for (int i = 0; i < 32; ++i) {
q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
}
}
@@ -0,0 +1,174 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
#ifdef cl_khr_integer_dot_product
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
#endif
#define KQ_TN 32
#define KQ_MB 8
#define KQ_TM 64
#define KQ_WG 64
#define KQ_DK_MAX 256
__attribute__((qcom_wave_pair_mode(1)))
kernel void kernel_mul_mm_q8_kq_p8(
global const uint * kq, // K int8, [head_kv][n_kv][dk] as packed uints
global const half * kd, // K scales, [head_kv][n_kv][dk/32]
global const uint * qq, // Q int8, [head][n_q][dk] as packed uints
global const half * qd, // Q scales, [head][n_q][dk/32]
global uchar * qp, // u8 P, [head][n_q][p_pitch]
global float * bmax, // per 32-row block max, [head][n_q][p_pitch/32]
global float * bsum, // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
global const char * mask, // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
ulong offset_mask,
ulong mask_nb1,
float scale,
float max_bias,
float m0,
float m1,
int n_head_log2,
int dk,
int n_kv,
int n_q,
int n_head,
int n_head_kv,
int p_pitch // bytes per P row, >= n_kv, multiple of 32
) {
const int lid = get_local_id(0);
const int gsz = n_head / n_head_kv;
const int g0 = get_group_id(0);
const int head_kv = get_group_id(2);
const int head = head_kv*gsz + (g0 % gsz);
const int qn0 = (g0 / gsz) * KQ_TN;
const int kv0 = get_group_id(1)*KQ_TM*KQ_MB + lid;
const int nblk = dk / 32;
const int nu = dk / 4;
__local uint sh_q [KQ_TN][KQ_DK_MAX/4];
__local half sh_qd[KQ_TN][KQ_DK_MAX/32];
__local float p8_lds[1024];
const size_t qbase = (size_t)head*n_q;
for (int i = lid; i < KQ_TN*(KQ_DK_MAX/16); i += KQ_WG) {
const int t = i / (KQ_DK_MAX/16);
const int u = (i % (KQ_DK_MAX/16)) * 4;
const int qi = qn0 + t;
uint4 v = (uint4)(0u);
if (qi < n_q && u < nu) {
v = vload4(0, &qq[(qbase + qi)*nu + u]);
}
vstore4(v, 0, &sh_q[t][u]);
}
for (int i = lid; i < KQ_TN*(KQ_DK_MAX/32); i += KQ_WG) {
const int t = i / (KQ_DK_MAX/32);
const int b = i % (KQ_DK_MAX/32);
const int qi = qn0 + t;
sh_qd[t][b] = (qi < n_q && b < nblk) ? qd[(qbase + qi)*nblk + b] : (half)0.0f;
}
barrier(CLK_LOCAL_MEM_FENCE);
float slope = 1.0f;
if (max_bias > 0.0f) {
const float base = head < n_head_log2 ? m0 : m1;
const int ex = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
slope = pow(base, ex);
}
global const half * mrow = (global const half *)(mask + offset_mask);
const uint mstride = (uint)(mask_nb1 / 2);
const int pblk = p_pitch / 32;
for (int m = 0; m < KQ_MB; ++m) {
const int kv = kv0 + m*KQ_TM;
if (kv - lid >= n_kv) {
break; // uniform across the workgroup
}
float acc[KQ_TN];
#pragma unroll
for (int t = 0; t < KQ_TN; ++t) {
acc[t] = 0.0f;
}
const int kl = min(kv, n_kv - 1);
const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
for (int b = 0; b < nblk; ++b) {
const uint4 w0 = vload4(0, &kq[kbase + b*8]);
const uint4 w1 = vload4(0, &kq[kbase + b*8 + 4]);
const float dks = (float)kd[kdbas + b];
#pragma unroll
for (int t = 0; t < KQ_TN; ++t) {
const uint4 a0 = vload4(0, &sh_q[t][b*8]);
const uint4 a1 = vload4(0, &sh_q[t][b*8 + 4]);
int raw = 0;
raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
}
}
// Each lane owns one complete 32-row softmax block after the transpose.
const int kvblk0 = kv - lid;
#pragma unroll
for (int half_ = 0; half_ < 2; ++half_) {
barrier(CLK_LOCAL_MEM_FENCE);
#pragma unroll
for (int j = 0; j < 16; ++j) {
p8_lds[j*64 + lid] = acc[half_*16 + j];
}
barrier(CLK_LOCAL_MEM_FENCE);
if (lid < 32) {
const int j = lid >> 1;
const int b = lid & 1;
const int n = qn0 + half_*16 + j;
if (n < n_q) {
const int kvb = kvblk0 + b*32;
const __local float * src = p8_lds + j*64 + b*32;
float16 s0 = vload16(0, src);
float16 s1 = vload16(1, src);
const global half * mp = mrow + (size_t)n*mstride + kvb;
const float16 m0v = convert_float16(vload16(0, mp));
const float16 m1v = convert_float16(vload16(1, mp));
s0 = s0*scale + slope*m0v;
s1 = s1*scale + slope*m1v;
const float16 mx16 = fmax(s0, s1);
const float8 mx8 = fmax(mx16.lo, mx16.hi);
const float4 mx4 = fmax(mx8.lo, mx8.hi);
const float2 mx2 = fmax(mx4.lo, mx4.hi);
const float amax = fmax(mx2.x, mx2.y);
const size_t prow = qbase + n;
const int blk = kvb >> 5;
float sum = 0.0f;
uchar16 q0 = (uchar16)(0);
uchar16 q1 = (uchar16)(0);
// finite sentinel: the program builds with -cl-finite-math-only
if (amax > -1.0e30f) {
const float16 e0 = exp(s0 - amax);
const float16 e1 = exp(s1 - amax);
const float16 a16 = e0 + e1;
const float8 a8 = a16.lo + a16.hi;
const float4 a4 = a8.lo + a8.hi;
const float2 a2 = a4.lo + a4.hi;
sum = a2.x + a2.y;
q0 = convert_uchar16_sat_rte(e0*255.0f);
q1 = convert_uchar16_sat_rte(e1*255.0f);
}
global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
vstore16(q0, 0, qdst);
vstore16(q1, 1, qdst);
bmax[prow*pblk + blk] = amax;
bsum[prow*pblk + blk] = sum;
}
}
}
}
}
@@ -0,0 +1,170 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
#ifdef cl_khr_integer_dot_product
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
#endif
#define KQ_TN 32
#define KQ_MB 8
#define KQ_TM 64
#define KQ_WG 64
#define KQ_DKC 256
__attribute__((qcom_wave_pair_mode(1)))
kernel void kernel_mul_mm_q8_kq_p8_dk512(
global const uint * kq, // K int8, [head_kv][n_kv][dk] as packed uints
global const half * kd, // K scales, [head_kv][n_kv][dk/32]
global const uint * qq, // Q int8, [head][n_q][dk] as packed uints
global const half * qd, // Q scales, [head][n_q][dk/32]
global uchar * qp, // u8 P, [head][n_q][p_pitch]
global float * bmax, // per 32-row block max, [head][n_q][p_pitch/32]
global float * bsum, // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
global const char * mask, // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
ulong offset_mask,
ulong mask_nb1,
float scale,
float max_bias,
float m0,
float m1,
int n_head_log2,
int dk,
int n_kv,
int n_q,
int n_head,
int n_head_kv,
int p_pitch // bytes per P row, >= n_kv, multiple of 32
) {
const int lid = get_local_id(0);
const int gsz = n_head / n_head_kv;
const int g0 = get_group_id(0);
const int head_kv = get_group_id(2);
const int head = head_kv*gsz + (g0 % gsz);
const int qn0 = (g0 / gsz) * KQ_TN;
const int kv0 = get_group_id(1)*KQ_TM*KQ_MB + lid;
const int nblk = dk / 32;
const int nu = dk / 4;
// Reuse the 256-wide Q slice for the F32 softmax scores after the contraction.
__local uint sh_q [KQ_TN*KQ_DKC/4];
__local half sh_qd[KQ_TN][KQ_DKC/32];
__local float * p8_lds = (__local float *)sh_q;
const size_t qbase = (size_t)head*n_q;
float slope = 1.0f;
if (max_bias > 0.0f) {
const float base = head < n_head_log2 ? m0 : m1;
const int ex = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
slope = pow(base, ex);
}
global const half * mrow = (global const half *)(mask + offset_mask);
const uint mstride = (uint)(mask_nb1 / 2);
const int pblk = p_pitch / 32;
for (int m = 0; m < KQ_MB; ++m) {
const int kv = kv0 + m*KQ_TM;
if (kv - lid >= n_kv) {
break; // uniform across the workgroup
}
float acc[KQ_TN];
#pragma unroll
for (int t = 0; t < KQ_TN; ++t) {
acc[t] = 0.0f;
}
const int kl = min(kv, n_kv - 1);
const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
for (int c0 = 0; c0 < nblk; c0 += KQ_DKC/32) {
const int cb = min(KQ_DKC/32, nblk - c0);
barrier(CLK_LOCAL_MEM_FENCE);
for (int i = lid; i < KQ_TN*(KQ_DKC/16); i += KQ_WG) {
const int t = i / (KQ_DKC/16);
const int u = (i % (KQ_DKC/16)) * 4;
const int qi = qn0 + t;
uint4 v = (uint4)(0u);
if (qi < n_q && u < cb*8) {
v = vload4(0, &qq[(qbase + qi)*nu + c0*8 + u]);
}
vstore4(v, 0, &sh_q[t*(KQ_DKC/4) + u]);
}
for (int i = lid; i < KQ_TN*(KQ_DKC/32); i += KQ_WG) {
const int t = i / (KQ_DKC/32);
const int b = i % (KQ_DKC/32);
const int qi = qn0 + t;
sh_qd[t][b] = (qi < n_q && b < cb) ? qd[(qbase + qi)*nblk + c0 + b] : (half)0.0f;
}
barrier(CLK_LOCAL_MEM_FENCE);
for (int b = 0; b < cb; ++b) {
const uint4 w0 = vload4(0, &kq[kbase + (c0 + b)*8]);
const uint4 w1 = vload4(0, &kq[kbase + (c0 + b)*8 + 4]);
const float dks = (float)kd[kdbas + c0 + b];
#pragma unroll
for (int t = 0; t < KQ_TN; ++t) {
const uint4 a0 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8]);
const uint4 a1 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8 + 4]);
int raw = 0;
raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
}
}
}
// Finish reading the Q slice before writing scores to the same local memory.
const int kvblk0 = kv - lid;
barrier(CLK_LOCAL_MEM_FENCE);
#pragma unroll
for (int t = 0; t < KQ_TN; ++t) {
p8_lds[t*64 + lid] = acc[t];
}
barrier(CLK_LOCAL_MEM_FENCE);
{
const int j = lid >> 1;
const int b = lid & 1;
const int n = qn0 + j;
if (n < n_q) {
const int kvb = kvblk0 + b*32;
__local float * src = p8_lds + j*64 + b*32;
const global half * mp = mrow + (size_t)n*mstride + kvb;
float4 s = vload4(0, src)*scale + slope*convert_float4(vload4(0, mp));
vstore4(s, 0, src);
float amax = fmax(fmax(s.x, s.y), fmax(s.z, s.w));
for (int c = 1; c < 8; ++c) {
s = vload4(c, src)*scale + slope*convert_float4(vload4(c, mp));
vstore4(s, c, src);
amax = fmax(amax, fmax(fmax(s.x, s.y), fmax(s.z, s.w)));
}
const size_t prow = qbase + n;
const int blk = kvb >> 5;
global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
float sum = 0.0f;
// finite sentinel: the program builds with -cl-finite-math-only
if (amax > -1.0e30f) {
for (int c = 0; c < 8; ++c) {
const float4 e = exp(vload4(c, src) - amax);
sum += (e.x + e.y) + (e.z + e.w);
vstore4(convert_uchar4_sat_rte(e*255.0f), c, qdst);
}
} else {
for (int c = 0; c < 8; ++c) {
vstore4((uchar4)(0), c, qdst);
}
}
bmax[prow*pblk + blk] = amax;
bsum[prow*pblk + blk] = sum;
}
}
}
}
+6
View File
@@ -1766,6 +1766,9 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
!GgmlOvDecoder::is_conv_state_snapshot_writeback(op)) {
return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"};
}
if (op->op == GGML_OP_SET_ROWS && op->src[0]->type == GGML_TYPE_BF16) {
return {false, "SET_ROWS with BF16 src0 is not supported"};
}
if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K ||
op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_1)) {
// These are all f16-arithmetic dequant rounding errors that intermittently exceed the
@@ -2035,6 +2038,9 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
if (op->view_src != nullptr && !ggml_is_contiguous(op->src[0])) {
return {false, "ROPE on VIEW / non-contiguous input is not supported"};
}
if (op->type == GGML_TYPE_BF16) {
return {false, "ROPE with BF16 type is not supported"};
}
break;
}
case GGML_OP_TRANSPOSE: {
+2 -2
View File
@@ -7434,8 +7434,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
if (op->type == GGML_TYPE_TQ2_0 || op->type == GGML_TYPE_TQ1_0) {
return false;
}
auto res = (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16 ||
op->src[0]->type == GGML_TYPE_BF16) &&
auto res = (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
(op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32);
return res;
}
@@ -7624,6 +7623,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons
}
case GGML_OP_ROPE:
case GGML_OP_ROPE_BACK:
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
case GGML_OP_IM2COL:
case GGML_OP_IM2COL_3D:
case GGML_OP_UPSCALE:
+2 -1
View File
@@ -16729,8 +16729,9 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
case GGML_OP_REPEAT_BACK:
return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32;
case GGML_OP_ROPE:
return ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0]);
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && ggml_is_contiguous_rows(op) && ggml_is_contiguous_rows(op->src[0]);
case GGML_OP_ROPE_BACK:
return op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16;
case GGML_OP_NONE:
case GGML_OP_RESHAPE:
case GGML_OP_VIEW:
+1 -1
View File
@@ -4019,7 +4019,7 @@ struct ggml_tensor * ggml_set_rows(
GGML_ASSERT(b->ne[2] % c->ne[1] == 0);
GGML_ASSERT(b->ne[3] % c->ne[2] == 0);
GGML_ASSERT(c->ne[3] == 1);
GGML_ASSERT(b->type == GGML_TYPE_F32 || b->type == GGML_TYPE_F16);
GGML_ASSERT(b->type == GGML_TYPE_F32 || b->type == GGML_TYPE_F16 || b->type == GGML_TYPE_BF16);
GGML_ASSERT(c->type == GGML_TYPE_I64 || c->type == GGML_TYPE_I32);
GGML_ASSERT(ggml_is_contiguous_rows(a));
+11 -2
View File
@@ -1,5 +1,7 @@
#include "models.h"
#include <cinttypes>
// HRM-Text: alternating low/high transformer stacks over the same token stream.
// Reference: HrmTextModel in transformers, DFM Mimir 1B.
@@ -19,8 +21,15 @@ void llama_model_hrm_text::load_arch_hparams(llama_model_loader & ml) {
GGML_ASSERT(hparams.n_hrm_l_cycles > 0);
// the GGUF block count is the expanded cache-slot count
const uint32_t n_slot = hparams.n_hrm_layers_per_stack * hparams.n_hrm_h_cycles * (hparams.n_hrm_l_cycles + 1);
GGML_ASSERT(hparams.n_layer() == n_slot);
// 64-bit: a 32-bit product can wrap and match n_layer(), then load_arch_tensors() indexes past model.layers
const uint64_t n_slot = (uint64_t) hparams.n_hrm_layers_per_stack * hparams.n_hrm_h_cycles * (hparams.n_hrm_l_cycles + 1);
if (hparams.n_hrm_layers_per_stack >= hparams.n_layer() || n_slot != hparams.n_layer()) {
throw std::runtime_error(format(
"hrm: cache-slot count %" PRIu64 " (layers_per_stack %u, h_cycles %u, l_cycles %u) "
"does not match block_count %u",
n_slot, hparams.n_hrm_layers_per_stack, hparams.n_hrm_h_cycles,
hparams.n_hrm_l_cycles, hparams.n_layer()));
}
switch (hparams.n_embd) {
case 1536:
+57 -9
View File
@@ -2830,6 +2830,12 @@ struct test_set_rows : public test_case {
if (type_src == GGML_TYPE_F16) {
err_estimate *= 16.0f;
}
if (type_src == GGML_TYPE_BF16) {
// BF16 values have 8 significant bits, so many of them land exactly on a rounding tie,
// and backends break ties differently (e.g. the CPU q8_0 rounds half to even, CUDA half away from zero).
// Allow 1 in 32 values to be off by one instead of one value in total.
return err_estimate / (0.25f*32.0f);
}
err_estimate /= 0.25f*float(ne[0] * r * ne[2]*nr23[0] * ne[3]*nr23[1]);
return err_estimate;
}
@@ -2860,10 +2866,15 @@ struct test_rope_set_rows : public test_case {
const int n_dims;
const bool yarn;
const bool truncate;
const ggml_type type_a; // type of the rope input
const int n_ctx{512};
std::string vars() override {
return VARS_TO_STR7(type, type_idx, ne_a, mode, n_dims, yarn, truncate);
std::string s = VARS_TO_STR7(type, type_idx, ne_a, mode, n_dims, yarn, truncate);
if (type_a != GGML_TYPE_F32) {
s += "," + VAR_TO_STR(type_a);
}
return s;
}
std::string op_desc(ggml_tensor * t) override {
@@ -2871,17 +2882,27 @@ struct test_rope_set_rows : public test_case {
return "ROPE_SET_ROWS";
}
double max_nmse_err() override {
if (type_a != GGML_TYPE_BF16) {
return 1e-7;
}
// into F16 the reference rounds the BF16 rope result to BF16 first, the fused rope + set_rows does not
// otherwise the limit of a BF16 test_rope: the angle error n_ctx*FLT_EPSILON can flip a BF16 rounding
return type == GGML_TYPE_F16 ? 1e-5 : n_ctx * FLT_EPSILON / 128.0;
}
bool run_whole_graph() override { return true; }
test_rope_set_rows(ggml_type type,
ggml_type type_idx,
std::array<int64_t, 4> ne_a,
int mode,
int n_dims = 128, bool yarn = false, bool truncate = true)
: type(type), type_idx(type_idx), ne_a(ne_a), mode(mode), n_dims(n_dims), yarn(yarn), truncate(truncate) {}
int n_dims = 128, bool yarn = false, bool truncate = true,
ggml_type type_a = GGML_TYPE_F32)
: type(type), type_idx(type_idx), ne_a(ne_a), mode(mode), n_dims(n_dims), yarn(yarn), truncate(truncate), type_a(type_a) {}
ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne_a[0], ne_a[1], ne_a[2], 1);
ggml_tensor * a = ggml_new_tensor_4d(ctx, type_a, ne_a[0], ne_a[1], ne_a[2], 1);
ggml_set_name(a, "a");
const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE;
@@ -6793,6 +6814,10 @@ struct test_rope : public test_case {
// the f32 rotation angle of a position below n_ctx carries a rounding error of up to n_ctx*FLT_EPSILON radians
// backends compute the angle with different rounding sequences, so their agreement is bounded by that error
const double eps_theta = n_ctx * FLT_EPSILON;
if (type == GGML_TYPE_BF16) {
// with BF16 output that error can also change the rounding: a step of 2^-7 with a chance of about eps_theta/2^-7
return std::max(test_case::max_nmse_err(), eps_theta / 128.0);
}
return std::max(test_case::max_nmse_err(), eps_theta * eps_theta);
}
@@ -10895,6 +10920,12 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, true));
test_cases.emplace_back(new test_set_rows(GGML_TYPE_F16, GGML_TYPE_F16, GGML_TYPE_I32, { 1, 8, 1, 3 }, { 1, 1 }, 2, true));
// BF16 source: one case per type covers batch dims, broadcast indices and gaps between rows
for (ggml_type type : all_types) {
test_cases.emplace_back(new test_set_rows(GGML_TYPE_BF16, type, GGML_TYPE_I64, { 256, 11, 1, 1 }, { 2, 3, }, 7, true));
}
test_cases.emplace_back(new test_set_rows(GGML_TYPE_BF16, GGML_TYPE_Q8_0, GGML_TYPE_I32, { 256, 5, 1, 3 }, { 1, 1, }, 1, false));
test_cases.emplace_back(new test_set_rows(GGML_TYPE_BF16, GGML_TYPE_F16, GGML_TYPE_I64, { 1, 8, 1, 3 }, { 1, 1 }, 2, false));
for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_VISION, GGML_ROPE_TYPE_IMROPE }) {
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
@@ -10905,11 +10936,19 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
test_cases.emplace_back(new test_rope_set_rows(GGML_TYPE_F32, GGML_TYPE_I32, { 128, 32, 8, 1 }, GGML_ROPE_TYPE_IMROPE));
for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_VISION, GGML_ROPE_TYPE_IMROPE }) {
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_BF16}) {
for (int ne2 : {1, 8, 512}) {
test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, { 128, 32, ne2, 1 }, mode, 128, false, true, GGML_TYPE_BF16));
}
}
}
// partial rotation
for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE }) {
for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) {
test_cases.emplace_back(new test_rope_set_rows(type, GGML_TYPE_I64, { 256, 4, 8, 1 }, mode, 64));
}
test_cases.emplace_back(new test_rope_set_rows(GGML_TYPE_F16, GGML_TYPE_I64, { 256, 4, 8, 1 }, mode, 64, false, true, GGML_TYPE_BF16));
}
for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX}) {
@@ -12368,6 +12407,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, b, 50, 200, 64));
}
// One very large one to trigger a code path in CUDA requiring >1 pass over expert IDs due to shared memory limits:
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 16, 16, false, 32, 65536, 256));
// For issue 27873
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_IQ2_XXS, GGML_TYPE_F32, 1, 1, false, 1, 8192, 4096));
@@ -12402,8 +12444,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 28, 10, false, 1024, 1, 4096));
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 128, 8, false, 1024, 1, 2048));
// TODO: disabled until CUDA backend support is added [TAG_MUL_MAT_ID_DUPLICATE_ID]
#if 0
// repeated expert ids within a row
for (ggml_type ta : {GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) {
for (int n : {9, 16, 33, 64}) {
@@ -12415,7 +12455,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 1025, 10, false, 64, 33, 256, 0, true));
test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 2050, 10, true, 64, 33, 256));
}
#endif
for (ggml_type type_a : all_types) {
test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 2, false, 64, 16, 3*ggml_blck_size(type_a)));
@@ -12716,8 +12755,17 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}
}
// BF16: each mode forward on a strided view and backward on an offset view, with and without freq factors, and one in place
for (int mode : {GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE, GGML_ROPE_TYPE_VISION}) {
for (bool ff : {false, true}) {
test_cases.emplace_back(new test_rope(GGML_TYPE_BF16, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, ff, 1, true));
test_cases.emplace_back(new test_rope(GGML_TYPE_BF16, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, ff, 2, false));
}
test_cases.emplace_back(new test_rope(GGML_TYPE_BF16, {128, 32, 2, 1}, 128, mode, 512, 1.4245f, 0.7465f, 1.4245f, false, 0, true, true));
}
// rotated dims window at an offset (ggml_rope_set_offset), not supported for vision mode
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
for (bool fw : {true, false}) { // fw == forward
for (bool ff : {false, true}) {
test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 32, GGML_ROPE_TYPE_NORMAL, 512, 1.4245f, 0.7465f, 1.4245f, ff, 0, fw, false, 32));
@@ -14108,7 +14156,7 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_perf() {
}
for (bool fw : {true, false}) { // fw == forward
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16}) {
for (ggml_type type : {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16}) {
for (bool ff : {false, true}) { // freq_factors
for (float v : { 0, 1 }) {
test_cases.emplace_back(new test_rope(type, {128, 32, 512, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, 1.0f, 0.0f, 1.0f, ff, v, fw)); // llama 7B
+1
View File
@@ -644,6 +644,7 @@ static void mtmd_owned_llama_set_tensor_data(ggml_tensor * t, void * user_data)
return name.size() >= n && name.compare(name.size() - n, n, suffix) == 0;
};
const bool optional = ends_with("scale") || ends_with(".bias") || name == "dense_2.weight" ||
name == "cls.output.weight" || // Qwen3 reranking head, unused by YuE2 NAR
name == "rope_freqs.weight" || // granite, voxcpm2 residual LM
name == "token_embd.weight" || name == "output.weight"; // inputs are embedding rows, outputs are hidden states
if (!optional || t->type != GGML_TYPE_F32) {