Compare commits

...
3 Commits
Author SHA1 Message Date
Johannes Gäßler b94959a64d CUDA: implement support for duplicate expert IDs (#32505)
implement support for duplicate expert IDs
2026-11-19 23:21:24 +01:00
Imdadullah Mohammed 50518b64c6 model : hrm : do not compute slot count as uint32_t (#32601)
load_arch_hparams multiplies n_hrm_layers_per_stack, n_hrm_h_cycles and
(n_hrm_l_cycles + 1) into a uint32_t, then asserts the product equals
block_count.

Any factors whose true product is block_count + k*2^32 still pass the
assert; 1073741825 * 2 * 2 = 2^32 + 4 compares equal to block_count = 4.

load_arch_tensors iterates the same three values in int and writes past
the end of the block_count-sized layers vector.
2026-11-19 23:21:03 +01:00
c406f4367e opencl: support dp4a for KQ mm in decomposed FA (#32577)
* opencl: support int8 KQ for DK = 512

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
Assisted-by: Codex

* opencl: allow query chunk up to 96 for int8 KQ

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
Assisted-by: Codex

* opencl: extend `build_program_from_source` to allow non-fatal failure

Assisted-by: Codex

* opencl: add int8 KQ for decomposed FA with DK <= 256

Co-authored-by: Shaofei Qi <shaoqi@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
Assisted-by: Codex

---------

Co-authored-by: Li He <lih@qti.qualcomm.com>
Co-authored-by: Hongqiang Wang <wangh@qti.qualcomm.com>
2026-11-19 11:30:19 -08:00
15 changed files with 925 additions and 332 deletions
-6
View File
@@ -1316,12 +1316,6 @@ extern "C" {
// indirect matrix multiplication
// an id of -1 skips the slot and zeroes the matching row of the result
//
// [TAG_MUL_MAT_ID_DUPLICATE_ID]
// note: this op has undefined behaviour if a given `a` is referenced more than once for a given `b`
// (i.e. duplicate experts per token are not guaranteed to be supported in all backends)
// TODO: add backend support: https://github.com/ggml-org/llama.cpp/pull/29998#issuecomment-6153742533
// the CPU backend skips negative ids, their rows of the result are zero
GGML_API struct ggml_tensor * ggml_mul_mat_id(
struct ggml_context * ctx,
struct ggml_tensor * as,
+10
View File
@@ -452,6 +452,16 @@ static __device__ __forceinline__ ggml_cuda_thread_mask_t ggml_cuda_ballot(int p
return __ballot_sync(mask, predicate);
}
static __device__ __forceinline__ int ggml_cuda_popc(ggml_cuda_thread_mask_t x) {
static_assert(sizeof(ggml_cuda_thread_mask_t) == 4 || sizeof(ggml_cuda_thread_mask_t) == 8, "unsupported thread mask type");
if constexpr (sizeof(ggml_cuda_thread_mask_t) == 4) {
return __popc(x);
} else {
return __popcll(x);
}
}
template<typename T> static __device__ __forceinline__ int ggml_cuda_popc(T x) = delete; // Makes implicit type casts result in an error.
static __device__ __forceinline__ int ggml_cuda_all(int predicate, ggml_cuda_thread_mask_t mask = GGML_CUDA_FULL_THREAD_MASK) {
return __all_sync(mask, predicate);
}
+7 -16
View File
@@ -17,15 +17,6 @@ static __global__ void flash_attn_mask_to_sparse_indices(
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
constexpr int values_per_lane = 8;
using lane_mask_t = std::conditional_t<warp_size == 64, uint64_t, uint32_t>;
constexpr uint64_t ballot_mask = ~(lane_mask_t)0;
const auto popc = [](const lane_mask_t mask) {
if constexpr (warp_size == 64) {
return __popcll(mask);
} else {
return __popc(mask);
}
};
const int tid = threadIdx.x;
const int warp = tid / warp_size;
const int lane = tid % warp_size;
@@ -48,7 +39,7 @@ static __global__ void flash_attn_mask_to_sparse_indices(
__syncthreads();
for (int i0 = 0; i0 < ne30; i0 += blockDim.x*values_per_lane) {
lane_mask_t selected_warp[values_per_lane];
ggml_cuda_thread_mask_t selected_warp[values_per_lane];
int warp_count = 0;
#pragma unroll
for (int item = 0; item < values_per_lane; ++item) {
@@ -60,8 +51,8 @@ static __global__ void flash_attn_mask_to_sparse_indices(
selected |= (!oob || q < q1 - q0) && isfinite(__half2float(mask[q*s31 + i]));
}
}
selected_warp[item] = (lane_mask_t)ggml_cuda_ballot((int) selected, ballot_mask);
warp_count += popc(selected_warp[item]);
selected_warp[item] = ggml_cuda_ballot((int) selected);
warp_count += ggml_cuda_popc(selected_warp[item]);
}
if (lane == 0) {
@@ -81,16 +72,16 @@ static __global__ void flash_attn_mask_to_sparse_indices(
}
__syncthreads();
const lane_mask_t lane_mask = lane == 0 ? 0 : ((lane_mask_t)1 << lane) - 1;
const ggml_cuda_thread_mask_t lane_mask = lane == 0 ? 0 : ((ggml_cuda_thread_mask_t)1 << lane) - 1;
int warp_item_offset = 0;
#pragma unroll
for (int item = 0; item < values_per_lane; ++item) {
const int i = i0 + (warp*values_per_lane + item)*warp_size + lane;
const int dst = row_count + warp_offsets[warp] + warp_item_offset + popc(selected_warp[item] & lane_mask);
if ((selected_warp[item] & ((lane_mask_t)1 << lane)) && dst < n_kv_max) {
const int dst = row_count + warp_offsets[warp] + warp_item_offset + ggml_cuda_popc(selected_warp[item] & lane_mask);
if ((selected_warp[item] & ((ggml_cuda_thread_mask_t)1 << lane)) && dst < n_kv_max) {
indices[dst] = i;
}
warp_item_offset += popc(selected_warp[item]);
warp_item_offset += ggml_cuda_popc(selected_warp[item]);
}
__syncthreads();
+2 -1
View File
@@ -2730,7 +2730,8 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
ids_from_sorted_host[i12*n_expert_used + iex] = ids_to_sorted_host.size();
ids_to_sorted_host.push_back(i12*ne11 + iex % ne11);
tokens_per_expert[i02]++;
break;
// TODO: removing the break here was done to make duplicate IDs work but this caused a ~10% performance regression, better solution needed.
// break;
}
}
}
+4 -5
View File
@@ -43,7 +43,6 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
const int64_t s13 = src1->nb[3] / ts_src1;
const int64_t s3 = dst->nb[3] / ts_dst;
const int64_t ids_s0 = ids ? ids->nb[0] / ggml_type_size(ids->type) : 0;
const int64_t ids_s1 = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;
mmf_ids_data ids_info{};
@@ -70,7 +69,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
nchannels_y = ids->ne[0];
}
if (ids && ncols_dst > 16) {
if (ids) {
const int64_t n_expert_used = ids->ne[0];
const int64_t n_experts = ne02;
const int64_t n_tokens = ne12;
@@ -110,7 +109,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
constexpr int vals_per_T = 1;
mul_mat_f_switch_rows_per_block<float>(
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
} break;
case GGML_TYPE_F16: {
@@ -118,7 +117,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
constexpr int vals_per_T = 2;
mul_mat_f_switch_rows_per_block<half2>(
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
} break;
case GGML_TYPE_BF16: {
@@ -126,7 +125,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
constexpr int vals_per_T = 2;
mul_mat_f_switch_rows_per_block<nv_bfloat162>(
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
} break;
default:
+69 -176
View File
@@ -47,12 +47,11 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
bool ggml_cuda_should_use_mmf(enum ggml_type type, int cc, int warp_size, const int64_t * scr0_ne, const size_t * src0_nb, const int src1_ncols, bool mul_mat_id);
template <typename T, int rows_per_block, int cols_per_block, int nwarps, bool has_ids>
template <typename T, int rows_per_block, int cols_per_block, int nwarps>
__launch_bounds__(ggml_cuda_get_physical_warp_size()*nwarps, 1)
static __global__ void mul_mat_f(
const T * __restrict__ x, const float * __restrict__ y, const int32_t * __restrict__ ids, float * __restrict__ dst,
const int ncols, const int ncols_dst_total, const int nchannels_dst, const int stride_row, const int stride_col_y, const int stride_col_dst,
const int stride_col_id, const int stride_row_id,
const T * __restrict__ x, const float * __restrict__ y, float * __restrict__ dst,
const int ncols, const int stride_row, const int stride_col_y, const int stride_col_dst,
const int channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
const int sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst) {
// TODO: handle this in a consistent and simpler way after AMD MFMA support has been added
@@ -90,85 +89,26 @@ static __global__ void mul_mat_f(
constexpr int ntA = rows_per_block / tile_A::I;
constexpr int ntB = (cols_per_block + tile_B::I - 1) / tile_B::I;
const int row0 = blockIdx.x * rows_per_block;
const int row0 = blockIdx.x * rows_per_block;
int expert_idx = 0;
[[maybe_unused]] int col_base = 0;
const int channel_dst = has_ids ? 0 : blockIdx.y;
if constexpr (has_ids) {
// experts + tiles of ncols_dst are packed in the y dimension
int col_tiles = (ncols_dst_total + cols_per_block - 1) / cols_per_block;
const int nchannels_x = gridDim.y / col_tiles;
const int tile_idx = blockIdx.y / nchannels_x;
expert_idx = blockIdx.y - tile_idx * nchannels_x;
col_base = tile_idx * cols_per_block;
}
const int channel_x = has_ids ? expert_idx : (channel_dst / channel_ratio);
const int channel_dst = blockIdx.y;
const int channel_x = channel_dst / channel_ratio;
const int channel_y = channel_dst;
const int sample_dst = blockIdx.z;
const int sample_x = sample_dst / sample_ratio;
const int sample_y = sample_dst;
x += int64_t(sample_x) *stride_sample_x + channel_x *stride_channel_x + row0*stride_row ;
y += int64_t(sample_y) *stride_sample_y + (has_ids ? 0 : channel_y *stride_channel_y);
dst += int64_t(sample_dst)*stride_sample_dst + (has_ids ? 0 : channel_dst*stride_channel_dst);
if constexpr (has_ids) {
constexpr int y_stride_scale = std::is_same_v<T, float> ? 1 : 2;
const int64_t col_offset = col_base;
y += col_offset * stride_col_y * y_stride_scale;
dst += col_offset * stride_col_dst;
ids += col_offset * stride_row_id;
}
y += int64_t(sample_y) *stride_sample_y + channel_y *stride_channel_y;
dst += int64_t(sample_dst)*stride_sample_dst + channel_dst*stride_channel_dst;
[[maybe_unused]] const float2 * y2 = (const float2 *) y;
extern __shared__ char data_mmv[];
char * shmem_base = data_mmv;
[[maybe_unused]] int * slot_map = (int *) shmem_base;
char * compute_base = has_ids ? (shmem_base + GGML_PAD(cols_per_block, 16) * sizeof(int)) : shmem_base;
tile_C C[ntA][ntB];
T * tile_xy = (T *) compute_base + threadIdx.y*(tile_A::I * tile_k_padded);
if constexpr (has_ids) {
int found = 0;
for (int j0 = 0; j0 < cols_per_block; j0 += nwarps) {
const int j = j0 + threadIdx.y;
if (threadIdx.x == 0) {
slot_map[j] = -1;
}
ggml_cuda_syncwarp();
if (col_base + j >= ncols_dst_total) {
continue;
}
const int32_t * __restrict__ id_row = ids + j*stride_row_id;
for (int k = threadIdx.x; k < nchannels_dst; k += warp_size) {
int match = id_row[k*stride_col_id] == expert_idx;
if (match) {
slot_map[j] = k;
found = 1;
break;
}
}
}
if (!__syncthreads_or(found)) {
return;
}
}
T * tile_xy = (T *) data_mmv + threadIdx.y*(tile_A::I * tile_k_padded);
for (int col = threadIdx.y*warp_size + threadIdx.x; col < ncols; col += nwarps*warp_size) {
tile_A A[ntA][warp_size / tile_A::J];
@@ -194,26 +134,15 @@ static __global__ void mul_mat_f(
for (int j0 = 0; j0 < tile_B::I; ++j0) {
const int j = j0 + itB*tile_B::I;
if constexpr (!has_ids) {
tile_xy[j0*tile_k_padded + threadIdx.x] = j < cols_per_block ? y[j*stride_col_y + col] : 0.0f;
} else {
const bool valid = j < cols_per_block && (col_base + j) < ncols_dst_total && slot_map[j] >= 0;
tile_xy[j0*tile_k_padded + threadIdx.x] = valid ? y[slot_map[j]*stride_channel_y + j*stride_col_y + col] : 0.0f;
}
tile_xy[j0*tile_k_padded + threadIdx.x] = j < cols_per_block ? y[j*stride_col_y + col] : 0.0f;
}
} else if constexpr (std::is_same_v<T, half2> || std::is_same_v<T, nv_bfloat162>) {
#pragma unroll
for (int j0 = 0; j0 < tile_B::I; ++j0) {
const int j = j0 + itB*tile_B::I;
if constexpr (!has_ids) {
const float2 tmp = j < cols_per_block ? y2[j*stride_col_y + col] : make_float2(0.0f, 0.0f);
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
} else {
const bool valid = j < cols_per_block && (col_base + j) < ncols_dst_total && slot_map[j] >= 0;
float2 tmp = valid ? *(const float2*) &y[slot_map[j]*stride_channel_y + 2*(j*stride_col_y + col)] : make_float2(0.0f, 0.0f);
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
}
const float2 tmp = j < cols_per_block ? y2[j*stride_col_y + col] : make_float2(0.0f, 0.0f);
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
}
} else {
static_assert(std::is_same_v<T, void>, "unsupported type");
@@ -231,7 +160,7 @@ static __global__ void mul_mat_f(
}
}
float * buf_iw = (float *) compute_base;
float * buf_iw = (float *) data_mmv;
constexpr int kiw = nwarps*rows_per_block + mmf_get_padding();
if (nwarps > 1) {
@@ -278,26 +207,15 @@ static __global__ void mul_mat_f(
}
}
if constexpr (!has_ids) {
#pragma unroll
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
dst[j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
}
} else {
const int slot = (j < cols_per_block) ? slot_map[j] : -1;
if (slot >= 0 && (col_base + j) < ncols_dst_total) {
#pragma unroll
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
dst[slot*stride_channel_dst + j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
}
}
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
dst[j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
}
}
}
#else
GGML_UNUSED_VARS(x, y, ids, dst,
ncols, ncols_dst_total, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id,
GGML_UNUSED_VARS(x, y, dst,
ncols, stride_row, stride_col_y, stride_col_dst,
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
NO_DEVICE_CODE;
@@ -588,22 +506,21 @@ static __global__ void mul_mat_f_ids(
template<typename T, int rows_per_block, int cols_per_block, int nwarps>
static inline void mul_mat_f_switch_ids(
const T * x, const float * y, const int32_t * ids, float * dst,
const T * x, const float * y, float * dst,
const int64_t ncols_x, const int64_t ncols_dst, const int64_t nchannels_dst,
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
const int64_t stride_col_id, const int64_t stride_row_id,
const int64_t channel_ratio, const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst,
const int64_t sample_ratio, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
const dim3 & block_nums, const dim3 & block_dims, const int nbytes_shared_total, cudaStream_t stream,
const mmf_ids_data * ids_data) {
const bool has_ids_data = ids_data && ids_data->ids_src_compact;
// Use the compact-ids kernel only for larger tiles; for small ncols_dst (< 16)
// we prefer the normal mul_mat_f path with has_ids=true.
if (has_ids_data && ncols_dst > 16) {
if (has_ids_data) {
if (ids_data->max_tiles == 0) {
return;
}
GGML_ASSERT(block_nums.z == 1);
dim3 block_nums_ids(block_nums.x, ids_data->max_tiles, 1);
const uint3 sis1_fd = ids_data->sis1 > 0 ? init_fastdiv_values((uint32_t) ids_data->sis1) : make_uint3(0, 0, 1);
@@ -615,19 +532,10 @@ static inline void mul_mat_f_switch_ids(
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
sis1_fd, nch_fd);
} else if (ids) {
const int64_t col_tiles = (ncols_dst + cols_per_block - 1) / cols_per_block;
dim3 block_nums_ids = block_nums;
block_nums_ids.y *= col_tiles;
mul_mat_f<T, rows_per_block, cols_per_block, nwarps, true><<<block_nums_ids, block_dims, nbytes_shared_total, stream>>>
(x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
} else {
mul_mat_f<T, rows_per_block, cols_per_block, nwarps, false><<<block_nums, block_dims, nbytes_shared_total, stream>>>
(x, y, ids, dst, ncols_x, cols_per_block, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
mul_mat_f<T, rows_per_block, cols_per_block, nwarps><<<block_nums, block_dims, nbytes_shared_total, stream>>>
(x, y, dst, ncols_x, stride_row, stride_col_y, stride_col_dst,
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
}
}
@@ -637,7 +545,6 @@ void mul_mat_f_cuda(
const T * x, const float * y, const int32_t * ids, float * dst,
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
const int64_t stride_col_id, const int64_t stride_row_id,
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
@@ -673,70 +580,59 @@ void mul_mat_f_cuda(
const int nbytes_shared_iter = nwarps_best * (volta_mma_available(cc) ? tile_A_32::I : tile_A_16::I) * (warp_size + mmf_get_padding(cc)) * 4;
const int nbytes_cols_per_block_pad = (amd_wmma_available(cc) || amd_mfma_available(cc)) ? tile_B_16::I : tile_B_8::I;
const int nbytes_shared_combine = GGML_PAD(cols_per_block, nbytes_cols_per_block_pad) * (nwarps_best*rows_per_block + mmf_get_padding(cc)) * 4;
const int nbytes_shared = std::max(nbytes_shared_iter, nbytes_shared_combine);
const int nbytes_slotmap = ids ? GGML_PAD(cols_per_block, 16) * sizeof(int) : 0;
const int nbytes_shared_total = nbytes_shared + nbytes_slotmap;
const int64_t grid_y = ids ? nchannels_x : nchannels_dst;
const int nbytes_shared_total = std::max(nbytes_shared_iter, nbytes_shared_combine);
const dim3 block_nums(nrows_x/rows_per_block, grid_y, nsamples_dst);
const dim3 block_nums(nrows_x/rows_per_block, nchannels_dst, nsamples_dst);
const dim3 block_dims(warp_size, nwarps_best, 1);
switch (nwarps_best) {
case 1: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 1>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 2: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 2>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 3: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 3>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 4: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 4>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 5: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 5>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 6: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 6>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 7: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 7>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
case 8: {
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 8>(
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
ids_data);
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
} break;
default:
GGML_ABORT("fatal error");
@@ -750,95 +646,94 @@ static void mul_mat_f_switch_cols_per_block(
const T * x, const float * y, const int32_t * ids, float * dst,
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
const int64_t stride_col_id, const int stride_row_id,
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
cudaStream_t stream, const mmf_ids_data * ids_data) {
const int ncols_case = (ids && ncols_dst > 16) ? 16 : ncols_dst;
const int ncols_case = ids ? 16 : ncols_dst;
GGML_ASSERT(ids || ncols_dst <= 16);
switch (ncols_case) {
case 1: {
mul_mat_f_cuda<T, rows_per_block, 1>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 2: {
mul_mat_f_cuda<T, rows_per_block, 2>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 3: {
mul_mat_f_cuda<T, rows_per_block, 3>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 4: {
mul_mat_f_cuda<T, rows_per_block, 4>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 5: {
mul_mat_f_cuda<T, rows_per_block, 5>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 6: {
mul_mat_f_cuda<T, rows_per_block, 6>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 7: {
mul_mat_f_cuda<T, rows_per_block, 7>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 8: {
mul_mat_f_cuda<T, rows_per_block, 8>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 9: {
mul_mat_f_cuda<T, rows_per_block, 9>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 10: {
mul_mat_f_cuda<T, rows_per_block, 10>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 11: {
mul_mat_f_cuda<T, rows_per_block, 11>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 12: {
mul_mat_f_cuda<T, rows_per_block, 12>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 13: {
mul_mat_f_cuda<T, rows_per_block, 13>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 14: {
mul_mat_f_cuda<T, rows_per_block, 14>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 15: {
mul_mat_f_cuda<T, rows_per_block, 15>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case 16: {
mul_mat_f_cuda<T, rows_per_block, 16>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
default:
@@ -851,7 +746,6 @@ static void mul_mat_f_switch_rows_per_block(
const int rows_per_block, const T * x, const float * y, const int32_t * ids, float * dst,
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
const int64_t stride_col_id, const int stride_row_id,
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
@@ -860,13 +754,13 @@ static void mul_mat_f_switch_rows_per_block(
case MMF_ROWS_PER_BLOCK: {
mul_mat_f_switch_cols_per_block<T, MMF_ROWS_PER_BLOCK>(
x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
case MMF_ROWS_PER_BLOCK_CDNA: {
mul_mat_f_switch_cols_per_block<T, MMF_ROWS_PER_BLOCK_CDNA>(
x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
} break;
default:
@@ -878,7 +772,6 @@ static void mul_mat_f_switch_rows_per_block(
template void mul_mat_f_cuda<T, nrows_dst, ncols_dst>( \
const T * x, const float * y, const int32_t * ids, float * dst, \
const int64_t ncols_x, const int64_t nrows_x, int64_t ncols_dst_total, const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst, \
const int64_t stride_col_id, const int64_t stride_row_id, \
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst, \
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,\
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst, \
+159 -60
View File
@@ -24,6 +24,50 @@ template <int n> struct mm_ids_pow2 { static constexpr int value = 2*mm_ids_pow2
template <> struct mm_ids_pow2<1> { static constexpr int value = 1; };
template <> struct mm_ids_pow2<0> { static constexpr int value = 1; };
template <bool increment_nex_prev, bool write_to_store>
static __device__ __forceinline__ void check_experts(
mm_ids_helper_store * store, int & nex_prev, int & it_compact,
const int expert, const int it, const int iex, const int expert_found) {
if (increment_nex_prev) {
nex_prev += expert_found < expert;
}
if (write_to_store) {
const ggml_cuda_thread_mask_t iex_used = ggml_cuda_ballot(expert_found == expert);
const ggml_cuda_thread_mask_t mask = (ggml_cuda_thread_mask_t(1) << threadIdx.x) - 1;
const int it_compact_offset = ggml_cuda_popc(iex_used & mask);
if (expert_found == expert) {
store[it_compact + it_compact_offset] = mm_ids_helper_store(it, iex);
}
it_compact += ggml_cuda_popc(iex_used);
}
}
static __device__ __forceinline__ void dump_store(
int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst,
const int n_expert_used, const int nchannels_y, const int sis1, const bool write_inverse,
const mm_ids_helper_store * __restrict__ store, const int nex_prev, int & n_dumped, const int it_compact) {
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {
const mm_ids_helper_store store_it = store[itc];
const int it = store_it.it();
const int iex_used = store_it.iex_used();
ids_dst[nex_prev + n_dumped + itc] = it*n_expert_used + iex_used;
// ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup
if (write_inverse) {
ids_src1[it*n_expert_used + iex_used] = nex_prev + n_dumped + itc;
} else {
ids_src1[nex_prev + n_dumped + itc] = it*sis1 + iex_used % nchannels_y;
}
}
n_dumped += it_compact;
}
constexpr int n_unroll = 4; // Manually unroll this many loop iterations for better performance.
// Helper function for mul_mat_id, converts ids to a more convenient format.
// ids_src1 describes how to permute the flattened column indices of src1 in order to get a compact src1 tensor sorted by expert.
// ids_dst describes the same mapping but for the dst tensor.
@@ -36,85 +80,138 @@ static __global__ void mm_ids_helper(
const int32_t * __restrict__ ids,
int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec,
const int n_tokens, const int n_expert_used_var, const int nchannels_y, const uint3 tile_width,
const int si1, const int sis1, const bool write_inverse) {
const int si1, const int sis1, const bool write_inverse, const int ne_store) {
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
const int n_expert_used = n_expert_used_template == 0 ? n_expert_used_var : n_expert_used_template;
const int expert = blockIdx.x;
const int expert = blockIdx.x; // The expert this CUDA block handles.
extern __shared__ char data_mm_ids_helper[];
mm_ids_helper_store * store = (mm_ids_helper_store *) data_mm_ids_helper;
int nex_prev = 0; // Number of columns for experts with a lower index.
int n_dumped = 0; // Number of data values that have been dumped to output buffers.
int it_compact = 0; // Running index for the compact slice of this expert.
// Step 1: Iterate over n_tokens, increment nex_prev to determine the output indices, fill up store.
// Step 1.5: If the store does not have enough space to hold all values, iterate over the rest of the tokens just to increment nex_prev.
// Step 2: Dump the stored values using nex_prev to get the correct output indices.
// Step 3: If we were not able to process all values, resume at the index up to which we were able to dump the output and re-do.
if constexpr (n_expert_used_template == 0) {
// Generic implementation:
for (int it = 0; it < n_tokens; ++it) {
int iex_used = -1; // The index at which the expert is used, if any.
for (int iex = threadIdx.x; iex < n_expert_used; iex += warp_size) {
const int expert_used = ids[it*si1 + iex];
nex_prev += expert_used < expert;
if (expert_used == expert) {
iex_used = iex;
int it = 0;
for (; it < n_tokens && it_compact + max(warp_size, n_expert_used) <= ne_store; ++it) {
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
const int iex = iex0 + threadIdx.x;
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
constexpr bool increment_nex_prev = true;
constexpr bool write_to_store = true;
check_experts<increment_nex_prev, write_to_store>
(store, nex_prev, it_compact, expert, it, iex, expert_found);
}
}
int it_dumped = it;
for (; it < n_tokens; ++it) {
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
const int iex = iex0 + threadIdx.x;
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
constexpr bool increment_nex_prev = true;
constexpr bool write_to_store = false;
check_experts<increment_nex_prev, write_to_store>
(store, nex_prev, it_compact, expert, it, iex, expert_found);
}
}
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
ggml_cuda_syncwarp();
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
it_compact = 0;
for (int it = it_dumped; it < n_tokens; ++it) {
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
const int iex = iex0 + threadIdx.x;
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
constexpr bool increment_nex_prev = false;
constexpr bool write_to_store = true;
check_experts<increment_nex_prev, write_to_store>
(store, nex_prev, it_compact, expert, it, iex, expert_found);
if (it_compact + max(warp_size, n_expert_used) >= ne_store) {
ggml_cuda_syncwarp();
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
it_compact = 0;
}
}
if (iex_used != -1) {
store[it_compact] = mm_ids_helper_store(it, iex_used);
}
if (warp_reduce_any<warp_size>(iex_used != -1)) {
it_compact++;
}
}
if (it_compact > 0) {
ggml_cuda_syncwarp();
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
}
} else {
// Implementation optimized for specific numbers of experts used:
// a warp holds a whole number of token slots, so the slot count is padded to a power of 2
constexpr int neu_padded = mm_ids_pow2<n_expert_used_template>::value;
static_assert(neu_padded <= warp_size && warp_size % neu_padded == 0, "bad n_expert_used");
for (int it0 = 0; it0 < n_tokens; it0 += warp_size/neu_padded) {
const int it = it0 + threadIdx.x / neu_padded;
const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any.
const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any.
const int expert_used = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
ids[it*si1 + iex] : INT_MAX;
const int iex_used = expert_used == expert ? iex : -1;
nex_prev += expert_used < expert;
// Whether the threads at this token position have used the expert:
const int it_compact_add_self = warp_reduce_any<neu_padded>(iex_used != -1);
// Do a scan over threads at lower token positions in warp to get the correct index for writing data:
int it_compact_add_lower = 0;
int it0 = 0;
for (; it0 < n_tokens && it_compact + n_unroll*warp_size <= ne_store; it0 += n_unroll*warp_size/neu_padded) {
#pragma unroll
for (int offset = neu_padded; offset < warp_size; offset += neu_padded) {
const int tmp = ggml_cuda_shfl_up(it_compact_add_self, offset, warp_size);
if (threadIdx.x >= static_cast<unsigned int>(offset)) {
it_compact_add_lower += tmp;
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
const int it = it0 + it1 + threadIdx.x / neu_padded;
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
ids[it*si1 + iex] : INT_MAX;
constexpr bool increment_nex_prev = true;
constexpr bool write_to_store = true;
check_experts<increment_nex_prev, write_to_store>
(store, nex_prev, it_compact, expert, it, iex, expert_found);
}
}
int it0_dumped = it0;
for (; it0 < n_tokens; it0 += n_unroll*warp_size/neu_padded) {
#pragma unroll
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
const int it = it0 + it1 + threadIdx.x / neu_padded;
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
ids[it*si1 + iex] : INT_MAX;
constexpr bool increment_nex_prev = true;
constexpr bool write_to_store = false;
check_experts<increment_nex_prev, write_to_store>
(store, nex_prev, it_compact, expert, it, iex, expert_found);
}
}
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
ggml_cuda_syncwarp();
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
it_compact = 0;
for (it0 = it0_dumped; it0 < n_tokens; it0 += n_unroll*warp_size/neu_padded) {
#pragma unroll
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
const int it = it0 + it1 + threadIdx.x / neu_padded;
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
ids[it*si1 + iex] : INT_MAX;
constexpr bool increment_nex_prev = false;
constexpr bool write_to_store = true;
check_experts<increment_nex_prev, write_to_store>
(store, nex_prev, it_compact, expert, it, iex, expert_found);
if (it_compact + warp_size >= ne_store) {
ggml_cuda_syncwarp();
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
it_compact = 0;
}
}
if (iex_used != -1) {
store[it_compact + it_compact_add_lower] = mm_ids_helper_store(it, iex_used);
}
// The thread with the highest index in the warp always has the sum over the whole warp, use it to increment all threads:
it_compact += ggml_cuda_shfl(it_compact_add_lower + it_compact_add_self, warp_size - 1, warp_size);
}
}
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
ggml_cuda_syncwarp();
for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {
const mm_ids_helper_store store_it = store[itc];
const int it = store_it.it();
const int iex_used = store_it.iex_used();
ids_dst[nex_prev + itc] = it*n_expert_used + iex_used;
// ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup
if (write_inverse) {
ids_src1[it*n_expert_used + iex_used] = nex_prev + itc;
} else {
ids_src1[nex_prev + itc] = it*sis1 + iex_used % nchannels_y;
if (it_compact > 0) {
ggml_cuda_syncwarp();
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
}
}
@@ -123,7 +220,7 @@ static __global__ void mm_ids_helper(
}
int col_low = nex_prev;
int col_high = nex_prev + min(int(tile_width.z), it_compact);
int col_high = nex_prev + min(int(tile_width.z), n_dumped);
// The while loop covers all tiles at the seams containing the column with % tile_width == 0.
while (col_high > fastdiv(col_low + tile_width.z - 1, tile_width) * tile_width.z) {
@@ -131,7 +228,7 @@ static __global__ void mm_ids_helper(
= {expert, col_low, col_high, /*padding =*/ 0};
col_low = col_high;
col_high = min(col_high + int(tile_width.z), nex_prev + it_compact);
col_high = min(col_high + int(tile_width.z), nex_prev + n_dumped);
}
// Finally, at most one extra tile_spec between the seams per expert.
@@ -157,10 +254,12 @@ static void launch_mm_ids_helper(
const dim3 num_blocks(n_experts, 1, 1);
const dim3 block_size(warp_size, 1, 1);
const size_t nbytes_shared = n_tokens*sizeof(mm_ids_helper_store);
GGML_ASSERT(nbytes_shared <= smpbo);
const size_t ne_shared_pad = n_expert_used_template == 0 ? std::max(n_expert_used_var, warp_size) : n_unroll*warp_size;
const size_t nbytes_store_core = std::min(smpbo - ne_shared_pad * sizeof(mm_ids_helper_store), n_tokens * sizeof(mm_ids_helper_store));
const size_t nbytes_shared = nbytes_store_core + ne_shared_pad * sizeof(mm_ids_helper_store);
const size_t ne_store = nbytes_shared / sizeof(mm_ids_helper_store);
mm_ids_helper<n_expert_used_template><<<num_blocks, block_size, nbytes_shared, stream>>>
(ids, ids_src1, ids_dst, tile_spec, n_tokens, n_expert_used_var, nchannels_y, tile_width_fd, si1, sis1, write_inverse);
(ids, ids_src1, ids_dst, tile_spec, n_tokens, n_expert_used_var, nchannels_y, tile_width_fd, si1, sis1, write_inverse, ne_store);
}
void ggml_cuda_launch_mm_ids_helper(
+2 -9
View File
@@ -184,13 +184,6 @@ static __global__ void top_k_radix_gather(
constexpr int NBINS = 1 << RADIX_BITS;
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
constexpr int nwarps = BLOCK_SIZE / warp_size;
const auto popc = [](const ggml_cuda_thread_mask_t mask) {
if constexpr (warp_size == 64) {
return __popcll(mask);
} else {
return __popc(mask);
}
};
__shared__ int warp_ties[nwarps];
const int row = blockIdx.x / blocks_per_row;
@@ -247,11 +240,11 @@ static __global__ void top_k_radix_gather(
const bool is_tie = col < warp_end && top_k_float_to_ordered(row_src[col]) == threshold;
const ggml_cuda_thread_mask_t ties = ggml_cuda_ballot(is_tie);
const int tie_pos = pos + popc(ties & (((ggml_cuda_thread_mask_t) 1 << lane) - 1));
const int tie_pos = pos + ggml_cuda_popc(ties & (((ggml_cuda_thread_mask_t) 1 << lane) - 1));
if (is_tie && tie_pos < pos_end) {
row_dst[k - rank + tie_pos] = col;
}
pos += popc(ties);
pos += ggml_cuda_popc(ties);
}
}
+3
View File
@@ -323,6 +323,9 @@ set(GGML_OPENCL_KERNELS
mul_mm_f16_f32_kq_kqv
mul_mm_f16_f32_kq_p8
mul_mm_q8_f32_kqv
mul_mm_q8_kq_p8
mul_mm_q8_kq_p8_dk512
flash_attn_q8_rows
conv2d
conv2d_f16_f32
conv2_dw
+234 -54
View File
@@ -781,6 +781,10 @@ struct ggml_backend_opencl_context {
ggml_cl_buffer prealloc_fa_pd;
ggml_cl_buffer prealloc_fa_bmax;
ggml_cl_buffer prealloc_fa_bsum;
ggml_cl_buffer prealloc_fa_k8;
ggml_cl_buffer prealloc_fa_k8_scale;
ggml_cl_buffer prealloc_fa_q8;
ggml_cl_buffer prealloc_fa_q8_scale;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
ggml_cl_buffer prealloc_adreno_xmem_const;
@@ -899,6 +903,10 @@ struct ggml_backend_opencl_context {
cl_kernel kernel_fa_kqv_direct_f16 = nullptr;
cl_kernel kernel_fa_kqv_direct_reduce = nullptr;
cl_kernel kernel_fa_v_transpose_q8 = nullptr;
cl_kernel kernel_fa_q8_rows_f16 = nullptr;
cl_kernel kernel_fa_q8_rows_f32 = nullptr;
cl_kernel kernel_mul_mm_q8_kq_p8 = nullptr;
cl_kernel kernel_mul_mm_q8_kq_p8_dk512 = nullptr;
ggml_opencl_fa_kernels fa;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
ggml_cl_adreno_xmem_attn_state adreno_xmem_attn;
@@ -1506,7 +1514,18 @@ struct ggml_backend_opencl_context {
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
~ggml_backend_opencl_context() {
for (cl_kernel kernel : {kernel_mul_mm_f16_f32_kq_p8, kernel_mul_mm_q8_f32_kqv, kernel_fa_p8_fixup, kernel_fa_v_transpose_q8, kernel_fa_kqv_direct_f16, kernel_fa_kqv_direct_reduce}) {
for (cl_kernel kernel : {
kernel_mul_mm_f16_f32_kq_p8,
kernel_mul_mm_q8_f32_kqv,
kernel_fa_p8_fixup,
kernel_fa_v_transpose_q8,
kernel_fa_kqv_direct_f16,
kernel_fa_kqv_direct_reduce,
kernel_fa_q8_rows_f16,
kernel_fa_q8_rows_f32,
kernel_mul_mm_q8_kq_p8,
kernel_mul_mm_q8_kq_p8_dk512
}) {
if (kernel != nullptr) {
CL_CHECK(clReleaseKernel(kernel));
}
@@ -1648,7 +1667,7 @@ static cl_program build_program_from_source_ex(cl_context ctx, cl_device_id dev,
return NULL;
}
static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts) {
static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts, bool fatal = true) {
cl_context ctx = backend_ctx->context;
cl_device_id dev = backend_ctx->device;
@@ -1660,7 +1679,7 @@ static cl_program build_program_from_source(ggml_backend_opencl_context * backen
return p_cached;
}
cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, /*fatal=*/true);
cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, fatal);
// Best-effort save of the freshly-built binary (no-op if cache disabled).
if (p != nullptr) {
@@ -3995,6 +4014,52 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
}
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string rows_src {
#include "flash_attn_q8_rows.cl.h"
};
const std::string kq_src {
#include "mul_mm_q8_kq_p8.cl.h"
};
#else
const std::string rows_src = read_file("flash_attn_q8_rows.cl");
const std::string kq_src = read_file("mul_mm_q8_kq_p8.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, rows_src.c_str(), compile_opts, /*fatal=*/false);
if (prog != nullptr) {
CL_CHECK((backend_ctx->kernel_fa_q8_rows_f16 = clCreateKernel(prog, "kernel_fa_q8_rows_f16", &err), err));
CL_CHECK((backend_ctx->kernel_fa_q8_rows_f32 = clCreateKernel(prog, "kernel_fa_q8_rows_f32", &err), err));
CL_CHECK(clReleaseProgram(prog));
}
prog = build_program_from_source(backend_ctx, kq_src.c_str(), compile_opts, /*fatal=*/false);
if (prog != nullptr) {
CL_CHECK((backend_ctx->kernel_mul_mm_q8_kq_p8 = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8", &err), err));
CL_CHECK(clReleaseProgram(prog));
}
GGML_LOG_CONT(".");
}
// mul_mm_q8_kq_p8_dk512
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
#ifdef GGML_OPENCL_EMBED_KERNELS
const std::string kernel_src {
#include "mul_mm_q8_kq_p8_dk512.cl.h"
};
#else
const std::string kernel_src = read_file("mul_mm_q8_kq_p8_dk512.cl");
#endif
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts, /*fatal=*/false);
if (prog != nullptr) {
cl_kernel kernel = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8_dk512", &err);
CL_CHECK(clReleaseProgram(prog));
if (err == CL_SUCCESS) {
backend_ctx->kernel_mul_mm_q8_kq_p8_dk512 = kernel;
}
}
GGML_LOG_CONT(".");
}
// mul_mm_f16_f32_kq_p8
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
#ifdef GGML_OPENCL_EMBED_KERNELS
@@ -22922,12 +22987,58 @@ static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_te
}
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
static void ggml_cl_fa_q8_rows(
ggml_backend_t backend, const ggml_tensor * src, const ggml_tensor * dst,
cl_mem values, cl_mem scales, int64_t row0, int nrow
) {
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
auto * extra = (ggml_tensor_extra_cl *) (src->view_src ? src->view_src->extra : src->extra);
const cl_ulong offset = extra->offset + src->view_offs + row0 * src->nb[1];
const cl_ulong nb1 = src->nb[1];
const cl_ulong nb2 = src->nb[2];
const int dk = src->ne[0];
const int nhead = src->ne[2];
cl_kernel kernel;
switch (src->type) {
case GGML_TYPE_F16: kernel = backend_ctx->kernel_fa_q8_rows_f16; break;
case GGML_TYPE_F32: kernel = backend_ctx->kernel_fa_q8_rows_f32; break;
default: GGML_ABORT("unsupported int8 KQ input type");
}
cl_uint a = 0;
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra->data_device));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &offset));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb2));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &values));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &scales));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dk));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nrow));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nhead));
const size_t blocks = (size_t) (dk / 32) * nrow * nhead;
size_t gws[3] = {(blocks + 63) / 64 * 64, 1, 1};
size_t lws[3] = {64, 1, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
}
static void ggml_cl_fa_kq_int8_prepare(ggml_backend_t backend, const ggml_tensor * dst, int64_t n_q_chunk) {
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
const ggml_tensor * q = dst->src[0];
const ggml_tensor * k = dst->src[1];
const size_t k_bytes = (size_t) k->ne[0] * k->ne[1] * k->ne[2];
const size_t q_bytes = (size_t) q->ne[0] * n_q_chunk * q->ne[2];
backend_ctx->prealloc_fa_k8.allocate(backend_ctx->context, k_bytes);
backend_ctx->prealloc_fa_k8_scale.allocate(backend_ctx->context, k_bytes / 32 * sizeof(cl_half));
backend_ctx->prealloc_fa_q8.allocate(backend_ctx->context, q_bytes);
backend_ctx->prealloc_fa_q8_scale.allocate(backend_ctx->context, q_bytes / 32 * sizeof(cl_half));
ggml_cl_fa_q8_rows(backend, k, dst, backend_ctx->prealloc_fa_k8.buffer, backend_ctx->prealloc_fa_k8_scale.buffer, 0, k->ne[1]);
}
static void ggml_cl_fa_kq_p8_dispatch(
ggml_backend_t backend,
const ggml_tensor * q, const ggml_tensor * k,
const ggml_tensor * mask, const ggml_tensor * sinks,
const ggml_tensor * dst,
int64_t q0, int nqc, float scale, float max_bias, int p_pitch
int64_t q0, int nqc, float scale, float max_bias, int p_pitch, bool kq_int8
) {
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
auto * extra_k = (ggml_tensor_extra_cl *) (k->view_src ? k->view_src->extra : k->extra);
@@ -22940,64 +23051,94 @@ static void ggml_cl_fa_kq_p8_dispatch(
const int D_A = k->ne[2];
const int D_B = q->ne[2];
cl_int err;
cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
cl_image_format fmt = {CL_RGBA, CL_FLOAT};
cl_image_desc desc = {};
desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
desc.image_width = region.size / 16;
desc.buffer = k_sub;
cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
CL_CHECK(err);
region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
region.size = (size_t) K * N * D_B * sizeof(float);
cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
const cl_ulong off_mask = extra_m->offset + mask->view_offs + q0 * mask->nb[1];
const cl_ulong mask_nb1 = mask->nb[1];
const int n_head_log2 = 1u << (unsigned) floorf(log2f((float) D_B));
const float m0 = powf(2.0f, -max_bias / n_head_log2);
const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
cl_kernel kernel;
cl_uint a = 0;
cl_kernel kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
if (kq_int8) {
ggml_cl_fa_q8_rows(backend, q, dst, backend_ctx->prealloc_fa_q8.buffer, backend_ctx->prealloc_fa_q8_scale.buffer, q0, nqc);
kernel = K <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8_scale.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8_scale.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
size_t gws[3] = {64 * (((size_t) N + 31) / 32) * (D_B / D_A), ((size_t) M + 511) / 512, (size_t) D_A};
size_t lws[3] = {64, 1, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
} else {
cl_int err;
const size_t n_tiles = (N + 31) / 32;
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
cl_image_format fmt = {CL_RGBA, CL_FLOAT};
cl_image_desc desc = {};
desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
desc.image_width = region.size / 16;
desc.buffer = k_sub;
cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
CL_CHECK(err);
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
CL_CHECK(clReleaseMemObject(k_img));
CL_CHECK(clReleaseMemObject(k_sub));
CL_CHECK(clReleaseMemObject(q_sub));
region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
region.size = (size_t) K * N * D_B * sizeof(float);
cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, &region, &err);
CL_CHECK(err);
kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
const size_t n_tiles = (N + 31) / 32;
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
CL_CHECK(clReleaseMemObject(k_img));
CL_CHECK(clReleaseMemObject(k_sub));
CL_CHECK(clReleaseMemObject(q_sub));
}
cl_mem sinks_buf = extra_q->data_device;
cl_ulong sinks_off = 0;
@@ -23261,7 +23402,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
ggml_backend_t backend,
const ggml_tensor * q, const ggml_tensor * k,
ggml_tensor * dst,
int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct
int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct, bool kq_int8
) {
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
const ggml_tensor * v = dst->src[2];
@@ -23345,13 +23486,17 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
}
if (kq_int8) {
ggml_cl_fa_kq_int8_prepare(backend, dst, n_q_chunk);
}
ggml_tensor_extra_cl extra_kqv;
ggml_tensor kqv;
for (int64_t q0 = 0; q0 < n_q; q0 += n_q_chunk) {
const int64_t nqc = MIN(n_q_chunk, n_q - q0);
ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch);
ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch, kq_int8);
ggml_cl_fa_decompose_scratch_tensor(kqv, extra_kqv, backend_ctx->prealloc_fa_kqv.buffer, GGML_TYPE_F32, dv, nqc, n_head);
@@ -23408,6 +23553,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
// head size. GGML_OPENCL_FA_DECOMPOSE_MB sets the KQ scratch budget (default 256).
// GGML_OPENCL_FA_KQ_P8=0 disables U8 probabilities and int8 KQV.
// GGML_OPENCL_FA_KQV_DIRECT=0 disables direct PV for small query batches.
// GGML_OPENCL_FA_KQ_INT8=0 keeps F16 KQ; =1 enables int8 KQ for supported shapes.
static bool ggml_cl_flash_attn_decompose_dispatch(
ggml_backend_t backend,
const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst
@@ -23537,12 +23683,46 @@ static bool ggml_cl_flash_attn_decompose_dispatch(
return false;
}
static const int kq_int8_env = [] {
const char * e = getenv("GGML_OPENCL_FA_KQ_INT8");
return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
}();
static const int dk512_env = [] {
const char * e = getenv("GGML_OPENCL_FA_KQ_DK512_F16");
return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
}();
const bool dk512 = dk > 256 && dk <= 512 && n_q >= 32 &&
(dk512_env == 1 || (dk512_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E));
const bool kq_int8_possible = kq_p8 && (dk <= 256 || dk512) && dk % 32 == 0 &&
backend_ctx->kernel_fa_q8_rows_f16 != nullptr && backend_ctx->kernel_fa_q8_rows_f32 != nullptr &&
n_head <= INT_MAX && n_head_kv <= INT_MAX &&
n_q <= INT_MAX / n_head / (dk / 32) && n_kv <= INT_MAX / n_head_kv / (dk / 32) &&
(dst->src[4] == nullptr || dk > 64) &&
(kq_int8_env == 1 || (kq_int8_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && n_kv >= 2048)) &&
backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f16) >= 64 &&
backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f32) >= 64;
cl_kernel kq_int8_kernel = nullptr;
if (kq_int8_possible) {
kq_int8_kernel = dk <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
}
const bool kq_int8 = kq_int8_kernel != nullptr && backend_ctx->get_kernel_workgroup_size(kq_int8_kernel) >= 64;
// Fewer query tiles per KV block reduce cache pressure in the int8 KQ kernel.
static const int64_t kq_int8_max_chunk = [] {
const char * e = getenv("GGML_OPENCL_FA_KQ_INT8_MAX_CHUNK");
const int v = (e && e[0]) ? atoi(e) : 96;
return (int64_t) (v >= 32 ? (v / 32) * 32 : 96);
}();
if (kq_int8) {
n_q_chunk = MIN(n_q_chunk, kq_int8_max_chunk);
}
// even out the chunks, then align to the 32-wide n-tile
const int64_t n_chunks = (n_q + n_q_chunk - 1) / n_q_chunk;
n_q_chunk = MIN(n_q_chunk, ((n_q + n_chunks - 1) / n_chunks + 31) & ~(int64_t) 31);
if (kq_p8) {
ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct);
ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct, kq_int8);
} else {
ggml_cl_flash_attn_decompose(backend, q, k, dst, n_q_chunk, scale, max_bias);
}
@@ -0,0 +1,77 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
// Store 32-element blocks as separate int8 values and half scales.
kernel void kernel_fa_q8_rows_f16(
global const char * src, ulong off_src, ulong nb1, ulong nb2,
global char * dq, global half * dd,
int ne0, int nrow, int nhead
) {
const int nblk = ne0 / 32;
const int gid = get_global_id(0);
if (gid >= nblk*nrow*nhead) {
return;
}
const int b = gid % nblk;
const int r = (gid / nblk) % nrow;
const int h = gid / (nblk*nrow);
global const half * p = (global const half *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
float v[32];
float amax = 0.0f;
#pragma unroll
for (int i = 0; i < 8; ++i) {
const float4 x = convert_float4(vload4(i, p));
v[i*4+0] = x.s0;
v[i*4+1] = x.s1;
v[i*4+2] = x.s2;
v[i*4+3] = x.s3;
amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
}
const float d = amax / 127.0f;
const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
const size_t row = (size_t)h*nrow + r;
dd[row*nblk + b] = (half)d;
global char * q = dq + row*ne0 + b*32;
#pragma unroll
for (int i = 0; i < 32; ++i) {
q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
}
}
kernel void kernel_fa_q8_rows_f32(
global const char * src, ulong off_src, ulong nb1, ulong nb2,
global char * dq, global half * dd,
int ne0, int nrow, int nhead
) {
const int nblk = ne0 / 32;
const int gid = get_global_id(0);
if (gid >= nblk*nrow*nhead) {
return;
}
const int b = gid % nblk;
const int r = (gid / nblk) % nrow;
const int h = gid / (nblk*nrow);
global const float * p = (global const float *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
float v[32];
float amax = 0.0f;
#pragma unroll
for (int i = 0; i < 8; ++i) {
const float4 x = vload4(i, p);
v[i*4+0] = x.s0;
v[i*4+1] = x.s1;
v[i*4+2] = x.s2;
v[i*4+3] = x.s3;
amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
}
const float d = amax / 127.0f;
const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
const size_t row = (size_t)h*nrow + r;
dd[row*nblk + b] = (half)d;
global char * q = dq + row*ne0 + b*32;
#pragma unroll
for (int i = 0; i < 32; ++i) {
q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
}
}
@@ -0,0 +1,174 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
#ifdef cl_khr_integer_dot_product
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
#endif
#define KQ_TN 32
#define KQ_MB 8
#define KQ_TM 64
#define KQ_WG 64
#define KQ_DK_MAX 256
__attribute__((qcom_wave_pair_mode(1)))
kernel void kernel_mul_mm_q8_kq_p8(
global const uint * kq, // K int8, [head_kv][n_kv][dk] as packed uints
global const half * kd, // K scales, [head_kv][n_kv][dk/32]
global const uint * qq, // Q int8, [head][n_q][dk] as packed uints
global const half * qd, // Q scales, [head][n_q][dk/32]
global uchar * qp, // u8 P, [head][n_q][p_pitch]
global float * bmax, // per 32-row block max, [head][n_q][p_pitch/32]
global float * bsum, // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
global const char * mask, // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
ulong offset_mask,
ulong mask_nb1,
float scale,
float max_bias,
float m0,
float m1,
int n_head_log2,
int dk,
int n_kv,
int n_q,
int n_head,
int n_head_kv,
int p_pitch // bytes per P row, >= n_kv, multiple of 32
) {
const int lid = get_local_id(0);
const int gsz = n_head / n_head_kv;
const int g0 = get_group_id(0);
const int head_kv = get_group_id(2);
const int head = head_kv*gsz + (g0 % gsz);
const int qn0 = (g0 / gsz) * KQ_TN;
const int kv0 = get_group_id(1)*KQ_TM*KQ_MB + lid;
const int nblk = dk / 32;
const int nu = dk / 4;
__local uint sh_q [KQ_TN][KQ_DK_MAX/4];
__local half sh_qd[KQ_TN][KQ_DK_MAX/32];
__local float p8_lds[1024];
const size_t qbase = (size_t)head*n_q;
for (int i = lid; i < KQ_TN*(KQ_DK_MAX/16); i += KQ_WG) {
const int t = i / (KQ_DK_MAX/16);
const int u = (i % (KQ_DK_MAX/16)) * 4;
const int qi = qn0 + t;
uint4 v = (uint4)(0u);
if (qi < n_q && u < nu) {
v = vload4(0, &qq[(qbase + qi)*nu + u]);
}
vstore4(v, 0, &sh_q[t][u]);
}
for (int i = lid; i < KQ_TN*(KQ_DK_MAX/32); i += KQ_WG) {
const int t = i / (KQ_DK_MAX/32);
const int b = i % (KQ_DK_MAX/32);
const int qi = qn0 + t;
sh_qd[t][b] = (qi < n_q && b < nblk) ? qd[(qbase + qi)*nblk + b] : (half)0.0f;
}
barrier(CLK_LOCAL_MEM_FENCE);
float slope = 1.0f;
if (max_bias > 0.0f) {
const float base = head < n_head_log2 ? m0 : m1;
const int ex = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
slope = pow(base, ex);
}
global const half * mrow = (global const half *)(mask + offset_mask);
const uint mstride = (uint)(mask_nb1 / 2);
const int pblk = p_pitch / 32;
for (int m = 0; m < KQ_MB; ++m) {
const int kv = kv0 + m*KQ_TM;
if (kv - lid >= n_kv) {
break; // uniform across the workgroup
}
float acc[KQ_TN];
#pragma unroll
for (int t = 0; t < KQ_TN; ++t) {
acc[t] = 0.0f;
}
const int kl = min(kv, n_kv - 1);
const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
for (int b = 0; b < nblk; ++b) {
const uint4 w0 = vload4(0, &kq[kbase + b*8]);
const uint4 w1 = vload4(0, &kq[kbase + b*8 + 4]);
const float dks = (float)kd[kdbas + b];
#pragma unroll
for (int t = 0; t < KQ_TN; ++t) {
const uint4 a0 = vload4(0, &sh_q[t][b*8]);
const uint4 a1 = vload4(0, &sh_q[t][b*8 + 4]);
int raw = 0;
raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
}
}
// Each lane owns one complete 32-row softmax block after the transpose.
const int kvblk0 = kv - lid;
#pragma unroll
for (int half_ = 0; half_ < 2; ++half_) {
barrier(CLK_LOCAL_MEM_FENCE);
#pragma unroll
for (int j = 0; j < 16; ++j) {
p8_lds[j*64 + lid] = acc[half_*16 + j];
}
barrier(CLK_LOCAL_MEM_FENCE);
if (lid < 32) {
const int j = lid >> 1;
const int b = lid & 1;
const int n = qn0 + half_*16 + j;
if (n < n_q) {
const int kvb = kvblk0 + b*32;
const __local float * src = p8_lds + j*64 + b*32;
float16 s0 = vload16(0, src);
float16 s1 = vload16(1, src);
const global half * mp = mrow + (size_t)n*mstride + kvb;
const float16 m0v = convert_float16(vload16(0, mp));
const float16 m1v = convert_float16(vload16(1, mp));
s0 = s0*scale + slope*m0v;
s1 = s1*scale + slope*m1v;
const float16 mx16 = fmax(s0, s1);
const float8 mx8 = fmax(mx16.lo, mx16.hi);
const float4 mx4 = fmax(mx8.lo, mx8.hi);
const float2 mx2 = fmax(mx4.lo, mx4.hi);
const float amax = fmax(mx2.x, mx2.y);
const size_t prow = qbase + n;
const int blk = kvb >> 5;
float sum = 0.0f;
uchar16 q0 = (uchar16)(0);
uchar16 q1 = (uchar16)(0);
// finite sentinel: the program builds with -cl-finite-math-only
if (amax > -1.0e30f) {
const float16 e0 = exp(s0 - amax);
const float16 e1 = exp(s1 - amax);
const float16 a16 = e0 + e1;
const float8 a8 = a16.lo + a16.hi;
const float4 a4 = a8.lo + a8.hi;
const float2 a2 = a4.lo + a4.hi;
sum = a2.x + a2.y;
q0 = convert_uchar16_sat_rte(e0*255.0f);
q1 = convert_uchar16_sat_rte(e1*255.0f);
}
global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
vstore16(q0, 0, qdst);
vstore16(q1, 1, qdst);
bmax[prow*pblk + blk] = amax;
bsum[prow*pblk + blk] = sum;
}
}
}
}
}
@@ -0,0 +1,170 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
#ifdef cl_khr_integer_dot_product
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
#endif
#define KQ_TN 32
#define KQ_MB 8
#define KQ_TM 64
#define KQ_WG 64
#define KQ_DKC 256
__attribute__((qcom_wave_pair_mode(1)))
kernel void kernel_mul_mm_q8_kq_p8_dk512(
global const uint * kq, // K int8, [head_kv][n_kv][dk] as packed uints
global const half * kd, // K scales, [head_kv][n_kv][dk/32]
global const uint * qq, // Q int8, [head][n_q][dk] as packed uints
global const half * qd, // Q scales, [head][n_q][dk/32]
global uchar * qp, // u8 P, [head][n_q][p_pitch]
global float * bmax, // per 32-row block max, [head][n_q][p_pitch/32]
global float * bsum, // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
global const char * mask, // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
ulong offset_mask,
ulong mask_nb1,
float scale,
float max_bias,
float m0,
float m1,
int n_head_log2,
int dk,
int n_kv,
int n_q,
int n_head,
int n_head_kv,
int p_pitch // bytes per P row, >= n_kv, multiple of 32
) {
const int lid = get_local_id(0);
const int gsz = n_head / n_head_kv;
const int g0 = get_group_id(0);
const int head_kv = get_group_id(2);
const int head = head_kv*gsz + (g0 % gsz);
const int qn0 = (g0 / gsz) * KQ_TN;
const int kv0 = get_group_id(1)*KQ_TM*KQ_MB + lid;
const int nblk = dk / 32;
const int nu = dk / 4;
// Reuse the 256-wide Q slice for the F32 softmax scores after the contraction.
__local uint sh_q [KQ_TN*KQ_DKC/4];
__local half sh_qd[KQ_TN][KQ_DKC/32];
__local float * p8_lds = (__local float *)sh_q;
const size_t qbase = (size_t)head*n_q;
float slope = 1.0f;
if (max_bias > 0.0f) {
const float base = head < n_head_log2 ? m0 : m1;
const int ex = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
slope = pow(base, ex);
}
global const half * mrow = (global const half *)(mask + offset_mask);
const uint mstride = (uint)(mask_nb1 / 2);
const int pblk = p_pitch / 32;
for (int m = 0; m < KQ_MB; ++m) {
const int kv = kv0 + m*KQ_TM;
if (kv - lid >= n_kv) {
break; // uniform across the workgroup
}
float acc[KQ_TN];
#pragma unroll
for (int t = 0; t < KQ_TN; ++t) {
acc[t] = 0.0f;
}
const int kl = min(kv, n_kv - 1);
const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
for (int c0 = 0; c0 < nblk; c0 += KQ_DKC/32) {
const int cb = min(KQ_DKC/32, nblk - c0);
barrier(CLK_LOCAL_MEM_FENCE);
for (int i = lid; i < KQ_TN*(KQ_DKC/16); i += KQ_WG) {
const int t = i / (KQ_DKC/16);
const int u = (i % (KQ_DKC/16)) * 4;
const int qi = qn0 + t;
uint4 v = (uint4)(0u);
if (qi < n_q && u < cb*8) {
v = vload4(0, &qq[(qbase + qi)*nu + c0*8 + u]);
}
vstore4(v, 0, &sh_q[t*(KQ_DKC/4) + u]);
}
for (int i = lid; i < KQ_TN*(KQ_DKC/32); i += KQ_WG) {
const int t = i / (KQ_DKC/32);
const int b = i % (KQ_DKC/32);
const int qi = qn0 + t;
sh_qd[t][b] = (qi < n_q && b < cb) ? qd[(qbase + qi)*nblk + c0 + b] : (half)0.0f;
}
barrier(CLK_LOCAL_MEM_FENCE);
for (int b = 0; b < cb; ++b) {
const uint4 w0 = vload4(0, &kq[kbase + (c0 + b)*8]);
const uint4 w1 = vload4(0, &kq[kbase + (c0 + b)*8 + 4]);
const float dks = (float)kd[kdbas + c0 + b];
#pragma unroll
for (int t = 0; t < KQ_TN; ++t) {
const uint4 a0 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8]);
const uint4 a1 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8 + 4]);
int raw = 0;
raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
}
}
}
// Finish reading the Q slice before writing scores to the same local memory.
const int kvblk0 = kv - lid;
barrier(CLK_LOCAL_MEM_FENCE);
#pragma unroll
for (int t = 0; t < KQ_TN; ++t) {
p8_lds[t*64 + lid] = acc[t];
}
barrier(CLK_LOCAL_MEM_FENCE);
{
const int j = lid >> 1;
const int b = lid & 1;
const int n = qn0 + j;
if (n < n_q) {
const int kvb = kvblk0 + b*32;
__local float * src = p8_lds + j*64 + b*32;
const global half * mp = mrow + (size_t)n*mstride + kvb;
float4 s = vload4(0, src)*scale + slope*convert_float4(vload4(0, mp));
vstore4(s, 0, src);
float amax = fmax(fmax(s.x, s.y), fmax(s.z, s.w));
for (int c = 1; c < 8; ++c) {
s = vload4(c, src)*scale + slope*convert_float4(vload4(c, mp));
vstore4(s, c, src);
amax = fmax(amax, fmax(fmax(s.x, s.y), fmax(s.z, s.w)));
}
const size_t prow = qbase + n;
const int blk = kvb >> 5;
global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
float sum = 0.0f;
// finite sentinel: the program builds with -cl-finite-math-only
if (amax > -1.0e30f) {
for (int c = 0; c < 8; ++c) {
const float4 e = exp(vload4(c, src) - amax);
sum += (e.x + e.y) + (e.z + e.w);
vstore4(convert_uchar4_sat_rte(e*255.0f), c, qdst);
}
} else {
for (int c = 0; c < 8; ++c) {
vstore4((uchar4)(0), c, qdst);
}
}
bmax[prow*pblk + blk] = amax;
bsum[prow*pblk + blk] = sum;
}
}
}
}
+11 -2
View File
@@ -1,5 +1,7 @@
#include "models.h"
#include <cinttypes>
// HRM-Text: alternating low/high transformer stacks over the same token stream.
// Reference: HrmTextModel in transformers, DFM Mimir 1B.
@@ -19,8 +21,15 @@ void llama_model_hrm_text::load_arch_hparams(llama_model_loader & ml) {
GGML_ASSERT(hparams.n_hrm_l_cycles > 0);
// the GGUF block count is the expanded cache-slot count
const uint32_t n_slot = hparams.n_hrm_layers_per_stack * hparams.n_hrm_h_cycles * (hparams.n_hrm_l_cycles + 1);
GGML_ASSERT(hparams.n_layer() == n_slot);
// 64-bit: a 32-bit product can wrap and match n_layer(), then load_arch_tensors() indexes past model.layers
const uint64_t n_slot = (uint64_t) hparams.n_hrm_layers_per_stack * hparams.n_hrm_h_cycles * (hparams.n_hrm_l_cycles + 1);
if (hparams.n_hrm_layers_per_stack >= hparams.n_layer() || n_slot != hparams.n_layer()) {
throw std::runtime_error(format(
"hrm: cache-slot count %" PRIu64 " (layers_per_stack %u, h_cycles %u, l_cycles %u) "
"does not match block_count %u",
n_slot, hparams.n_hrm_layers_per_stack, hparams.n_hrm_h_cycles,
hparams.n_hrm_l_cycles, hparams.n_layer()));
}
switch (hparams.n_embd) {
case 1536:
+3 -3
View File
@@ -12407,6 +12407,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, b, 50, 200, 64));
}
// One very large one to trigger a code path in CUDA requiring >1 pass over expert IDs due to shared memory limits:
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 16, 16, false, 32, 65536, 256));
// For issue 27873
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_IQ2_XXS, GGML_TYPE_F32, 1, 1, false, 1, 8192, 4096));
@@ -12441,8 +12444,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 28, 10, false, 1024, 1, 4096));
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 128, 8, false, 1024, 1, 2048));
// TODO: disabled until CUDA backend support is added [TAG_MUL_MAT_ID_DUPLICATE_ID]
#if 0
// repeated expert ids within a row
for (ggml_type ta : {GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) {
for (int n : {9, 16, 33, 64}) {
@@ -12454,7 +12455,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 1025, 10, false, 64, 33, 256, 0, true));
test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 2050, 10, true, 64, 33, 256));
}
#endif
for (ggml_type type_a : all_types) {
test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 2, false, 64, 16, 3*ggml_blck_size(type_a)));