mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-11-19 17:27:41 -06:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b94959a64d | ||
|
|
50518b64c6 | ||
|
|
c406f4367e |
@@ -1316,12 +1316,6 @@ extern "C" {
|
||||
|
||||
// indirect matrix multiplication
|
||||
// an id of -1 skips the slot and zeroes the matching row of the result
|
||||
//
|
||||
// [TAG_MUL_MAT_ID_DUPLICATE_ID]
|
||||
// note: this op has undefined behaviour if a given `a` is referenced more than once for a given `b`
|
||||
// (i.e. duplicate experts per token are not guaranteed to be supported in all backends)
|
||||
// TODO: add backend support: https://github.com/ggml-org/llama.cpp/pull/29998#issuecomment-6153742533
|
||||
// the CPU backend skips negative ids, their rows of the result are zero
|
||||
GGML_API struct ggml_tensor * ggml_mul_mat_id(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * as,
|
||||
|
||||
@@ -452,6 +452,16 @@ static __device__ __forceinline__ ggml_cuda_thread_mask_t ggml_cuda_ballot(int p
|
||||
return __ballot_sync(mask, predicate);
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ int ggml_cuda_popc(ggml_cuda_thread_mask_t x) {
|
||||
static_assert(sizeof(ggml_cuda_thread_mask_t) == 4 || sizeof(ggml_cuda_thread_mask_t) == 8, "unsupported thread mask type");
|
||||
if constexpr (sizeof(ggml_cuda_thread_mask_t) == 4) {
|
||||
return __popc(x);
|
||||
} else {
|
||||
return __popcll(x);
|
||||
}
|
||||
}
|
||||
template<typename T> static __device__ __forceinline__ int ggml_cuda_popc(T x) = delete; // Makes implicit type casts result in an error.
|
||||
|
||||
static __device__ __forceinline__ int ggml_cuda_all(int predicate, ggml_cuda_thread_mask_t mask = GGML_CUDA_FULL_THREAD_MASK) {
|
||||
return __all_sync(mask, predicate);
|
||||
}
|
||||
|
||||
@@ -17,15 +17,6 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
constexpr int values_per_lane = 8;
|
||||
using lane_mask_t = std::conditional_t<warp_size == 64, uint64_t, uint32_t>;
|
||||
constexpr uint64_t ballot_mask = ~(lane_mask_t)0;
|
||||
const auto popc = [](const lane_mask_t mask) {
|
||||
if constexpr (warp_size == 64) {
|
||||
return __popcll(mask);
|
||||
} else {
|
||||
return __popc(mask);
|
||||
}
|
||||
};
|
||||
const int tid = threadIdx.x;
|
||||
const int warp = tid / warp_size;
|
||||
const int lane = tid % warp_size;
|
||||
@@ -48,7 +39,7 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
__syncthreads();
|
||||
|
||||
for (int i0 = 0; i0 < ne30; i0 += blockDim.x*values_per_lane) {
|
||||
lane_mask_t selected_warp[values_per_lane];
|
||||
ggml_cuda_thread_mask_t selected_warp[values_per_lane];
|
||||
int warp_count = 0;
|
||||
#pragma unroll
|
||||
for (int item = 0; item < values_per_lane; ++item) {
|
||||
@@ -60,8 +51,8 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
selected |= (!oob || q < q1 - q0) && isfinite(__half2float(mask[q*s31 + i]));
|
||||
}
|
||||
}
|
||||
selected_warp[item] = (lane_mask_t)ggml_cuda_ballot((int) selected, ballot_mask);
|
||||
warp_count += popc(selected_warp[item]);
|
||||
selected_warp[item] = ggml_cuda_ballot((int) selected);
|
||||
warp_count += ggml_cuda_popc(selected_warp[item]);
|
||||
}
|
||||
|
||||
if (lane == 0) {
|
||||
@@ -81,16 +72,16 @@ static __global__ void flash_attn_mask_to_sparse_indices(
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
const lane_mask_t lane_mask = lane == 0 ? 0 : ((lane_mask_t)1 << lane) - 1;
|
||||
const ggml_cuda_thread_mask_t lane_mask = lane == 0 ? 0 : ((ggml_cuda_thread_mask_t)1 << lane) - 1;
|
||||
int warp_item_offset = 0;
|
||||
#pragma unroll
|
||||
for (int item = 0; item < values_per_lane; ++item) {
|
||||
const int i = i0 + (warp*values_per_lane + item)*warp_size + lane;
|
||||
const int dst = row_count + warp_offsets[warp] + warp_item_offset + popc(selected_warp[item] & lane_mask);
|
||||
if ((selected_warp[item] & ((lane_mask_t)1 << lane)) && dst < n_kv_max) {
|
||||
const int dst = row_count + warp_offsets[warp] + warp_item_offset + ggml_cuda_popc(selected_warp[item] & lane_mask);
|
||||
if ((selected_warp[item] & ((ggml_cuda_thread_mask_t)1 << lane)) && dst < n_kv_max) {
|
||||
indices[dst] = i;
|
||||
}
|
||||
warp_item_offset += popc(selected_warp[item]);
|
||||
warp_item_offset += ggml_cuda_popc(selected_warp[item]);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
|
||||
@@ -2730,7 +2730,8 @@ static void ggml_cuda_mul_mat_id(ggml_backend_cuda_context & ctx, ggml_tensor *
|
||||
ids_from_sorted_host[i12*n_expert_used + iex] = ids_to_sorted_host.size();
|
||||
ids_to_sorted_host.push_back(i12*ne11 + iex % ne11);
|
||||
tokens_per_expert[i02]++;
|
||||
break;
|
||||
// TODO: removing the break here was done to make duplicate IDs work but this caused a ~10% performance regression, better solution needed.
|
||||
// break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,7 +43,6 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
const int64_t s13 = src1->nb[3] / ts_src1;
|
||||
const int64_t s3 = dst->nb[3] / ts_dst;
|
||||
|
||||
const int64_t ids_s0 = ids ? ids->nb[0] / ggml_type_size(ids->type) : 0;
|
||||
const int64_t ids_s1 = ids ? ids->nb[1] / ggml_type_size(ids->type) : 0;
|
||||
|
||||
mmf_ids_data ids_info{};
|
||||
@@ -70,7 +69,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
nchannels_y = ids->ne[0];
|
||||
}
|
||||
|
||||
if (ids && ncols_dst > 16) {
|
||||
if (ids) {
|
||||
const int64_t n_expert_used = ids->ne[0];
|
||||
const int64_t n_experts = ne02;
|
||||
const int64_t n_tokens = ne12;
|
||||
@@ -110,7 +109,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
constexpr int vals_per_T = 1;
|
||||
mul_mat_f_switch_rows_per_block<float>(
|
||||
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
|
||||
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
|
||||
} break;
|
||||
case GGML_TYPE_F16: {
|
||||
@@ -118,7 +117,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
constexpr int vals_per_T = 2;
|
||||
mul_mat_f_switch_rows_per_block<half2>(
|
||||
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
|
||||
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
|
||||
} break;
|
||||
case GGML_TYPE_BF16: {
|
||||
@@ -126,7 +125,7 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
constexpr int vals_per_T = 2;
|
||||
mul_mat_f_switch_rows_per_block<nv_bfloat162>(
|
||||
rows_per_block, src0_d, src1_d, ids_d, dst_d, ne00/vals_per_T, ne01, ncols_dst, s01/vals_per_T, stride_col_y/vals_per_T, stride_col_dst,
|
||||
ids_s0, ids_s1, ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne02, nchannels_y, nchannels_dst, s02/vals_per_T, stride_channel_y, stride_channel_dst,
|
||||
ne03, ne3, s03/vals_per_T, s13, s3, ctx.stream(), ids_info_ptr);
|
||||
} break;
|
||||
default:
|
||||
|
||||
+69
-176
@@ -47,12 +47,11 @@ void ggml_cuda_mul_mat_f(ggml_backend_cuda_context & ctx, const ggml_tensor * sr
|
||||
|
||||
bool ggml_cuda_should_use_mmf(enum ggml_type type, int cc, int warp_size, const int64_t * scr0_ne, const size_t * src0_nb, const int src1_ncols, bool mul_mat_id);
|
||||
|
||||
template <typename T, int rows_per_block, int cols_per_block, int nwarps, bool has_ids>
|
||||
template <typename T, int rows_per_block, int cols_per_block, int nwarps>
|
||||
__launch_bounds__(ggml_cuda_get_physical_warp_size()*nwarps, 1)
|
||||
static __global__ void mul_mat_f(
|
||||
const T * __restrict__ x, const float * __restrict__ y, const int32_t * __restrict__ ids, float * __restrict__ dst,
|
||||
const int ncols, const int ncols_dst_total, const int nchannels_dst, const int stride_row, const int stride_col_y, const int stride_col_dst,
|
||||
const int stride_col_id, const int stride_row_id,
|
||||
const T * __restrict__ x, const float * __restrict__ y, float * __restrict__ dst,
|
||||
const int ncols, const int stride_row, const int stride_col_y, const int stride_col_dst,
|
||||
const int channel_ratio, const int stride_channel_x, const int stride_channel_y, const int stride_channel_dst,
|
||||
const int sample_ratio, const int stride_sample_x, const int stride_sample_y, const int stride_sample_dst) {
|
||||
// TODO: handle this in a consistent and simpler way after AMD MFMA support has been added
|
||||
@@ -90,85 +89,26 @@ static __global__ void mul_mat_f(
|
||||
constexpr int ntA = rows_per_block / tile_A::I;
|
||||
constexpr int ntB = (cols_per_block + tile_B::I - 1) / tile_B::I;
|
||||
|
||||
const int row0 = blockIdx.x * rows_per_block;
|
||||
const int row0 = blockIdx.x * rows_per_block;
|
||||
|
||||
int expert_idx = 0;
|
||||
[[maybe_unused]] int col_base = 0;
|
||||
|
||||
const int channel_dst = has_ids ? 0 : blockIdx.y;
|
||||
|
||||
if constexpr (has_ids) {
|
||||
// experts + tiles of ncols_dst are packed in the y dimension
|
||||
int col_tiles = (ncols_dst_total + cols_per_block - 1) / cols_per_block;
|
||||
const int nchannels_x = gridDim.y / col_tiles;
|
||||
const int tile_idx = blockIdx.y / nchannels_x;
|
||||
expert_idx = blockIdx.y - tile_idx * nchannels_x;
|
||||
col_base = tile_idx * cols_per_block;
|
||||
}
|
||||
|
||||
const int channel_x = has_ids ? expert_idx : (channel_dst / channel_ratio);
|
||||
const int channel_dst = blockIdx.y;
|
||||
const int channel_x = channel_dst / channel_ratio;
|
||||
const int channel_y = channel_dst;
|
||||
const int sample_dst = blockIdx.z;
|
||||
const int sample_x = sample_dst / sample_ratio;
|
||||
const int sample_y = sample_dst;
|
||||
|
||||
x += int64_t(sample_x) *stride_sample_x + channel_x *stride_channel_x + row0*stride_row ;
|
||||
y += int64_t(sample_y) *stride_sample_y + (has_ids ? 0 : channel_y *stride_channel_y);
|
||||
dst += int64_t(sample_dst)*stride_sample_dst + (has_ids ? 0 : channel_dst*stride_channel_dst);
|
||||
|
||||
if constexpr (has_ids) {
|
||||
constexpr int y_stride_scale = std::is_same_v<T, float> ? 1 : 2;
|
||||
const int64_t col_offset = col_base;
|
||||
y += col_offset * stride_col_y * y_stride_scale;
|
||||
dst += col_offset * stride_col_dst;
|
||||
ids += col_offset * stride_row_id;
|
||||
}
|
||||
y += int64_t(sample_y) *stride_sample_y + channel_y *stride_channel_y;
|
||||
dst += int64_t(sample_dst)*stride_sample_dst + channel_dst*stride_channel_dst;
|
||||
|
||||
[[maybe_unused]] const float2 * y2 = (const float2 *) y;
|
||||
|
||||
extern __shared__ char data_mmv[];
|
||||
|
||||
char * shmem_base = data_mmv;
|
||||
[[maybe_unused]] int * slot_map = (int *) shmem_base;
|
||||
char * compute_base = has_ids ? (shmem_base + GGML_PAD(cols_per_block, 16) * sizeof(int)) : shmem_base;
|
||||
|
||||
tile_C C[ntA][ntB];
|
||||
|
||||
T * tile_xy = (T *) compute_base + threadIdx.y*(tile_A::I * tile_k_padded);
|
||||
|
||||
if constexpr (has_ids) {
|
||||
int found = 0;
|
||||
|
||||
for (int j0 = 0; j0 < cols_per_block; j0 += nwarps) {
|
||||
const int j = j0 + threadIdx.y;
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
slot_map[j] = -1;
|
||||
}
|
||||
ggml_cuda_syncwarp();
|
||||
|
||||
if (col_base + j >= ncols_dst_total) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const int32_t * __restrict__ id_row = ids + j*stride_row_id;
|
||||
|
||||
for (int k = threadIdx.x; k < nchannels_dst; k += warp_size) {
|
||||
int match = id_row[k*stride_col_id] == expert_idx;
|
||||
|
||||
if (match) {
|
||||
slot_map[j] = k;
|
||||
found = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!__syncthreads_or(found)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
T * tile_xy = (T *) data_mmv + threadIdx.y*(tile_A::I * tile_k_padded);
|
||||
|
||||
for (int col = threadIdx.y*warp_size + threadIdx.x; col < ncols; col += nwarps*warp_size) {
|
||||
tile_A A[ntA][warp_size / tile_A::J];
|
||||
@@ -194,26 +134,15 @@ static __global__ void mul_mat_f(
|
||||
for (int j0 = 0; j0 < tile_B::I; ++j0) {
|
||||
const int j = j0 + itB*tile_B::I;
|
||||
|
||||
if constexpr (!has_ids) {
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = j < cols_per_block ? y[j*stride_col_y + col] : 0.0f;
|
||||
} else {
|
||||
const bool valid = j < cols_per_block && (col_base + j) < ncols_dst_total && slot_map[j] >= 0;
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = valid ? y[slot_map[j]*stride_channel_y + j*stride_col_y + col] : 0.0f;
|
||||
}
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = j < cols_per_block ? y[j*stride_col_y + col] : 0.0f;
|
||||
}
|
||||
} else if constexpr (std::is_same_v<T, half2> || std::is_same_v<T, nv_bfloat162>) {
|
||||
#pragma unroll
|
||||
for (int j0 = 0; j0 < tile_B::I; ++j0) {
|
||||
const int j = j0 + itB*tile_B::I;
|
||||
|
||||
if constexpr (!has_ids) {
|
||||
const float2 tmp = j < cols_per_block ? y2[j*stride_col_y + col] : make_float2(0.0f, 0.0f);
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
|
||||
} else {
|
||||
const bool valid = j < cols_per_block && (col_base + j) < ncols_dst_total && slot_map[j] >= 0;
|
||||
float2 tmp = valid ? *(const float2*) &y[slot_map[j]*stride_channel_y + 2*(j*stride_col_y + col)] : make_float2(0.0f, 0.0f);
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
|
||||
}
|
||||
const float2 tmp = j < cols_per_block ? y2[j*stride_col_y + col] : make_float2(0.0f, 0.0f);
|
||||
tile_xy[j0*tile_k_padded + threadIdx.x] = ggml_cuda_cast<T>(tmp);
|
||||
}
|
||||
} else {
|
||||
static_assert(std::is_same_v<T, void>, "unsupported type");
|
||||
@@ -231,7 +160,7 @@ static __global__ void mul_mat_f(
|
||||
}
|
||||
}
|
||||
|
||||
float * buf_iw = (float *) compute_base;
|
||||
float * buf_iw = (float *) data_mmv;
|
||||
constexpr int kiw = nwarps*rows_per_block + mmf_get_padding();
|
||||
|
||||
if (nwarps > 1) {
|
||||
@@ -278,26 +207,15 @@ static __global__ void mul_mat_f(
|
||||
}
|
||||
}
|
||||
|
||||
if constexpr (!has_ids) {
|
||||
#pragma unroll
|
||||
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
|
||||
dst[j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
|
||||
}
|
||||
} else {
|
||||
const int slot = (j < cols_per_block) ? slot_map[j] : -1;
|
||||
if (slot >= 0 && (col_base + j) < ncols_dst_total) {
|
||||
#pragma unroll
|
||||
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
|
||||
dst[slot*stride_channel_dst + j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
|
||||
}
|
||||
}
|
||||
for (int i0 = 0; i0 < sizeof(sum)/sizeof(sum[0]); ++i0) {
|
||||
dst[j*stride_col_dst + row0 + i0*warp_size + threadIdx.x] = sum[i0];
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
GGML_UNUSED_VARS(x, y, ids, dst,
|
||||
ncols, ncols_dst_total, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id,
|
||||
GGML_UNUSED_VARS(x, y, dst,
|
||||
ncols, stride_row, stride_col_y, stride_col_dst,
|
||||
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
|
||||
NO_DEVICE_CODE;
|
||||
@@ -588,22 +506,21 @@ static __global__ void mul_mat_f_ids(
|
||||
|
||||
template<typename T, int rows_per_block, int cols_per_block, int nwarps>
|
||||
static inline void mul_mat_f_switch_ids(
|
||||
const T * x, const float * y, const int32_t * ids, float * dst,
|
||||
const T * x, const float * y, float * dst,
|
||||
const int64_t ncols_x, const int64_t ncols_dst, const int64_t nchannels_dst,
|
||||
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
|
||||
const int64_t stride_col_id, const int64_t stride_row_id,
|
||||
const int64_t channel_ratio, const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst,
|
||||
const int64_t sample_ratio, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
|
||||
const dim3 & block_nums, const dim3 & block_dims, const int nbytes_shared_total, cudaStream_t stream,
|
||||
const mmf_ids_data * ids_data) {
|
||||
|
||||
const bool has_ids_data = ids_data && ids_data->ids_src_compact;
|
||||
|
||||
// Use the compact-ids kernel only for larger tiles; for small ncols_dst (< 16)
|
||||
// we prefer the normal mul_mat_f path with has_ids=true.
|
||||
if (has_ids_data && ncols_dst > 16) {
|
||||
if (has_ids_data) {
|
||||
if (ids_data->max_tiles == 0) {
|
||||
return;
|
||||
}
|
||||
GGML_ASSERT(block_nums.z == 1);
|
||||
dim3 block_nums_ids(block_nums.x, ids_data->max_tiles, 1);
|
||||
|
||||
const uint3 sis1_fd = ids_data->sis1 > 0 ? init_fastdiv_values((uint32_t) ids_data->sis1) : make_uint3(0, 0, 1);
|
||||
@@ -615,19 +532,10 @@ static inline void mul_mat_f_switch_ids(
|
||||
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
sis1_fd, nch_fd);
|
||||
} else if (ids) {
|
||||
const int64_t col_tiles = (ncols_dst + cols_per_block - 1) / cols_per_block;
|
||||
dim3 block_nums_ids = block_nums;
|
||||
block_nums_ids.y *= col_tiles;
|
||||
|
||||
mul_mat_f<T, rows_per_block, cols_per_block, nwarps, true><<<block_nums_ids, block_dims, nbytes_shared_total, stream>>>
|
||||
(x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
|
||||
} else {
|
||||
mul_mat_f<T, rows_per_block, cols_per_block, nwarps, false><<<block_nums, block_dims, nbytes_shared_total, stream>>>
|
||||
(x, y, ids, dst, ncols_x, cols_per_block, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
mul_mat_f<T, rows_per_block, cols_per_block, nwarps><<<block_nums, block_dims, nbytes_shared_total, stream>>>
|
||||
(x, y, dst, ncols_x, stride_row, stride_col_y, stride_col_dst,
|
||||
channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst);
|
||||
}
|
||||
}
|
||||
@@ -637,7 +545,6 @@ void mul_mat_f_cuda(
|
||||
const T * x, const float * y, const int32_t * ids, float * dst,
|
||||
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
|
||||
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
|
||||
const int64_t stride_col_id, const int64_t stride_row_id,
|
||||
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
|
||||
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
|
||||
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
|
||||
@@ -673,70 +580,59 @@ void mul_mat_f_cuda(
|
||||
const int nbytes_shared_iter = nwarps_best * (volta_mma_available(cc) ? tile_A_32::I : tile_A_16::I) * (warp_size + mmf_get_padding(cc)) * 4;
|
||||
const int nbytes_cols_per_block_pad = (amd_wmma_available(cc) || amd_mfma_available(cc)) ? tile_B_16::I : tile_B_8::I;
|
||||
const int nbytes_shared_combine = GGML_PAD(cols_per_block, nbytes_cols_per_block_pad) * (nwarps_best*rows_per_block + mmf_get_padding(cc)) * 4;
|
||||
const int nbytes_shared = std::max(nbytes_shared_iter, nbytes_shared_combine);
|
||||
const int nbytes_slotmap = ids ? GGML_PAD(cols_per_block, 16) * sizeof(int) : 0;
|
||||
const int nbytes_shared_total = nbytes_shared + nbytes_slotmap;
|
||||
const int64_t grid_y = ids ? nchannels_x : nchannels_dst;
|
||||
const int nbytes_shared_total = std::max(nbytes_shared_iter, nbytes_shared_combine);
|
||||
|
||||
const dim3 block_nums(nrows_x/rows_per_block, grid_y, nsamples_dst);
|
||||
const dim3 block_nums(nrows_x/rows_per_block, nchannels_dst, nsamples_dst);
|
||||
const dim3 block_dims(warp_size, nwarps_best, 1);
|
||||
|
||||
switch (nwarps_best) {
|
||||
case 1: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 1>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 2: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 2>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 3: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 3>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 4: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 4>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 5: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 5>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 6: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 6>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 7: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 7>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
case 8: {
|
||||
mul_mat_f_switch_ids<T, rows_per_block, cols_per_block, 8>(
|
||||
x, y, ids, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, channel_ratio, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst, block_nums, block_dims, nbytes_shared_total, stream,
|
||||
ids_data);
|
||||
x, y, dst, ncols_x, ncols_dst, nchannels_dst, stride_row, stride_col_y, stride_col_dst, channel_ratio,
|
||||
stride_channel_x, stride_channel_y, stride_channel_dst, sample_ratio, stride_sample_x, stride_sample_y, stride_sample_dst,
|
||||
block_nums, block_dims, nbytes_shared_total, stream, ids_data);
|
||||
} break;
|
||||
default:
|
||||
GGML_ABORT("fatal error");
|
||||
@@ -750,95 +646,94 @@ static void mul_mat_f_switch_cols_per_block(
|
||||
const T * x, const float * y, const int32_t * ids, float * dst,
|
||||
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
|
||||
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
|
||||
const int64_t stride_col_id, const int stride_row_id,
|
||||
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
|
||||
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
|
||||
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
|
||||
cudaStream_t stream, const mmf_ids_data * ids_data) {
|
||||
|
||||
const int ncols_case = (ids && ncols_dst > 16) ? 16 : ncols_dst;
|
||||
const int ncols_case = ids ? 16 : ncols_dst;
|
||||
|
||||
GGML_ASSERT(ids || ncols_dst <= 16);
|
||||
|
||||
switch (ncols_case) {
|
||||
case 1: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 1>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 2: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 2>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 3: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 3>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 4: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 4>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 5: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 5>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 6: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 6>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 7: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 7>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 8: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 8>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 9: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 9>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 10: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 10>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 11: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 11>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 12: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 12>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 13: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 13>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 14: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 14>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 15: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 15>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case 16: {
|
||||
mul_mat_f_cuda<T, rows_per_block, 16>(x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
default:
|
||||
@@ -851,7 +746,6 @@ static void mul_mat_f_switch_rows_per_block(
|
||||
const int rows_per_block, const T * x, const float * y, const int32_t * ids, float * dst,
|
||||
const int64_t ncols_x, const int64_t nrows_x, const int64_t ncols_dst,
|
||||
const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst,
|
||||
const int64_t stride_col_id, const int stride_row_id,
|
||||
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst,
|
||||
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,
|
||||
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst,
|
||||
@@ -860,13 +754,13 @@ static void mul_mat_f_switch_rows_per_block(
|
||||
case MMF_ROWS_PER_BLOCK: {
|
||||
mul_mat_f_switch_cols_per_block<T, MMF_ROWS_PER_BLOCK>(
|
||||
x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
case MMF_ROWS_PER_BLOCK_CDNA: {
|
||||
mul_mat_f_switch_cols_per_block<T, MMF_ROWS_PER_BLOCK_CDNA>(
|
||||
x, y, ids, dst, ncols_x, nrows_x, ncols_dst, stride_row, stride_col_y, stride_col_dst,
|
||||
stride_col_id, stride_row_id, nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst,
|
||||
nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, stream, ids_data);
|
||||
} break;
|
||||
default:
|
||||
@@ -878,7 +772,6 @@ static void mul_mat_f_switch_rows_per_block(
|
||||
template void mul_mat_f_cuda<T, nrows_dst, ncols_dst>( \
|
||||
const T * x, const float * y, const int32_t * ids, float * dst, \
|
||||
const int64_t ncols_x, const int64_t nrows_x, int64_t ncols_dst_total, const int64_t stride_row, const int64_t stride_col_y, const int64_t stride_col_dst, \
|
||||
const int64_t stride_col_id, const int64_t stride_row_id, \
|
||||
const int64_t nchannels_x, const int64_t nchannels_y, const int64_t nchannels_dst, \
|
||||
const int64_t stride_channel_x, const int64_t stride_channel_y, const int64_t stride_channel_dst, const int64_t nsamples_x,\
|
||||
const int64_t nsamples_dst, const int64_t stride_sample_x, const int64_t stride_sample_y, const int64_t stride_sample_dst, \
|
||||
|
||||
+159
-60
@@ -24,6 +24,50 @@ template <int n> struct mm_ids_pow2 { static constexpr int value = 2*mm_ids_pow2
|
||||
template <> struct mm_ids_pow2<1> { static constexpr int value = 1; };
|
||||
template <> struct mm_ids_pow2<0> { static constexpr int value = 1; };
|
||||
|
||||
template <bool increment_nex_prev, bool write_to_store>
|
||||
static __device__ __forceinline__ void check_experts(
|
||||
mm_ids_helper_store * store, int & nex_prev, int & it_compact,
|
||||
const int expert, const int it, const int iex, const int expert_found) {
|
||||
if (increment_nex_prev) {
|
||||
nex_prev += expert_found < expert;
|
||||
}
|
||||
|
||||
if (write_to_store) {
|
||||
const ggml_cuda_thread_mask_t iex_used = ggml_cuda_ballot(expert_found == expert);
|
||||
const ggml_cuda_thread_mask_t mask = (ggml_cuda_thread_mask_t(1) << threadIdx.x) - 1;
|
||||
const int it_compact_offset = ggml_cuda_popc(iex_used & mask);
|
||||
|
||||
if (expert_found == expert) {
|
||||
store[it_compact + it_compact_offset] = mm_ids_helper_store(it, iex);
|
||||
}
|
||||
|
||||
it_compact += ggml_cuda_popc(iex_used);
|
||||
}
|
||||
}
|
||||
|
||||
static __device__ __forceinline__ void dump_store(
|
||||
int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst,
|
||||
const int n_expert_used, const int nchannels_y, const int sis1, const bool write_inverse,
|
||||
const mm_ids_helper_store * __restrict__ store, const int nex_prev, int & n_dumped, const int it_compact) {
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
|
||||
for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {
|
||||
const mm_ids_helper_store store_it = store[itc];
|
||||
const int it = store_it.it();
|
||||
const int iex_used = store_it.iex_used();
|
||||
ids_dst[nex_prev + n_dumped + itc] = it*n_expert_used + iex_used;
|
||||
// ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup
|
||||
if (write_inverse) {
|
||||
ids_src1[it*n_expert_used + iex_used] = nex_prev + n_dumped + itc;
|
||||
} else {
|
||||
ids_src1[nex_prev + n_dumped + itc] = it*sis1 + iex_used % nchannels_y;
|
||||
}
|
||||
}
|
||||
n_dumped += it_compact;
|
||||
}
|
||||
|
||||
constexpr int n_unroll = 4; // Manually unroll this many loop iterations for better performance.
|
||||
|
||||
// Helper function for mul_mat_id, converts ids to a more convenient format.
|
||||
// ids_src1 describes how to permute the flattened column indices of src1 in order to get a compact src1 tensor sorted by expert.
|
||||
// ids_dst describes the same mapping but for the dst tensor.
|
||||
@@ -36,85 +80,138 @@ static __global__ void mm_ids_helper(
|
||||
const int32_t * __restrict__ ids,
|
||||
int32_t * __restrict__ ids_src1, int32_t * __restrict__ ids_dst, ggml_cuda_mm_id_tile_spec * __restrict__ tile_spec,
|
||||
const int n_tokens, const int n_expert_used_var, const int nchannels_y, const uint3 tile_width,
|
||||
const int si1, const int sis1, const bool write_inverse) {
|
||||
const int si1, const int sis1, const bool write_inverse, const int ne_store) {
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
const int n_expert_used = n_expert_used_template == 0 ? n_expert_used_var : n_expert_used_template;
|
||||
const int expert = blockIdx.x;
|
||||
|
||||
const int expert = blockIdx.x; // The expert this CUDA block handles.
|
||||
|
||||
extern __shared__ char data_mm_ids_helper[];
|
||||
mm_ids_helper_store * store = (mm_ids_helper_store *) data_mm_ids_helper;
|
||||
|
||||
int nex_prev = 0; // Number of columns for experts with a lower index.
|
||||
int n_dumped = 0; // Number of data values that have been dumped to output buffers.
|
||||
int it_compact = 0; // Running index for the compact slice of this expert.
|
||||
|
||||
// Step 1: Iterate over n_tokens, increment nex_prev to determine the output indices, fill up store.
|
||||
// Step 1.5: If the store does not have enough space to hold all values, iterate over the rest of the tokens just to increment nex_prev.
|
||||
// Step 2: Dump the stored values using nex_prev to get the correct output indices.
|
||||
// Step 3: If we were not able to process all values, resume at the index up to which we were able to dump the output and re-do.
|
||||
|
||||
if constexpr (n_expert_used_template == 0) {
|
||||
// Generic implementation:
|
||||
for (int it = 0; it < n_tokens; ++it) {
|
||||
int iex_used = -1; // The index at which the expert is used, if any.
|
||||
for (int iex = threadIdx.x; iex < n_expert_used; iex += warp_size) {
|
||||
const int expert_used = ids[it*si1 + iex];
|
||||
nex_prev += expert_used < expert;
|
||||
if (expert_used == expert) {
|
||||
iex_used = iex;
|
||||
int it = 0;
|
||||
for (; it < n_tokens && it_compact + max(warp_size, n_expert_used) <= ne_store; ++it) {
|
||||
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
|
||||
const int iex = iex0 + threadIdx.x;
|
||||
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = true;
|
||||
constexpr bool write_to_store = true;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
}
|
||||
}
|
||||
int it_dumped = it;
|
||||
for (; it < n_tokens; ++it) {
|
||||
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
|
||||
const int iex = iex0 + threadIdx.x;
|
||||
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = true;
|
||||
constexpr bool write_to_store = false;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
}
|
||||
}
|
||||
|
||||
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
it_compact = 0;
|
||||
|
||||
for (int it = it_dumped; it < n_tokens; ++it) {
|
||||
for (int iex0 = 0; iex0 < n_expert_used; iex0 += warp_size) {
|
||||
const int iex = iex0 + threadIdx.x;
|
||||
const int expert_found = iex < n_expert_used ? ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = false;
|
||||
constexpr bool write_to_store = true;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
if (it_compact + max(warp_size, n_expert_used) >= ne_store) {
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
it_compact = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (iex_used != -1) {
|
||||
store[it_compact] = mm_ids_helper_store(it, iex_used);
|
||||
}
|
||||
|
||||
if (warp_reduce_any<warp_size>(iex_used != -1)) {
|
||||
it_compact++;
|
||||
}
|
||||
}
|
||||
if (it_compact > 0) {
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
}
|
||||
} else {
|
||||
// Implementation optimized for specific numbers of experts used:
|
||||
// a warp holds a whole number of token slots, so the slot count is padded to a power of 2
|
||||
constexpr int neu_padded = mm_ids_pow2<n_expert_used_template>::value;
|
||||
static_assert(neu_padded <= warp_size && warp_size % neu_padded == 0, "bad n_expert_used");
|
||||
for (int it0 = 0; it0 < n_tokens; it0 += warp_size/neu_padded) {
|
||||
const int it = it0 + threadIdx.x / neu_padded;
|
||||
const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any.
|
||||
|
||||
const int iex = threadIdx.x % neu_padded; // The index at which the expert is used, if any.
|
||||
const int expert_used = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
|
||||
ids[it*si1 + iex] : INT_MAX;
|
||||
const int iex_used = expert_used == expert ? iex : -1;
|
||||
nex_prev += expert_used < expert;
|
||||
|
||||
// Whether the threads at this token position have used the expert:
|
||||
const int it_compact_add_self = warp_reduce_any<neu_padded>(iex_used != -1);
|
||||
|
||||
// Do a scan over threads at lower token positions in warp to get the correct index for writing data:
|
||||
int it_compact_add_lower = 0;
|
||||
int it0 = 0;
|
||||
for (; it0 < n_tokens && it_compact + n_unroll*warp_size <= ne_store; it0 += n_unroll*warp_size/neu_padded) {
|
||||
#pragma unroll
|
||||
for (int offset = neu_padded; offset < warp_size; offset += neu_padded) {
|
||||
const int tmp = ggml_cuda_shfl_up(it_compact_add_self, offset, warp_size);
|
||||
if (threadIdx.x >= static_cast<unsigned int>(offset)) {
|
||||
it_compact_add_lower += tmp;
|
||||
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
|
||||
const int it = it0 + it1 + threadIdx.x / neu_padded;
|
||||
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
|
||||
ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = true;
|
||||
constexpr bool write_to_store = true;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
}
|
||||
}
|
||||
int it0_dumped = it0;
|
||||
for (; it0 < n_tokens; it0 += n_unroll*warp_size/neu_padded) {
|
||||
#pragma unroll
|
||||
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
|
||||
const int it = it0 + it1 + threadIdx.x / neu_padded;
|
||||
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
|
||||
ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = true;
|
||||
constexpr bool write_to_store = false;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
}
|
||||
}
|
||||
|
||||
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
it_compact = 0;
|
||||
|
||||
for (it0 = it0_dumped; it0 < n_tokens; it0 += n_unroll*warp_size/neu_padded) {
|
||||
#pragma unroll
|
||||
for (int it1 = 0; it1 < n_unroll*warp_size/neu_padded; it1 += warp_size/neu_padded) {
|
||||
const int it = it0 + it1 + threadIdx.x / neu_padded;
|
||||
const int expert_found = (neu_padded == n_expert_used || iex < n_expert_used) && it < n_tokens ?
|
||||
ids[it*si1 + iex] : INT_MAX;
|
||||
|
||||
constexpr bool increment_nex_prev = false;
|
||||
constexpr bool write_to_store = true;
|
||||
check_experts<increment_nex_prev, write_to_store>
|
||||
(store, nex_prev, it_compact, expert, it, iex, expert_found);
|
||||
if (it_compact + warp_size >= ne_store) {
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
it_compact = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (iex_used != -1) {
|
||||
store[it_compact + it_compact_add_lower] = mm_ids_helper_store(it, iex_used);
|
||||
}
|
||||
|
||||
// The thread with the highest index in the warp always has the sum over the whole warp, use it to increment all threads:
|
||||
it_compact += ggml_cuda_shfl(it_compact_add_lower + it_compact_add_self, warp_size - 1, warp_size);
|
||||
}
|
||||
}
|
||||
nex_prev = warp_reduce_sum<warp_size>(nex_prev);
|
||||
ggml_cuda_syncwarp();
|
||||
|
||||
for (int itc = threadIdx.x; itc < it_compact; itc += warp_size) {
|
||||
const mm_ids_helper_store store_it = store[itc];
|
||||
const int it = store_it.it();
|
||||
const int iex_used = store_it.iex_used();
|
||||
ids_dst[nex_prev + itc] = it*n_expert_used + iex_used;
|
||||
// ids_src1 holds the forward map, or the inverse map (token slot -> compact row) for quant dedup
|
||||
if (write_inverse) {
|
||||
ids_src1[it*n_expert_used + iex_used] = nex_prev + itc;
|
||||
} else {
|
||||
ids_src1[nex_prev + itc] = it*sis1 + iex_used % nchannels_y;
|
||||
if (it_compact > 0) {
|
||||
ggml_cuda_syncwarp();
|
||||
dump_store(ids_src1, ids_dst, n_expert_used, nchannels_y, sis1, write_inverse, store, nex_prev, n_dumped, it_compact);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,7 +220,7 @@ static __global__ void mm_ids_helper(
|
||||
}
|
||||
|
||||
int col_low = nex_prev;
|
||||
int col_high = nex_prev + min(int(tile_width.z), it_compact);
|
||||
int col_high = nex_prev + min(int(tile_width.z), n_dumped);
|
||||
|
||||
// The while loop covers all tiles at the seams containing the column with % tile_width == 0.
|
||||
while (col_high > fastdiv(col_low + tile_width.z - 1, tile_width) * tile_width.z) {
|
||||
@@ -131,7 +228,7 @@ static __global__ void mm_ids_helper(
|
||||
= {expert, col_low, col_high, /*padding =*/ 0};
|
||||
|
||||
col_low = col_high;
|
||||
col_high = min(col_high + int(tile_width.z), nex_prev + it_compact);
|
||||
col_high = min(col_high + int(tile_width.z), nex_prev + n_dumped);
|
||||
}
|
||||
|
||||
// Finally, at most one extra tile_spec between the seams per expert.
|
||||
@@ -157,10 +254,12 @@ static void launch_mm_ids_helper(
|
||||
|
||||
const dim3 num_blocks(n_experts, 1, 1);
|
||||
const dim3 block_size(warp_size, 1, 1);
|
||||
const size_t nbytes_shared = n_tokens*sizeof(mm_ids_helper_store);
|
||||
GGML_ASSERT(nbytes_shared <= smpbo);
|
||||
const size_t ne_shared_pad = n_expert_used_template == 0 ? std::max(n_expert_used_var, warp_size) : n_unroll*warp_size;
|
||||
const size_t nbytes_store_core = std::min(smpbo - ne_shared_pad * sizeof(mm_ids_helper_store), n_tokens * sizeof(mm_ids_helper_store));
|
||||
const size_t nbytes_shared = nbytes_store_core + ne_shared_pad * sizeof(mm_ids_helper_store);
|
||||
const size_t ne_store = nbytes_shared / sizeof(mm_ids_helper_store);
|
||||
mm_ids_helper<n_expert_used_template><<<num_blocks, block_size, nbytes_shared, stream>>>
|
||||
(ids, ids_src1, ids_dst, tile_spec, n_tokens, n_expert_used_var, nchannels_y, tile_width_fd, si1, sis1, write_inverse);
|
||||
(ids, ids_src1, ids_dst, tile_spec, n_tokens, n_expert_used_var, nchannels_y, tile_width_fd, si1, sis1, write_inverse, ne_store);
|
||||
}
|
||||
|
||||
void ggml_cuda_launch_mm_ids_helper(
|
||||
|
||||
@@ -184,13 +184,6 @@ static __global__ void top_k_radix_gather(
|
||||
constexpr int NBINS = 1 << RADIX_BITS;
|
||||
constexpr int warp_size = ggml_cuda_get_physical_warp_size();
|
||||
constexpr int nwarps = BLOCK_SIZE / warp_size;
|
||||
const auto popc = [](const ggml_cuda_thread_mask_t mask) {
|
||||
if constexpr (warp_size == 64) {
|
||||
return __popcll(mask);
|
||||
} else {
|
||||
return __popc(mask);
|
||||
}
|
||||
};
|
||||
__shared__ int warp_ties[nwarps];
|
||||
|
||||
const int row = blockIdx.x / blocks_per_row;
|
||||
@@ -247,11 +240,11 @@ static __global__ void top_k_radix_gather(
|
||||
const bool is_tie = col < warp_end && top_k_float_to_ordered(row_src[col]) == threshold;
|
||||
|
||||
const ggml_cuda_thread_mask_t ties = ggml_cuda_ballot(is_tie);
|
||||
const int tie_pos = pos + popc(ties & (((ggml_cuda_thread_mask_t) 1 << lane) - 1));
|
||||
const int tie_pos = pos + ggml_cuda_popc(ties & (((ggml_cuda_thread_mask_t) 1 << lane) - 1));
|
||||
if (is_tie && tie_pos < pos_end) {
|
||||
row_dst[k - rank + tie_pos] = col;
|
||||
}
|
||||
pos += popc(ties);
|
||||
pos += ggml_cuda_popc(ties);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -323,6 +323,9 @@ set(GGML_OPENCL_KERNELS
|
||||
mul_mm_f16_f32_kq_kqv
|
||||
mul_mm_f16_f32_kq_p8
|
||||
mul_mm_q8_f32_kqv
|
||||
mul_mm_q8_kq_p8
|
||||
mul_mm_q8_kq_p8_dk512
|
||||
flash_attn_q8_rows
|
||||
conv2d
|
||||
conv2d_f16_f32
|
||||
conv2_dw
|
||||
|
||||
@@ -781,6 +781,10 @@ struct ggml_backend_opencl_context {
|
||||
ggml_cl_buffer prealloc_fa_pd;
|
||||
ggml_cl_buffer prealloc_fa_bmax;
|
||||
ggml_cl_buffer prealloc_fa_bsum;
|
||||
ggml_cl_buffer prealloc_fa_k8;
|
||||
ggml_cl_buffer prealloc_fa_k8_scale;
|
||||
ggml_cl_buffer prealloc_fa_q8;
|
||||
ggml_cl_buffer prealloc_fa_q8_scale;
|
||||
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
ggml_cl_buffer prealloc_adreno_xmem_const;
|
||||
@@ -899,6 +903,10 @@ struct ggml_backend_opencl_context {
|
||||
cl_kernel kernel_fa_kqv_direct_f16 = nullptr;
|
||||
cl_kernel kernel_fa_kqv_direct_reduce = nullptr;
|
||||
cl_kernel kernel_fa_v_transpose_q8 = nullptr;
|
||||
cl_kernel kernel_fa_q8_rows_f16 = nullptr;
|
||||
cl_kernel kernel_fa_q8_rows_f32 = nullptr;
|
||||
cl_kernel kernel_mul_mm_q8_kq_p8 = nullptr;
|
||||
cl_kernel kernel_mul_mm_q8_kq_p8_dk512 = nullptr;
|
||||
ggml_opencl_fa_kernels fa;
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
ggml_cl_adreno_xmem_attn_state adreno_xmem_attn;
|
||||
@@ -1506,7 +1514,18 @@ struct ggml_backend_opencl_context {
|
||||
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
|
||||
~ggml_backend_opencl_context() {
|
||||
for (cl_kernel kernel : {kernel_mul_mm_f16_f32_kq_p8, kernel_mul_mm_q8_f32_kqv, kernel_fa_p8_fixup, kernel_fa_v_transpose_q8, kernel_fa_kqv_direct_f16, kernel_fa_kqv_direct_reduce}) {
|
||||
for (cl_kernel kernel : {
|
||||
kernel_mul_mm_f16_f32_kq_p8,
|
||||
kernel_mul_mm_q8_f32_kqv,
|
||||
kernel_fa_p8_fixup,
|
||||
kernel_fa_v_transpose_q8,
|
||||
kernel_fa_kqv_direct_f16,
|
||||
kernel_fa_kqv_direct_reduce,
|
||||
kernel_fa_q8_rows_f16,
|
||||
kernel_fa_q8_rows_f32,
|
||||
kernel_mul_mm_q8_kq_p8,
|
||||
kernel_mul_mm_q8_kq_p8_dk512
|
||||
}) {
|
||||
if (kernel != nullptr) {
|
||||
CL_CHECK(clReleaseKernel(kernel));
|
||||
}
|
||||
@@ -1648,7 +1667,7 @@ static cl_program build_program_from_source_ex(cl_context ctx, cl_device_id dev,
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts) {
|
||||
static cl_program build_program_from_source(ggml_backend_opencl_context * backend_ctx, const char* program_buffer, const std::string &compile_opts, bool fatal = true) {
|
||||
cl_context ctx = backend_ctx->context;
|
||||
cl_device_id dev = backend_ctx->device;
|
||||
|
||||
@@ -1660,7 +1679,7 @@ static cl_program build_program_from_source(ggml_backend_opencl_context * backen
|
||||
return p_cached;
|
||||
}
|
||||
|
||||
cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, /*fatal=*/true);
|
||||
cl_program p = build_program_from_source_ex(ctx, dev, program_buffer, compile_opts, fatal);
|
||||
|
||||
// Best-effort save of the freshly-built binary (no-op if cache disabled).
|
||||
if (p != nullptr) {
|
||||
@@ -3995,6 +4014,52 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
|
||||
}
|
||||
|
||||
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
|
||||
#ifdef GGML_OPENCL_EMBED_KERNELS
|
||||
const std::string rows_src {
|
||||
#include "flash_attn_q8_rows.cl.h"
|
||||
};
|
||||
const std::string kq_src {
|
||||
#include "mul_mm_q8_kq_p8.cl.h"
|
||||
};
|
||||
#else
|
||||
const std::string rows_src = read_file("flash_attn_q8_rows.cl");
|
||||
const std::string kq_src = read_file("mul_mm_q8_kq_p8.cl");
|
||||
#endif
|
||||
cl_program prog = build_program_from_source(backend_ctx, rows_src.c_str(), compile_opts, /*fatal=*/false);
|
||||
if (prog != nullptr) {
|
||||
CL_CHECK((backend_ctx->kernel_fa_q8_rows_f16 = clCreateKernel(prog, "kernel_fa_q8_rows_f16", &err), err));
|
||||
CL_CHECK((backend_ctx->kernel_fa_q8_rows_f32 = clCreateKernel(prog, "kernel_fa_q8_rows_f32", &err), err));
|
||||
CL_CHECK(clReleaseProgram(prog));
|
||||
}
|
||||
prog = build_program_from_source(backend_ctx, kq_src.c_str(), compile_opts, /*fatal=*/false);
|
||||
if (prog != nullptr) {
|
||||
CL_CHECK((backend_ctx->kernel_mul_mm_q8_kq_p8 = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8", &err), err));
|
||||
CL_CHECK(clReleaseProgram(prog));
|
||||
}
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
// mul_mm_q8_kq_p8_dk512
|
||||
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
|
||||
#ifdef GGML_OPENCL_EMBED_KERNELS
|
||||
const std::string kernel_src {
|
||||
#include "mul_mm_q8_kq_p8_dk512.cl.h"
|
||||
};
|
||||
#else
|
||||
const std::string kernel_src = read_file("mul_mm_q8_kq_p8_dk512.cl");
|
||||
#endif
|
||||
cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts, /*fatal=*/false);
|
||||
if (prog != nullptr) {
|
||||
cl_kernel kernel = clCreateKernel(prog, "kernel_mul_mm_q8_kq_p8_dk512", &err);
|
||||
CL_CHECK(clReleaseProgram(prog));
|
||||
if (err == CL_SUCCESS) {
|
||||
backend_ctx->kernel_mul_mm_q8_kq_p8_dk512 = kernel;
|
||||
}
|
||||
}
|
||||
GGML_LOG_CONT(".");
|
||||
}
|
||||
|
||||
// mul_mm_f16_f32_kq_p8
|
||||
if (backend_ctx->gpu_family == ADRENO && backend_ctx->has_integer_dot) {
|
||||
#ifdef GGML_OPENCL_EMBED_KERNELS
|
||||
@@ -22922,12 +22987,58 @@ static void ggml_cl_flash_attn_prefill_bin(ggml_backend_t backend, const ggml_te
|
||||
}
|
||||
#endif // GGML_OPENCL_USE_ADRENO_KERNELS
|
||||
|
||||
static void ggml_cl_fa_q8_rows(
|
||||
ggml_backend_t backend, const ggml_tensor * src, const ggml_tensor * dst,
|
||||
cl_mem values, cl_mem scales, int64_t row0, int nrow
|
||||
) {
|
||||
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
|
||||
auto * extra = (ggml_tensor_extra_cl *) (src->view_src ? src->view_src->extra : src->extra);
|
||||
const cl_ulong offset = extra->offset + src->view_offs + row0 * src->nb[1];
|
||||
const cl_ulong nb1 = src->nb[1];
|
||||
const cl_ulong nb2 = src->nb[2];
|
||||
const int dk = src->ne[0];
|
||||
const int nhead = src->ne[2];
|
||||
cl_kernel kernel;
|
||||
switch (src->type) {
|
||||
case GGML_TYPE_F16: kernel = backend_ctx->kernel_fa_q8_rows_f16; break;
|
||||
case GGML_TYPE_F32: kernel = backend_ctx->kernel_fa_q8_rows_f32; break;
|
||||
default: GGML_ABORT("unsupported int8 KQ input type");
|
||||
}
|
||||
cl_uint a = 0;
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &offset));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &nb2));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &values));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &scales));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &dk));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nrow));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &nhead));
|
||||
const size_t blocks = (size_t) (dk / 32) * nrow * nhead;
|
||||
size_t gws[3] = {(blocks + 63) / 64 * 64, 1, 1};
|
||||
size_t lws[3] = {64, 1, 1};
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
}
|
||||
|
||||
static void ggml_cl_fa_kq_int8_prepare(ggml_backend_t backend, const ggml_tensor * dst, int64_t n_q_chunk) {
|
||||
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
|
||||
const ggml_tensor * q = dst->src[0];
|
||||
const ggml_tensor * k = dst->src[1];
|
||||
const size_t k_bytes = (size_t) k->ne[0] * k->ne[1] * k->ne[2];
|
||||
const size_t q_bytes = (size_t) q->ne[0] * n_q_chunk * q->ne[2];
|
||||
backend_ctx->prealloc_fa_k8.allocate(backend_ctx->context, k_bytes);
|
||||
backend_ctx->prealloc_fa_k8_scale.allocate(backend_ctx->context, k_bytes / 32 * sizeof(cl_half));
|
||||
backend_ctx->prealloc_fa_q8.allocate(backend_ctx->context, q_bytes);
|
||||
backend_ctx->prealloc_fa_q8_scale.allocate(backend_ctx->context, q_bytes / 32 * sizeof(cl_half));
|
||||
ggml_cl_fa_q8_rows(backend, k, dst, backend_ctx->prealloc_fa_k8.buffer, backend_ctx->prealloc_fa_k8_scale.buffer, 0, k->ne[1]);
|
||||
}
|
||||
|
||||
static void ggml_cl_fa_kq_p8_dispatch(
|
||||
ggml_backend_t backend,
|
||||
const ggml_tensor * q, const ggml_tensor * k,
|
||||
const ggml_tensor * mask, const ggml_tensor * sinks,
|
||||
const ggml_tensor * dst,
|
||||
int64_t q0, int nqc, float scale, float max_bias, int p_pitch
|
||||
int64_t q0, int nqc, float scale, float max_bias, int p_pitch, bool kq_int8
|
||||
) {
|
||||
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
|
||||
auto * extra_k = (ggml_tensor_extra_cl *) (k->view_src ? k->view_src->extra : k->extra);
|
||||
@@ -22940,64 +23051,94 @@ static void ggml_cl_fa_kq_p8_dispatch(
|
||||
const int D_A = k->ne[2];
|
||||
const int D_B = q->ne[2];
|
||||
|
||||
cl_int err;
|
||||
|
||||
cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
|
||||
cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
cl_image_format fmt = {CL_RGBA, CL_FLOAT};
|
||||
cl_image_desc desc = {};
|
||||
desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
|
||||
desc.image_width = region.size / 16;
|
||||
desc.buffer = k_sub;
|
||||
cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
|
||||
region.size = (size_t) K * N * D_B * sizeof(float);
|
||||
cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
const cl_ulong off_mask = extra_m->offset + mask->view_offs + q0 * mask->nb[1];
|
||||
const cl_ulong mask_nb1 = mask->nb[1];
|
||||
const int n_head_log2 = 1u << (unsigned) floorf(log2f((float) D_B));
|
||||
const float m0 = powf(2.0f, -max_bias / n_head_log2);
|
||||
const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
|
||||
|
||||
cl_kernel kernel;
|
||||
cl_uint a = 0;
|
||||
|
||||
cl_kernel kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
|
||||
if (kq_int8) {
|
||||
ggml_cl_fa_q8_rows(backend, q, dst, backend_ctx->prealloc_fa_q8.buffer, backend_ctx->prealloc_fa_q8_scale.buffer, q0, nqc);
|
||||
kernel = K <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_k8_scale.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_q8_scale.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
|
||||
size_t gws[3] = {64 * (((size_t) N + 31) / 32) * (D_B / D_A), ((size_t) M + 511) / 512, (size_t) D_A};
|
||||
size_t lws[3] = {64, 1, 1};
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
} else {
|
||||
cl_int err;
|
||||
|
||||
const size_t n_tiles = (N + 31) / 32;
|
||||
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
|
||||
cl_buffer_region region = {extra_k->offset + k->view_offs, k->nb[1] * k->ne[1]};
|
||||
cl_mem k_sub = clCreateSubBuffer(extra_k->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
|
||||
size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
|
||||
cl_image_format fmt = {CL_RGBA, CL_FLOAT};
|
||||
cl_image_desc desc = {};
|
||||
desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER;
|
||||
desc.image_width = region.size / 16;
|
||||
desc.buffer = k_sub;
|
||||
cl_mem k_img = clCreateImage(backend_ctx->context, CL_MEM_READ_ONLY, &fmt, &desc, nullptr, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
CL_CHECK(clReleaseMemObject(k_img));
|
||||
CL_CHECK(clReleaseMemObject(k_sub));
|
||||
CL_CHECK(clReleaseMemObject(q_sub));
|
||||
region.origin = extra_q->offset + q->view_offs + q0 * q->nb[1];
|
||||
region.size = (size_t) K * N * D_B * sizeof(float);
|
||||
cl_mem q_sub = clCreateSubBuffer(extra_q->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err);
|
||||
CL_CHECK(err);
|
||||
|
||||
kernel = backend_ctx->kernel_mul_mm_f16_f32_kq_p8;
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &k_img));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &q_sub));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_pq.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bmax.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &backend_ctx->prealloc_fa_bsum.buffer));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_mem), &extra_m->data_device));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &off_mask));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(cl_ulong), &mask_nb1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &scale));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &max_bias));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m0));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(float), &m1));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &n_head_log2));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &M));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &K));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &N));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_A));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &D_B));
|
||||
CL_CHECK(clSetKernelArg(kernel, a++, sizeof(int), &p_pitch));
|
||||
|
||||
const size_t n_tiles = (N + 31) / 32;
|
||||
size_t gws[3] = {64, n_tiles, (size_t)(M / 64) * D_B};
|
||||
|
||||
const bool two_tiles = n_tiles % 2 == 0 && backend_ctx->get_kernel_workgroup_size(kernel) >= 128;
|
||||
size_t lws[3] = {64, two_tiles ? (size_t) 2 : (size_t) 1, 1};
|
||||
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
CL_CHECK(clReleaseMemObject(k_img));
|
||||
CL_CHECK(clReleaseMemObject(k_sub));
|
||||
CL_CHECK(clReleaseMemObject(q_sub));
|
||||
}
|
||||
|
||||
cl_mem sinks_buf = extra_q->data_device;
|
||||
cl_ulong sinks_off = 0;
|
||||
@@ -23261,7 +23402,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
|
||||
ggml_backend_t backend,
|
||||
const ggml_tensor * q, const ggml_tensor * k,
|
||||
ggml_tensor * dst,
|
||||
int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct
|
||||
int64_t n_q_chunk, float scale, float max_bias, bool kqv_direct, bool kq_int8
|
||||
) {
|
||||
auto * backend_ctx = (ggml_backend_opencl_context *) backend->context;
|
||||
const ggml_tensor * v = dst->src[2];
|
||||
@@ -23345,13 +23486,17 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
|
||||
backend_ctx->enqueue_ndrange_kernel(kernel, 3, gws, lws, dst);
|
||||
}
|
||||
|
||||
if (kq_int8) {
|
||||
ggml_cl_fa_kq_int8_prepare(backend, dst, n_q_chunk);
|
||||
}
|
||||
|
||||
ggml_tensor_extra_cl extra_kqv;
|
||||
ggml_tensor kqv;
|
||||
|
||||
for (int64_t q0 = 0; q0 < n_q; q0 += n_q_chunk) {
|
||||
const int64_t nqc = MIN(n_q_chunk, n_q - q0);
|
||||
|
||||
ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch);
|
||||
ggml_cl_fa_kq_p8_dispatch(backend, q, k, mask, sinks, dst, q0, nqc, scale, max_bias, kv_pitch, kq_int8);
|
||||
|
||||
ggml_cl_fa_decompose_scratch_tensor(kqv, extra_kqv, backend_ctx->prealloc_fa_kqv.buffer, GGML_TYPE_F32, dv, nqc, n_head);
|
||||
|
||||
@@ -23408,6 +23553,7 @@ static void ggml_cl_flash_attn_decompose_kq_p8(
|
||||
// head size. GGML_OPENCL_FA_DECOMPOSE_MB sets the KQ scratch budget (default 256).
|
||||
// GGML_OPENCL_FA_KQ_P8=0 disables U8 probabilities and int8 KQV.
|
||||
// GGML_OPENCL_FA_KQV_DIRECT=0 disables direct PV for small query batches.
|
||||
// GGML_OPENCL_FA_KQ_INT8=0 keeps F16 KQ; =1 enables int8 KQ for supported shapes.
|
||||
static bool ggml_cl_flash_attn_decompose_dispatch(
|
||||
ggml_backend_t backend,
|
||||
const ggml_tensor * q, const ggml_tensor * k, ggml_tensor * dst
|
||||
@@ -23537,12 +23683,46 @@ static bool ggml_cl_flash_attn_decompose_dispatch(
|
||||
return false;
|
||||
}
|
||||
|
||||
static const int kq_int8_env = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KQ_INT8");
|
||||
return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
|
||||
}();
|
||||
static const int dk512_env = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KQ_DK512_F16");
|
||||
return (e && e[0]) ? (e[0] != '0' ? 1 : 0) : -1;
|
||||
}();
|
||||
const bool dk512 = dk > 256 && dk <= 512 && n_q >= 32 &&
|
||||
(dk512_env == 1 || (dk512_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E));
|
||||
const bool kq_int8_possible = kq_p8 && (dk <= 256 || dk512) && dk % 32 == 0 &&
|
||||
backend_ctx->kernel_fa_q8_rows_f16 != nullptr && backend_ctx->kernel_fa_q8_rows_f32 != nullptr &&
|
||||
n_head <= INT_MAX && n_head_kv <= INT_MAX &&
|
||||
n_q <= INT_MAX / n_head / (dk / 32) && n_kv <= INT_MAX / n_head_kv / (dk / 32) &&
|
||||
(dst->src[4] == nullptr || dk > 64) &&
|
||||
(kq_int8_env == 1 || (kq_int8_env < 0 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E && n_kv >= 2048)) &&
|
||||
backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f16) >= 64 &&
|
||||
backend_ctx->get_kernel_workgroup_size(backend_ctx->kernel_fa_q8_rows_f32) >= 64;
|
||||
cl_kernel kq_int8_kernel = nullptr;
|
||||
if (kq_int8_possible) {
|
||||
kq_int8_kernel = dk <= 256 ? backend_ctx->kernel_mul_mm_q8_kq_p8 : backend_ctx->kernel_mul_mm_q8_kq_p8_dk512;
|
||||
}
|
||||
const bool kq_int8 = kq_int8_kernel != nullptr && backend_ctx->get_kernel_workgroup_size(kq_int8_kernel) >= 64;
|
||||
|
||||
// Fewer query tiles per KV block reduce cache pressure in the int8 KQ kernel.
|
||||
static const int64_t kq_int8_max_chunk = [] {
|
||||
const char * e = getenv("GGML_OPENCL_FA_KQ_INT8_MAX_CHUNK");
|
||||
const int v = (e && e[0]) ? atoi(e) : 96;
|
||||
return (int64_t) (v >= 32 ? (v / 32) * 32 : 96);
|
||||
}();
|
||||
if (kq_int8) {
|
||||
n_q_chunk = MIN(n_q_chunk, kq_int8_max_chunk);
|
||||
}
|
||||
|
||||
// even out the chunks, then align to the 32-wide n-tile
|
||||
const int64_t n_chunks = (n_q + n_q_chunk - 1) / n_q_chunk;
|
||||
n_q_chunk = MIN(n_q_chunk, ((n_q + n_chunks - 1) / n_chunks + 31) & ~(int64_t) 31);
|
||||
|
||||
if (kq_p8) {
|
||||
ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct);
|
||||
ggml_cl_flash_attn_decompose_kq_p8(backend, q, k, dst, n_q_chunk, scale, max_bias, kqv_direct, kq_int8);
|
||||
} else {
|
||||
ggml_cl_flash_attn_decompose(backend, q, k, dst, n_q_chunk, scale, max_bias);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
|
||||
// Store 32-element blocks as separate int8 values and half scales.
|
||||
|
||||
kernel void kernel_fa_q8_rows_f16(
|
||||
global const char * src, ulong off_src, ulong nb1, ulong nb2,
|
||||
global char * dq, global half * dd,
|
||||
int ne0, int nrow, int nhead
|
||||
) {
|
||||
const int nblk = ne0 / 32;
|
||||
const int gid = get_global_id(0);
|
||||
if (gid >= nblk*nrow*nhead) {
|
||||
return;
|
||||
}
|
||||
const int b = gid % nblk;
|
||||
const int r = (gid / nblk) % nrow;
|
||||
const int h = gid / (nblk*nrow);
|
||||
|
||||
global const half * p = (global const half *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
|
||||
float v[32];
|
||||
float amax = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
const float4 x = convert_float4(vload4(i, p));
|
||||
v[i*4+0] = x.s0;
|
||||
v[i*4+1] = x.s1;
|
||||
v[i*4+2] = x.s2;
|
||||
v[i*4+3] = x.s3;
|
||||
amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
|
||||
}
|
||||
const float d = amax / 127.0f;
|
||||
const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
|
||||
const size_t row = (size_t)h*nrow + r;
|
||||
dd[row*nblk + b] = (half)d;
|
||||
global char * q = dq + row*ne0 + b*32;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
|
||||
}
|
||||
}
|
||||
|
||||
kernel void kernel_fa_q8_rows_f32(
|
||||
global const char * src, ulong off_src, ulong nb1, ulong nb2,
|
||||
global char * dq, global half * dd,
|
||||
int ne0, int nrow, int nhead
|
||||
) {
|
||||
const int nblk = ne0 / 32;
|
||||
const int gid = get_global_id(0);
|
||||
if (gid >= nblk*nrow*nhead) {
|
||||
return;
|
||||
}
|
||||
const int b = gid % nblk;
|
||||
const int r = (gid / nblk) % nrow;
|
||||
const int h = gid / (nblk*nrow);
|
||||
|
||||
global const float * p = (global const float *)(src + off_src + (ulong)h*nb2 + (ulong)r*nb1) + b*32;
|
||||
float v[32];
|
||||
float amax = 0.0f;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
const float4 x = vload4(i, p);
|
||||
v[i*4+0] = x.s0;
|
||||
v[i*4+1] = x.s1;
|
||||
v[i*4+2] = x.s2;
|
||||
v[i*4+3] = x.s3;
|
||||
amax = fmax(amax, fmax(fmax(fabs(x.s0), fabs(x.s1)), fmax(fabs(x.s2), fabs(x.s3))));
|
||||
}
|
||||
const float d = amax / 127.0f;
|
||||
const float id = amax > 0.0f ? 127.0f / amax : 0.0f;
|
||||
const size_t row = (size_t)h*nrow + r;
|
||||
dd[row*nblk + b] = (half)d;
|
||||
global char * q = dq + row*ne0 + b*32;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
q[i] = (char)clamp((int)rint(v[i]*id), -127, 127);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
|
||||
#ifdef cl_khr_integer_dot_product
|
||||
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
|
||||
#endif
|
||||
|
||||
#define KQ_TN 32
|
||||
#define KQ_MB 8
|
||||
#define KQ_TM 64
|
||||
#define KQ_WG 64
|
||||
#define KQ_DK_MAX 256
|
||||
|
||||
__attribute__((qcom_wave_pair_mode(1)))
|
||||
kernel void kernel_mul_mm_q8_kq_p8(
|
||||
global const uint * kq, // K int8, [head_kv][n_kv][dk] as packed uints
|
||||
global const half * kd, // K scales, [head_kv][n_kv][dk/32]
|
||||
global const uint * qq, // Q int8, [head][n_q][dk] as packed uints
|
||||
global const half * qd, // Q scales, [head][n_q][dk/32]
|
||||
global uchar * qp, // u8 P, [head][n_q][p_pitch]
|
||||
global float * bmax, // per 32-row block max, [head][n_q][p_pitch/32]
|
||||
global float * bsum, // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
|
||||
global const char * mask, // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
|
||||
ulong offset_mask,
|
||||
ulong mask_nb1,
|
||||
float scale,
|
||||
float max_bias,
|
||||
float m0,
|
||||
float m1,
|
||||
int n_head_log2,
|
||||
int dk,
|
||||
int n_kv,
|
||||
int n_q,
|
||||
int n_head,
|
||||
int n_head_kv,
|
||||
int p_pitch // bytes per P row, >= n_kv, multiple of 32
|
||||
) {
|
||||
const int lid = get_local_id(0);
|
||||
const int gsz = n_head / n_head_kv;
|
||||
const int g0 = get_group_id(0);
|
||||
const int head_kv = get_group_id(2);
|
||||
const int head = head_kv*gsz + (g0 % gsz);
|
||||
const int qn0 = (g0 / gsz) * KQ_TN;
|
||||
const int kv0 = get_group_id(1)*KQ_TM*KQ_MB + lid;
|
||||
|
||||
const int nblk = dk / 32;
|
||||
const int nu = dk / 4;
|
||||
|
||||
__local uint sh_q [KQ_TN][KQ_DK_MAX/4];
|
||||
__local half sh_qd[KQ_TN][KQ_DK_MAX/32];
|
||||
__local float p8_lds[1024];
|
||||
|
||||
const size_t qbase = (size_t)head*n_q;
|
||||
for (int i = lid; i < KQ_TN*(KQ_DK_MAX/16); i += KQ_WG) {
|
||||
const int t = i / (KQ_DK_MAX/16);
|
||||
const int u = (i % (KQ_DK_MAX/16)) * 4;
|
||||
const int qi = qn0 + t;
|
||||
uint4 v = (uint4)(0u);
|
||||
if (qi < n_q && u < nu) {
|
||||
v = vload4(0, &qq[(qbase + qi)*nu + u]);
|
||||
}
|
||||
vstore4(v, 0, &sh_q[t][u]);
|
||||
}
|
||||
for (int i = lid; i < KQ_TN*(KQ_DK_MAX/32); i += KQ_WG) {
|
||||
const int t = i / (KQ_DK_MAX/32);
|
||||
const int b = i % (KQ_DK_MAX/32);
|
||||
const int qi = qn0 + t;
|
||||
sh_qd[t][b] = (qi < n_q && b < nblk) ? qd[(qbase + qi)*nblk + b] : (half)0.0f;
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
|
||||
float slope = 1.0f;
|
||||
if (max_bias > 0.0f) {
|
||||
const float base = head < n_head_log2 ? m0 : m1;
|
||||
const int ex = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
|
||||
slope = pow(base, ex);
|
||||
}
|
||||
global const half * mrow = (global const half *)(mask + offset_mask);
|
||||
const uint mstride = (uint)(mask_nb1 / 2);
|
||||
const int pblk = p_pitch / 32;
|
||||
|
||||
for (int m = 0; m < KQ_MB; ++m) {
|
||||
const int kv = kv0 + m*KQ_TM;
|
||||
if (kv - lid >= n_kv) {
|
||||
break; // uniform across the workgroup
|
||||
}
|
||||
|
||||
float acc[KQ_TN];
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
acc[t] = 0.0f;
|
||||
}
|
||||
|
||||
const int kl = min(kv, n_kv - 1);
|
||||
const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
|
||||
const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
|
||||
|
||||
for (int b = 0; b < nblk; ++b) {
|
||||
const uint4 w0 = vload4(0, &kq[kbase + b*8]);
|
||||
const uint4 w1 = vload4(0, &kq[kbase + b*8 + 4]);
|
||||
const float dks = (float)kd[kdbas + b];
|
||||
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
const uint4 a0 = vload4(0, &sh_q[t][b*8]);
|
||||
const uint4 a1 = vload4(0, &sh_q[t][b*8 + 4]);
|
||||
int raw = 0;
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
|
||||
acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
|
||||
}
|
||||
}
|
||||
|
||||
// Each lane owns one complete 32-row softmax block after the transpose.
|
||||
const int kvblk0 = kv - lid;
|
||||
#pragma unroll
|
||||
for (int half_ = 0; half_ < 2; ++half_) {
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
#pragma unroll
|
||||
for (int j = 0; j < 16; ++j) {
|
||||
p8_lds[j*64 + lid] = acc[half_*16 + j];
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
if (lid < 32) {
|
||||
const int j = lid >> 1;
|
||||
const int b = lid & 1;
|
||||
const int n = qn0 + half_*16 + j;
|
||||
if (n < n_q) {
|
||||
const int kvb = kvblk0 + b*32;
|
||||
const __local float * src = p8_lds + j*64 + b*32;
|
||||
float16 s0 = vload16(0, src);
|
||||
float16 s1 = vload16(1, src);
|
||||
const global half * mp = mrow + (size_t)n*mstride + kvb;
|
||||
const float16 m0v = convert_float16(vload16(0, mp));
|
||||
const float16 m1v = convert_float16(vload16(1, mp));
|
||||
s0 = s0*scale + slope*m0v;
|
||||
s1 = s1*scale + slope*m1v;
|
||||
const float16 mx16 = fmax(s0, s1);
|
||||
const float8 mx8 = fmax(mx16.lo, mx16.hi);
|
||||
const float4 mx4 = fmax(mx8.lo, mx8.hi);
|
||||
const float2 mx2 = fmax(mx4.lo, mx4.hi);
|
||||
const float amax = fmax(mx2.x, mx2.y);
|
||||
const size_t prow = qbase + n;
|
||||
const int blk = kvb >> 5;
|
||||
float sum = 0.0f;
|
||||
uchar16 q0 = (uchar16)(0);
|
||||
uchar16 q1 = (uchar16)(0);
|
||||
// finite sentinel: the program builds with -cl-finite-math-only
|
||||
if (amax > -1.0e30f) {
|
||||
const float16 e0 = exp(s0 - amax);
|
||||
const float16 e1 = exp(s1 - amax);
|
||||
const float16 a16 = e0 + e1;
|
||||
const float8 a8 = a16.lo + a16.hi;
|
||||
const float4 a4 = a8.lo + a8.hi;
|
||||
const float2 a2 = a4.lo + a4.hi;
|
||||
sum = a2.x + a2.y;
|
||||
q0 = convert_uchar16_sat_rte(e0*255.0f);
|
||||
q1 = convert_uchar16_sat_rte(e1*255.0f);
|
||||
}
|
||||
global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
|
||||
vstore16(q0, 0, qdst);
|
||||
vstore16(q1, 1, qdst);
|
||||
bmax[prow*pblk + blk] = amax;
|
||||
bsum[prow*pblk + blk] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
|
||||
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
|
||||
#ifdef cl_khr_integer_dot_product
|
||||
#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable
|
||||
#endif
|
||||
|
||||
#define KQ_TN 32
|
||||
#define KQ_MB 8
|
||||
#define KQ_TM 64
|
||||
#define KQ_WG 64
|
||||
#define KQ_DKC 256
|
||||
|
||||
__attribute__((qcom_wave_pair_mode(1)))
|
||||
kernel void kernel_mul_mm_q8_kq_p8_dk512(
|
||||
global const uint * kq, // K int8, [head_kv][n_kv][dk] as packed uints
|
||||
global const half * kd, // K scales, [head_kv][n_kv][dk/32]
|
||||
global const uint * qq, // Q int8, [head][n_q][dk] as packed uints
|
||||
global const half * qd, // Q scales, [head][n_q][dk/32]
|
||||
global uchar * qp, // u8 P, [head][n_q][p_pitch]
|
||||
global float * bmax, // per 32-row block max, [head][n_q][p_pitch/32]
|
||||
global float * bsum, // per block sum of exp(s - bmax), [head][n_q][p_pitch/32]
|
||||
global const char * mask, // f16 [>=n_kv][n_q] view, row stride mask_nb1 bytes
|
||||
ulong offset_mask,
|
||||
ulong mask_nb1,
|
||||
float scale,
|
||||
float max_bias,
|
||||
float m0,
|
||||
float m1,
|
||||
int n_head_log2,
|
||||
int dk,
|
||||
int n_kv,
|
||||
int n_q,
|
||||
int n_head,
|
||||
int n_head_kv,
|
||||
int p_pitch // bytes per P row, >= n_kv, multiple of 32
|
||||
) {
|
||||
const int lid = get_local_id(0);
|
||||
const int gsz = n_head / n_head_kv;
|
||||
const int g0 = get_group_id(0);
|
||||
const int head_kv = get_group_id(2);
|
||||
const int head = head_kv*gsz + (g0 % gsz);
|
||||
const int qn0 = (g0 / gsz) * KQ_TN;
|
||||
const int kv0 = get_group_id(1)*KQ_TM*KQ_MB + lid;
|
||||
|
||||
const int nblk = dk / 32;
|
||||
const int nu = dk / 4;
|
||||
|
||||
// Reuse the 256-wide Q slice for the F32 softmax scores after the contraction.
|
||||
__local uint sh_q [KQ_TN*KQ_DKC/4];
|
||||
__local half sh_qd[KQ_TN][KQ_DKC/32];
|
||||
__local float * p8_lds = (__local float *)sh_q;
|
||||
|
||||
const size_t qbase = (size_t)head*n_q;
|
||||
|
||||
float slope = 1.0f;
|
||||
if (max_bias > 0.0f) {
|
||||
const float base = head < n_head_log2 ? m0 : m1;
|
||||
const int ex = head < n_head_log2 ? head + 1 : 2*(head - n_head_log2) + 1;
|
||||
slope = pow(base, ex);
|
||||
}
|
||||
global const half * mrow = (global const half *)(mask + offset_mask);
|
||||
const uint mstride = (uint)(mask_nb1 / 2);
|
||||
const int pblk = p_pitch / 32;
|
||||
|
||||
for (int m = 0; m < KQ_MB; ++m) {
|
||||
const int kv = kv0 + m*KQ_TM;
|
||||
if (kv - lid >= n_kv) {
|
||||
break; // uniform across the workgroup
|
||||
}
|
||||
|
||||
float acc[KQ_TN];
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
acc[t] = 0.0f;
|
||||
}
|
||||
|
||||
const int kl = min(kv, n_kv - 1);
|
||||
const size_t kbase = ((size_t)head_kv*n_kv + kl)*nu;
|
||||
const size_t kdbas = ((size_t)head_kv*n_kv + kl)*nblk;
|
||||
|
||||
for (int c0 = 0; c0 < nblk; c0 += KQ_DKC/32) {
|
||||
const int cb = min(KQ_DKC/32, nblk - c0);
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
for (int i = lid; i < KQ_TN*(KQ_DKC/16); i += KQ_WG) {
|
||||
const int t = i / (KQ_DKC/16);
|
||||
const int u = (i % (KQ_DKC/16)) * 4;
|
||||
const int qi = qn0 + t;
|
||||
uint4 v = (uint4)(0u);
|
||||
if (qi < n_q && u < cb*8) {
|
||||
v = vload4(0, &qq[(qbase + qi)*nu + c0*8 + u]);
|
||||
}
|
||||
vstore4(v, 0, &sh_q[t*(KQ_DKC/4) + u]);
|
||||
}
|
||||
for (int i = lid; i < KQ_TN*(KQ_DKC/32); i += KQ_WG) {
|
||||
const int t = i / (KQ_DKC/32);
|
||||
const int b = i % (KQ_DKC/32);
|
||||
const int qi = qn0 + t;
|
||||
sh_qd[t][b] = (qi < n_q && b < cb) ? qd[(qbase + qi)*nblk + c0 + b] : (half)0.0f;
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
|
||||
for (int b = 0; b < cb; ++b) {
|
||||
const uint4 w0 = vload4(0, &kq[kbase + (c0 + b)*8]);
|
||||
const uint4 w1 = vload4(0, &kq[kbase + (c0 + b)*8 + 4]);
|
||||
const float dks = (float)kd[kdbas + c0 + b];
|
||||
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
const uint4 a0 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8]);
|
||||
const uint4 a1 = vload4(0, &sh_q[t*(KQ_DKC/4) + b*8 + 4]);
|
||||
int raw = 0;
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s0, a0.s0, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s1, a0.s1, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s2, a0.s2, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w0.s3, a0.s3, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s0, a1.s0, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s1, a1.s1, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s2, a1.s2, raw);
|
||||
raw = dot_acc_sat_4x8packed_ss_int(w1.s3, a1.s3, raw);
|
||||
acc[t] += dks * (float)sh_qd[t][b] * (float)raw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Finish reading the Q slice before writing scores to the same local memory.
|
||||
const int kvblk0 = kv - lid;
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
#pragma unroll
|
||||
for (int t = 0; t < KQ_TN; ++t) {
|
||||
p8_lds[t*64 + lid] = acc[t];
|
||||
}
|
||||
barrier(CLK_LOCAL_MEM_FENCE);
|
||||
{
|
||||
const int j = lid >> 1;
|
||||
const int b = lid & 1;
|
||||
const int n = qn0 + j;
|
||||
if (n < n_q) {
|
||||
const int kvb = kvblk0 + b*32;
|
||||
__local float * src = p8_lds + j*64 + b*32;
|
||||
const global half * mp = mrow + (size_t)n*mstride + kvb;
|
||||
float4 s = vload4(0, src)*scale + slope*convert_float4(vload4(0, mp));
|
||||
vstore4(s, 0, src);
|
||||
float amax = fmax(fmax(s.x, s.y), fmax(s.z, s.w));
|
||||
for (int c = 1; c < 8; ++c) {
|
||||
s = vload4(c, src)*scale + slope*convert_float4(vload4(c, mp));
|
||||
vstore4(s, c, src);
|
||||
amax = fmax(amax, fmax(fmax(s.x, s.y), fmax(s.z, s.w)));
|
||||
}
|
||||
const size_t prow = qbase + n;
|
||||
const int blk = kvb >> 5;
|
||||
global uchar * qdst = qp + prow*(size_t)p_pitch + (size_t)kvb;
|
||||
float sum = 0.0f;
|
||||
// finite sentinel: the program builds with -cl-finite-math-only
|
||||
if (amax > -1.0e30f) {
|
||||
for (int c = 0; c < 8; ++c) {
|
||||
const float4 e = exp(vload4(c, src) - amax);
|
||||
sum += (e.x + e.y) + (e.z + e.w);
|
||||
vstore4(convert_uchar4_sat_rte(e*255.0f), c, qdst);
|
||||
}
|
||||
} else {
|
||||
for (int c = 0; c < 8; ++c) {
|
||||
vstore4((uchar4)(0), c, qdst);
|
||||
}
|
||||
}
|
||||
bmax[prow*pblk + blk] = amax;
|
||||
bsum[prow*pblk + blk] = sum;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+11
-2
@@ -1,5 +1,7 @@
|
||||
#include "models.h"
|
||||
|
||||
#include <cinttypes>
|
||||
|
||||
// HRM-Text: alternating low/high transformer stacks over the same token stream.
|
||||
// Reference: HrmTextModel in transformers, DFM Mimir 1B.
|
||||
|
||||
@@ -19,8 +21,15 @@ void llama_model_hrm_text::load_arch_hparams(llama_model_loader & ml) {
|
||||
GGML_ASSERT(hparams.n_hrm_l_cycles > 0);
|
||||
|
||||
// the GGUF block count is the expanded cache-slot count
|
||||
const uint32_t n_slot = hparams.n_hrm_layers_per_stack * hparams.n_hrm_h_cycles * (hparams.n_hrm_l_cycles + 1);
|
||||
GGML_ASSERT(hparams.n_layer() == n_slot);
|
||||
// 64-bit: a 32-bit product can wrap and match n_layer(), then load_arch_tensors() indexes past model.layers
|
||||
const uint64_t n_slot = (uint64_t) hparams.n_hrm_layers_per_stack * hparams.n_hrm_h_cycles * (hparams.n_hrm_l_cycles + 1);
|
||||
if (hparams.n_hrm_layers_per_stack >= hparams.n_layer() || n_slot != hparams.n_layer()) {
|
||||
throw std::runtime_error(format(
|
||||
"hrm: cache-slot count %" PRIu64 " (layers_per_stack %u, h_cycles %u, l_cycles %u) "
|
||||
"does not match block_count %u",
|
||||
n_slot, hparams.n_hrm_layers_per_stack, hparams.n_hrm_h_cycles,
|
||||
hparams.n_hrm_l_cycles, hparams.n_layer()));
|
||||
}
|
||||
|
||||
switch (hparams.n_embd) {
|
||||
case 1536:
|
||||
|
||||
@@ -12407,6 +12407,9 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_BF16, GGML_TYPE_F32, 16, 16, b, 50, 200, 64));
|
||||
}
|
||||
|
||||
// One very large one to trigger a code path in CUDA requiring >1 pass over expert IDs due to shared memory limits:
|
||||
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_Q8_0, GGML_TYPE_F32, 16, 16, false, 32, 65536, 256));
|
||||
|
||||
// For issue 27873
|
||||
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_IQ2_XXS, GGML_TYPE_F32, 1, 1, false, 1, 8192, 4096));
|
||||
|
||||
@@ -12441,8 +12444,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 28, 10, false, 1024, 1, 4096));
|
||||
test_cases.emplace_back(new test_mul_mat_id(GGML_TYPE_TQ1_0, GGML_TYPE_F32, 128, 8, false, 1024, 1, 2048));
|
||||
|
||||
// TODO: disabled until CUDA backend support is added [TAG_MUL_MAT_ID_DUPLICATE_ID]
|
||||
#if 0
|
||||
// repeated expert ids within a row
|
||||
for (ggml_type ta : {GGML_TYPE_F16, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0}) {
|
||||
for (int n : {9, 16, 33, 64}) {
|
||||
@@ -12454,7 +12455,6 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
|
||||
test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 1025, 10, false, 64, 33, 256, 0, true));
|
||||
test_cases.emplace_back(new test_mul_mat_id_dup(ta, GGML_TYPE_F32, 2050, 10, true, 64, 33, 256));
|
||||
}
|
||||
#endif
|
||||
|
||||
for (ggml_type type_a : all_types) {
|
||||
test_cases.emplace_back(new test_mul_mat_id(type_a, GGML_TYPE_F32, 4, 2, false, 64, 16, 3*ggml_blck_size(type_a)));
|
||||
|
||||
Reference in New Issue
Block a user