upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-23 11:10:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

hexagon: activation ops update (#25974)

Browse Source 
* hex-geglu: optimized all-in-one geglu microkernel

* hex-geglu: enable non-contiguous src and strided DMA

* hex-act: enable non-contiguous srs and strided DMA for rest of ACT ops

* hex-act: generalize GLU per-thread functions via DEFINE_GLU_PER_THREAD macro

* hexagon: move UNARY_SILU and UNARY_GELU to unary-ops

* hex-act: replace the generic ops_context scratchpad usage with a local htp_vtcm_layout computation per act op.

---------

Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
This commit is contained in:
Todor Boinovski
2026-07-22 09:25:04 -07:00
committed by GitHub
parent e0833bf686
commit 0278d8362d
5 changed files with 455 additions and 572 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
ggml/src/ggml-hexagon/ggml-hexagon.cpp
View File

@@ -3084,7 +3084,10 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session
return false;
}
if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) {
if (!ggml_is_contiguous_1(src0)) {
return false;
}
if (!ggml_is_contiguous(dst)) {
return false;
}
@@ -3095,7 +3098,7 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session
if (!ggml_are_same_shape(src0, src1)) {
return false;
}
if (!ggml_is_contiguous(src1)) {
if (!ggml_is_contiguous_1(src1)) {
return false;
}
}
@@ -4152,12 +4155,10 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
case GGML_UNARY_OP_SIGMOID:
case GGML_UNARY_OP_SOFTPLUS:
case GGML_UNARY_OP_TANH:
supp = ggml_hexagon_supported_unary(sess, op);
break;
case GGML_UNARY_OP_SILU:
case GGML_UNARY_OP_GELU:
case GGML_UNARY_OP_GELU_QUICK:
supp = ggml_hexagon_supported_activations(sess, op);
supp = ggml_hexagon_supported_unary(sess, op);
break;
default:
break;

954
ggml/src/ggml-hexagon/htp/act-ops.c
View File

File diff suppressed because it is too large Load Diff

4
ggml/src/ggml-hexagon/htp/main.c
View File

@@ -723,14 +723,14 @@ static int execute_op(struct htp_ops_context * octx) {
case HTP_OP_SQRT:
case HTP_OP_UNARY_SOFTPLUS:
case HTP_OP_UNARY_SIGMOID:
case HTP_OP_UNARY_SILU:
case HTP_OP_UNARY_GELU:
case HTP_OP_UNARY_NEG:
case HTP_OP_UNARY_EXP:
case HTP_OP_UNARY_TANH:
case HTP_OP_L2_NORM:
return op_unary(octx);
case HTP_OP_UNARY_SILU:
case HTP_OP_UNARY_GELU:
case HTP_OP_GLU_SWIGLU:
case HTP_OP_GLU_SWIGLU_OAI:
case HTP_OP_GLU_GEGLU:

56
ggml/src/ggml-hexagon/htp/unary-ops.c
View File

@@ -276,6 +276,39 @@ static void sigmoid_f32(const float * restrict src,
}
}
// silu(x) = x * sigmoid(x)
static void silu_f32(const float * restrict src,
float * restrict dst,
const uint32_t num_rows,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
hvx_sigmoid_f32_aa(dst_local, src_local, ne0);
hvx_mul_f32_aaa(dst_local, src_local, dst_local, ne0);
}
}
// gelu(x) = x * sigmoid(1.702 * x) (quick/sigmoid approximation, matches CPU GELU_QUICK reference)
static void gelu_f32(const float * restrict src,
float * restrict dst,
const uint32_t num_rows,
const struct htp_unary_context * uctx) {
htp_unary_op_preamble;
for (uint32_t ir = 0; ir < num_rows; ir++) {
const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned);
uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned);
hvx_mul_scalar_f32(dst_local, src_local, 1.702f, ne0);
hvx_sigmoid_f32_aa(dst_local, dst_local, ne0);
hvx_mul_f32_aaa(dst_local, src_local, dst_local, ne0);
}
}
static void tri_f32(const float * restrict src,
float * restrict dst,
const uint32_t num_rows,
@@ -566,6 +599,8 @@ DEFINE_UNARY_TASK(sqrt, false, false, sqrt_f32(src0_vtcm, dst_vtcm, bl
DEFINE_UNARY_TASK(unary_neg, false, false, neg_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(unary_exp, false, false, exp_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(unary_sigmoid, false, false, sigmoid_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(unary_silu, false, false, silu_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(unary_gelu, false, false, gelu_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(unary_softplus, false, false, softplus_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(unary_tanh, false, false, tanh_f32(src0_vtcm, dst_vtcm, block_size, uctx))
DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx))
@@ -717,6 +752,19 @@ static inline void tile_unary_softplus_f32(uint8_t * dst_vtcm, const uint8_t * s
}
}
// silu(x) = x * sigmoid(x)
static inline void tile_silu_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) {
hvx_sigmoid_f32_aa(dst_vtcm, src_vtcm, tw);
hvx_mul_f32_aaa(dst_vtcm, src_vtcm, dst_vtcm, tw);
}
// gelu(x) = x * sigmoid(1.702 * x) (quick/sigmoid approximation, matches CPU GELU_QUICK reference)
static inline void tile_gelu_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) {
hvx_mul_scalar_f32(dst_vtcm, src_vtcm, 1.702f, tw);
hvx_sigmoid_f32_aa(dst_vtcm, dst_vtcm, tw);
hvx_mul_f32_aaa(dst_vtcm, src_vtcm, dst_vtcm, tw);
}
// Triangular mask applied to one column tile. Boundary is an absolute column index, so
// each vector compares against its absolute column position (col_start + i*VLEN_FP32).
static inline void tri_apply_tile_f32(const uint8_t * restrict src, uint8_t * restrict dst,
@@ -798,6 +846,8 @@ DEFINE_UNARY_TILED_TASK(sqrt, false, hvx_sqrt_f32_aa(dst_vtcm, src_vtc
DEFINE_UNARY_TILED_TASK(unary_neg, false, hvx_scale_f32_aa(dst_vtcm, src_vtcm, tw, -1.0f))
DEFINE_UNARY_TILED_TASK(unary_exp, false, hvx_exp_f32(dst_vtcm, src_vtcm, tw, false))
DEFINE_UNARY_TILED_TASK(unary_sigmoid, false, hvx_sigmoid_f32_aa(dst_vtcm, src_vtcm, tw))
DEFINE_UNARY_TILED_TASK(unary_silu, false, tile_silu_f32(dst_vtcm, src_vtcm, tw))
DEFINE_UNARY_TILED_TASK(unary_gelu, false, tile_gelu_f32(dst_vtcm, src_vtcm, tw))
DEFINE_UNARY_TILED_TASK(unary_softplus, false, tile_unary_softplus_f32(dst_vtcm, src_vtcm, tw))
DEFINE_UNARY_TILED_TASK(unary_tanh, false, hvx_tanh_f32_aa(dst_vtcm, src_vtcm, tw))
DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype))
@@ -821,6 +871,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break;
case HTP_OP_UNARY_EXP: op_type = "exp-f32"; break;
case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break;
case HTP_OP_UNARY_SILU: op_type = "silu-f32"; break;
case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break;
case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break;
case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break;
case HTP_OP_L2_NORM: op_type = "l2norm-f32"; break;
@@ -917,6 +969,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
case HTP_OP_UNARY_NEG: task_func = unary_task_f32_tiled_unary_neg; break;
case HTP_OP_UNARY_EXP: task_func = unary_task_f32_tiled_unary_exp; break;
case HTP_OP_UNARY_SIGMOID: task_func = unary_task_f32_tiled_unary_sigmoid; break;
case HTP_OP_UNARY_SILU: task_func = unary_task_f32_tiled_unary_silu; break;
case HTP_OP_UNARY_GELU: task_func = unary_task_f32_tiled_unary_gelu; break;
case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_tiled_unary_softplus; break;
case HTP_OP_UNARY_TANH: task_func = unary_task_f32_tiled_unary_tanh; break;
case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break;
@@ -934,6 +988,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
case HTP_OP_UNARY_NEG: task_func = unary_task_f32_unary_neg; break;
case HTP_OP_UNARY_EXP: task_func = unary_task_f32_unary_exp; break;
case HTP_OP_UNARY_SIGMOID: task_func = unary_task_f32_unary_sigmoid; break;
case HTP_OP_UNARY_SILU: task_func = unary_task_f32_unary_silu; break;
case HTP_OP_UNARY_GELU: task_func = unary_task_f32_unary_gelu; break;
case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_unary_softplus; break;
case HTP_OP_UNARY_TANH: task_func = unary_task_f32_unary_tanh; break;
case HTP_OP_L2_NORM: task_func = unary_task_f32_l2_norm; break;

2
ggml/src/ggml-hexagon/htp/unary-ops.h
View File

@@ -51,6 +51,8 @@ static inline bool htp_op_is_unary(uint32_t opcode) {
case HTP_OP_UNARY_NEG:
case HTP_OP_UNARY_EXP:
case HTP_OP_UNARY_SIGMOID:
case HTP_OP_UNARY_SILU:
case HTP_OP_UNARY_GELU:
case HTP_OP_UNARY_SOFTPLUS:
case HTP_OP_UNARY_TANH:
case HTP_OP_L2_NORM: