From 0278d8362d78c5de291bc03b76016f7f74b2ab77 Mon Sep 17 00:00:00 2001 From: Todor Boinovski Date: Wed, 22 Jul 2026 09:25:04 -0700 Subject: [PATCH] hexagon: activation ops update (#25974) * hex-geglu: optimized all-in-one geglu microkernel * hex-geglu: enable non-contiguous src and strided DMA * hex-act: enable non-contiguous srs and strided DMA for rest of ACT ops * hex-act: generalize GLU per-thread functions via DEFINE_GLU_PER_THREAD macro * hexagon: move UNARY_SILU and UNARY_GELU to unary-ops * hex-act: replace the generic ops_context scratchpad usage with a local htp_vtcm_layout computation per act op. --------- Co-authored-by: Max Krasnyansky --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 11 +- ggml/src/ggml-hexagon/htp/act-ops.c | 954 ++++++++++--------------- ggml/src/ggml-hexagon/htp/main.c | 4 +- ggml/src/ggml-hexagon/htp/unary-ops.c | 56 ++ ggml/src/ggml-hexagon/htp/unary-ops.h | 2 + 5 files changed, 455 insertions(+), 572 deletions(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 54d313ade5..d19d2d5f03 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -3084,7 +3084,10 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session return false; } - if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(dst)) { + if (!ggml_is_contiguous_1(src0)) { + return false; + } + if (!ggml_is_contiguous(dst)) { return false; } @@ -3095,7 +3098,7 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session if (!ggml_are_same_shape(src0, src1)) { return false; } - if (!ggml_is_contiguous(src1)) { + if (!ggml_is_contiguous_1(src1)) { return false; } } @@ -4152,12 +4155,10 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons case GGML_UNARY_OP_SIGMOID: case GGML_UNARY_OP_SOFTPLUS: case GGML_UNARY_OP_TANH: - supp = ggml_hexagon_supported_unary(sess, op); - break; case GGML_UNARY_OP_SILU: case GGML_UNARY_OP_GELU: case GGML_UNARY_OP_GELU_QUICK: - supp = ggml_hexagon_supported_activations(sess, op); + supp = ggml_hexagon_supported_unary(sess, op); break; default: break; diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.c index e56f2b6a52..9973c088dd 100644 --- a/ggml/src/ggml-hexagon/htp/act-ops.c +++ b/ggml/src/ggml-hexagon/htp/act-ops.c @@ -17,6 +17,7 @@ #include "htp-ops.h" #include "htp-ops.h" #include "htp-tensor.h" +#include "htp-vtcm.h" #define htp_act_preamble \ const struct htp_tensor * src0 = actx->octx->src[0]; \ @@ -54,581 +55,413 @@ const uint32_t nb3 = dst->nb[3]; struct htp_act_context { - struct htp_ops_context * octx; + struct htp_ops_context * octx; // Precomputed values - const uint8_t * data_src0; - const uint8_t * data_src1; - uint8_t * data_dst; + const uint8_t * data_src0; + const uint8_t * data_src1; + uint8_t * data_dst; - size_t src0_row_size; - size_t src1_row_size; - size_t dst_row_size; + size_t src0_row_size; + size_t src1_row_size; + size_t dst_row_size; - size_t src0_row_size_aligned; - size_t src1_row_size_aligned; - size_t dst_row_size_aligned; + size_t src0_row_stride; + size_t src1_row_stride; - size_t src0_spad_half_size; - size_t src1_spad_half_size; - size_t dst_spad_half_size; + size_t src0_row_size_aligned; + size_t src1_row_size_aligned; + size_t dst_row_size_aligned; - uint32_t block; - uint32_t src0_nrows; - uint32_t src0_nrows_per_thread; - int nc; + size_t src0_spad_half_size; + size_t src1_spad_half_size; + size_t dst_spad_half_size; + + uint32_t block; + uint32_t src0_nrows; + uint32_t src0_nrows_per_thread; + int nc; + + uint8_t * vtcm_src0; + uint8_t * vtcm_src1; + uint8_t * vtcm_dst; + + size_t vtcm_src0_size_per_thread; + size_t vtcm_src1_size_per_thread; + size_t vtcm_dst_size_per_thread; }; -static void glu_swiglu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { - struct htp_act_context * actx = (struct htp_act_context *) data; - htp_act_preamble; +struct htp_act_vtcm_layout { + size_t total_bytes; + size_t off_src0; + size_t off_src1; + size_t off_dst; - size_t src0_row_size = actx->src0_row_size; - size_t src1_row_size = actx->src1_row_size; - size_t dst_row_size = actx->dst_row_size; + size_t src0_bytes_per_thread; + size_t src1_bytes_per_thread; + size_t dst_bytes_per_thread; - const uint32_t src0_nrows = actx->src0_nrows; - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + uint32_t vtcm_row_per_thread; +}; - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; - } +static inline void htp_act_vtcm_layout_build(struct htp_act_vtcm_layout * L, + size_t src0_row_size_aligned, + size_t src1_row_size_aligned, + size_t dst_row_size_aligned, + uint32_t n_threads, + size_t vtcm_size) { + const size_t spad_size_per_row = src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned; + const uint32_t vtcm_row_per_thread = (uint32_t) (vtcm_size / (n_threads * spad_size_per_row)); - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + L->vtcm_row_per_thread = vtcm_row_per_thread; - const uint8_t * restrict data_src0 = actx->data_src0; - const uint8_t * restrict data_src1 = actx->data_src1; - uint8_t * restrict data_dst = actx->data_dst; + L->src0_bytes_per_thread = src0_row_size_aligned * vtcm_row_per_thread; + L->src1_bytes_per_thread = src1_row_size_aligned * vtcm_row_per_thread; + L->dst_bytes_per_thread = dst_row_size_aligned * vtcm_row_per_thread; - const int nc = actx->nc; + L->off_src0 = 0; + L->off_src1 = L->off_src0 + L->src0_bytes_per_thread * n_threads; + L->off_dst = L->off_src1 + L->src1_bytes_per_thread * n_threads; - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; - const size_t src1_row_size_aligned = actx->src1_row_size_aligned; - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; - - uint8_t * restrict src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread); - uint8_t * restrict src1_spad_data = actx->octx->src1_spad.data + (ith * actx->octx->src1_spad.size_per_thread); - uint8_t * restrict dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread); - - size_t src0_spad_half_size = actx->src0_spad_half_size; - size_t src1_spad_half_size = actx->src1_spad_half_size; - size_t dst_spad_half_size = actx->dst_spad_half_size; - - const int BLOCK = actx->block; - if (BLOCK == 0) { - FARF(ERROR, - "swiglu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", - actx->octx->src0_spad.size_per_thread, src0_row_size_aligned); - return; - } - - dma_queue * dma_queue = actx->octx->ctx->dma[ith]; - - // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - // Dummy DMA transation for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), - dst_row_size, dst_row_size_aligned, 0); - - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)), - src0_row_size_aligned, src0_row_size, block_size); - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)), - src1_row_size_aligned, src1_row_size, block_size); - } - - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - float * dst_spad = (float *) dma_queue_pop(dma_queue).src; - float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; - float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; - - for (uint32_t ib = 0; ib < block_size; ib++) { - const float * src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float)); - const float * src1_spad_ptr = src1_spad + ib * (src1_row_size_aligned / sizeof(float)); - float * dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float)); - - //swiglu(x) = x1 * sigmoid(x0) - hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, nc); - hvx_mul_mul_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, - (const uint8_t *) src1_spad_ptr, nc); - } - - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size, - dst_row_size_aligned, block_size); - - // prefetch N+2 loop iteration if any - const uint32_t pref_block = (ir + BLOCK * 2); - if (pref_block < src0_end_row) { - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)), - src0_row_size_aligned, src0_row_size, pref_block_size); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)), - src1_row_size_aligned, src1_row_size, pref_block_size); - } - } - - dma_queue_flush(dma_queue); - - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "swiglu-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, - ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); + L->total_bytes = L->off_dst + L->dst_bytes_per_thread * n_threads; } -static void glu_swiglu_oai_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { - struct htp_act_context * actx = (struct htp_act_context *) data; - htp_act_preamble; +#define htp_glu_op_preamble \ + const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \ + const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \ + const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \ + const int nc = actx->nc; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); +// swiglu(x) = x1 * sigmoid(x0) +static void swiglu_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx) { + htp_glu_op_preamble; - size_t src0_row_size = actx->src0_row_size; - size_t src1_row_size = actx->src1_row_size; - size_t dst_row_size = actx->dst_row_size; + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); - const uint32_t src0_nrows = actx->src0_nrows; - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; - - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; + hvx_sigmoid_f32_aa(dst_ptr, src0_ptr, nc); + hvx_mul_mul_f32_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc); } +} - const uint8_t * restrict data_src0 = actx->data_src0; - const uint8_t * restrict data_src1 = actx->data_src1; - uint8_t * restrict data_dst = actx->data_dst; - - const int nc = actx->nc; - - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; - const size_t src1_row_size_aligned = actx->src1_row_size_aligned; - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; - - uint8_t * restrict src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread); - uint8_t * restrict src1_spad_data = actx->octx->src1_spad.data + (ith * actx->octx->src1_spad.size_per_thread); - uint8_t * restrict dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread); - - size_t src0_spad_half_size = actx->src0_spad_half_size; - size_t src1_spad_half_size = actx->src1_spad_half_size; - size_t dst_spad_half_size = actx->dst_spad_half_size; - - const int BLOCK = actx->block; - if (BLOCK == 0) { - FARF(ERROR, - "swiglu-oai-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least " - "%zu\n", - actx->octx->src0_spad.size_per_thread, src0_row_size_aligned); - return; - } +// out = x * sigmoid(alpha * x) * (clamp(y, -limit, limit) + 1.f) +static void swiglu_oai_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx) { + htp_glu_op_preamble; const float alpha = ((const float *) (actx->octx->op_params))[2]; const float limit = ((const float *) (actx->octx->op_params))[3]; - dma_queue * dma_queue = actx->octx->ctx->dma[ith]; + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); - // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - // Dummy DMA transation for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), - dst_row_size, dst_row_size_aligned, 0); - - dma_queue_push_ddr_to_vtcm( - dma_queue, - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)), - src0_row_size_aligned, src0_row_size, block_size); - dma_queue_push_ddr_to_vtcm( - dma_queue, - dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)), - src1_row_size_aligned, src1_row_size, block_size); + // x (src0_ptr) = std::min(src0_p[k], limit); + hvx_min_scalar_f32((uint8_t *) src0_ptr, src0_ptr, limit, nc); + // y1 (src1_ptr) = std::clamp(src1_p[k], -limit, limit); + hvx_clamp_scalar_f32((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc); + // y (src1_ptr) = y1 + 1.f + hvx_add_scalar_f32((uint8_t *) src1_ptr, src1_ptr, 1.0, nc); + // x1 (dst_ptr) = alpha * x + hvx_mul_scalar_f32(dst_ptr, src0_ptr, alpha, nc); + // x2 (dst_ptr) = sigmoid(x1) = 1/(1+exp(-x1)) + hvx_sigmoid_f32_aa(dst_ptr, dst_ptr, nc); + // out = x * sigmoid(alpha * x) * (y + 1.f) + hvx_mul_mul_f32_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc); } - - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - float * dst_spad = (float *) dma_queue_pop(dma_queue).src; - float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; - float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; - - for (uint32_t ib = 0; ib < block_size; ib++) { - const float * src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float)); - const float * src1_spad_ptr = src1_spad + ib * (src1_row_size_aligned / sizeof(float)); - float * dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float)); - - // x (src0_spad_data) = std::min(src0_p[k], limit); - hvx_min_scalar_f32((uint8_t *) src0_spad_ptr, (const uint8_t *) src0_spad_ptr, limit, nc); - // y1 (src1_spad_data) = std::clamp(src1_p[k], -limit, limit); - hvx_clamp_scalar_f32((uint8_t *) src1_spad_ptr, (const uint8_t *) src1_spad_ptr, -limit, limit, nc); - // y (src1_spad_data) = y1 + 1.f - hvx_add_scalar_f32((uint8_t *) src1_spad_ptr, (const uint8_t *) src1_spad_ptr, 1.0, nc); - // x1 (dst_spad_data) = alpha * (x) - hvx_mul_scalar_f32((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, alpha, nc); - // x2 (dst_spad_data) = sigmoid(x1) = 1/(1+exp(-x1)) - hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, nc); - // out = x * sigmoid(alpha * x) * (y + 1.f) - hvx_mul_mul_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, - (const uint8_t *) src1_spad_ptr, nc); - } - - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size, - dst_row_size_aligned, block_size); - - // prefetch N+2 loop iteration if any - const uint32_t pref_block = (ir + BLOCK * 2); - if (pref_block < src0_end_row) { - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)), - src0_row_size_aligned, src0_row_size, pref_block_size); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)), - src1_row_size_aligned, src1_row_size, pref_block_size); - } - } - - dma_queue_flush(dma_queue); - - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "swiglu-oai-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, src0->ne[0], - src0->ne[1], src0->ne[2], src0->ne[3], src0_start_row, src0_end_row, src1->ne[0], src1->ne[1], src1->ne[2], - src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); -} - - -static void unary_gelu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { - struct htp_act_context * actx = (struct htp_act_context *) data; - htp_act_preamble; - - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const size_t src0_row_size = actx->src0_row_size; - const size_t dst_row_size = actx->dst_row_size; - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; - - const uint32_t src0_nrows = actx->src0_nrows; - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; - - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; - } - - const uint8_t * data_src0 = actx->data_src0; - uint8_t * data_dst = actx->data_dst; - - // nc/ne0 matches. - const int ne0_val = actx->nc; // == dst->ne[0] - - uint8_t * src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread); - uint8_t * dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread); - - size_t src0_spad_half_size = actx->src0_spad_half_size; - size_t dst_spad_half_size = actx->dst_spad_half_size; - - // In gelu = x*sigmoid(x*1.702) - const int BLOCK = actx->block; - - if (BLOCK == 0) { - FARF(ERROR, "gelu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", - actx->octx->src0_spad.size_per_thread, src0_row_size_aligned); - return; - } - - dma_queue * dma_queue = actx->octx->ctx->dma[ith]; - - // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - // Dummy DMA transation for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), - dst_row_size, dst_row_size_aligned, 0); - - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)), - src0_row_size_aligned, src0_row_size, block_size); - } - - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - float* dst_spad = (float *) dma_queue_pop(dma_queue).src; - float* src0_spad = (float *) dma_queue_pop(dma_queue).dst; - - for (uint32_t ib = 0; ib < block_size; ib++) { - const float* src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float)); - float* dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float)); - - // gelu = x * sigmoid(1.702 * x) // current implementation - hvx_mul_scalar_f32((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (float) 1.702, ne0_val); - hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0_val); - hvx_mul_f32_aaa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0_val); - } - - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), - dst_row_size, dst_row_size_aligned, block_size); - - // prefetch N+2 loop iteration if any - const uint32_t pref_block = (ir + BLOCK * 2); - if (pref_block < src0_end_row) { - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)), - src0_row_size_aligned, src0_row_size, pref_block_size); - } - } - - dma_queue_flush(dma_queue); - - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "gelu-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", ith, nth, ne00, ne01, ne02, - ne03, src0_start_row, src0_end_row, ne0, ne1, ne2, ne3, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); -} - - -static void unary_silu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { - struct htp_act_context * actx = (struct htp_act_context *) data; - htp_act_preamble; - - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); - - const size_t src0_row_size = actx->src0_row_size; - const size_t dst_row_size = actx->dst_row_size; - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; - - const uint32_t src0_nrows = actx->src0_nrows; - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; - - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); - - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; - } - - const uint8_t * data_src0 = actx->data_src0; - uint8_t * data_dst = actx->data_dst; - - const int ne0_val = actx->nc; // == dst->ne[0] - - uint8_t * src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread); - uint8_t * dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread); - - size_t src0_spad_half_size = actx->src0_spad_half_size; - size_t dst_spad_half_size = actx->dst_spad_half_size; - - const int BLOCK = actx->block; - - if (BLOCK == 0) { - FARF(ERROR, "silu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", - actx->octx->src0_spad.size_per_thread, src0_row_size_aligned); - return; - } - - dma_queue * dma_queue = actx->octx->ctx->dma[ith]; - - // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - // Dummy DMA transation for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), - dst_row_size, dst_row_size_aligned, 0); - - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)), - src0_row_size_aligned, src0_row_size, block_size); - } - - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - float* dst_spad = (float *) dma_queue_pop(dma_queue).src; - float* src0_spad = (float *) dma_queue_pop(dma_queue).dst; - - for (uint32_t ib = 0; ib < block_size; ib++) { - const float* src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float)); - float* dst_spad_ptr = dst_spad + ib * (dst_row_size_aligned / sizeof(float)); - - // silu = x * sigmoid(x) - hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, ne0_val); - hvx_mul_f32_aaa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0_val); - } - - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), - dst_row_size, dst_row_size_aligned, block_size); - - // prefetch N+2 loop iteration if any - const uint32_t pref_block = (ir + BLOCK * 2); - if (pref_block < src0_end_row) { - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)), - src0_row_size_aligned, src0_row_size, pref_block_size); - } - } - - dma_queue_flush(dma_queue); - - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "silu-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", ith, nth, ne00, ne01, ne02, - ne03, src0_start_row, src0_end_row, ne0, ne1, ne2, ne3, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); } static const float GELU_COEF_A = 0.044715f; static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; -static void glu_geglu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { - struct htp_act_context * actx = (struct htp_act_context *) data; - htp_act_preamble; +static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); - size_t src0_row_size = actx->src0_row_size; - size_t src1_row_size = actx->src1_row_size; - size_t dst_row_size = actx->dst_row_size; + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; - uint64_t t1, t2; - t1 = HAP_perf_get_qtimer_count(); + const uint32_t epv = 128 / sizeof(float); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; - const uint32_t src0_nrows = actx->src0_nrows; - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; + const float GELU_COEF_A_TIMES_SQRT = GELU_COEF_A * SQRT_2_OVER_PI; - const uint32_t src0_start_row = src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); + const HVX_Vector v_coef_a_times_sqrt = hvx_vec_splat_f32(GELU_COEF_A_TIMES_SQRT); + const HVX_Vector v_sqrt_2_pi = hvx_vec_splat_f32(SQRT_2_OVER_PI); + const HVX_Vector v_half = hvx_vec_splat_f32(0.5f); + const HVX_Vector v_one = hvx_vec_splat_f32(1.0f); + const HVX_Vector v_two = hvx_vec_splat_f32(2.0f); - // no work for this thread - if (src0_start_row >= src0_end_row) { - return; + // Hoisted fast sigmoid / inverse constants to avoid loop-internal overhead + const HVX_Vector v_log2f = Q6_V_vsplat_R(FAST_SIGMOID_LOG2F); + const HVX_Vector v_c1 = Q6_V_vsplat_R(FAST_SIGMOID_C1); + const HVX_Vector v_c2 = Q6_V_vsplat_R(FAST_SIGMOID_C2); + const HVX_Vector v_inv_aprox = Q6_V_vsplat_R(0x7EEEEBB3); + const HVX_Vector v_max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector v_min_exp = hvx_vec_splat_f32(-87.0f); + + uint32_t i = 0; + + for (; i < nvec; i++) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + + HVX_Vector x2 = hvx_vec_mul_f32_f32(x, x); + HVX_Vector coef = hvx_vec_mul_f32_f32(x2, v_coef_a_times_sqrt); + coef = hvx_vec_add_f32_f32(coef, v_sqrt_2_pi); + HVX_Vector inner = hvx_vec_mul_f32_f32(x, coef); + + // y2 = 2 * inner + HVX_Vector y2 = hvx_vec_mul_f32_f32(inner, v_two); + + // Sigmoid guard check predicates + HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(v_max_exp, y2); + HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(y2, v_min_exp); + + // Fast sigmoid approximation + HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(y2, v_log2f); + v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), v_half); + + HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v)); + HVX_Vector x_sig = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); + HVX_Vector xx_sig = Q6_Vqf32_vmpy_Vqf32Vqf32(x_sig, x_sig); + + HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx_sig), v_c2); + v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, v_log2f); + + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x_sig), v_c1); + v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx_sig); + v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x_sig); + + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); + v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); + + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); + HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); + + // Fast division (Newton-Raphson with 2 iterations) + HVX_Vector i_sf = Q6_Vw_vsub_VwVw(v_inv_aprox, v5); + HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf( + i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5))))); + r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32( + r_qf, Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5)))); + HVX_Vector res_inv = Q6_Vsf_equals_Vqf32(r_qf); + + HVX_Vector sig2y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, res_inv)); + + // Sigmoid guards + sig2y = Q6_V_vmux_QVV(pred_max, sig2y, v_one); + sig2y = Q6_V_vmux_QVV(pred_min, sig2y, Q6_V_vzero()); + + // tanh(inner) = 2 * sigmoid(2 * inner) - 1 + HVX_Vector tanh_val = hvx_vec_mul_f32_f32(sig2y, v_two); + tanh_val = hvx_vec_sub_f32_f32(tanh_val, v_one); + + HVX_Vector tanh_plus_one = hvx_vec_add_f32_f32(tanh_val, v_one); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, v_half); + HVX_Vector gelu_x = hvx_vec_mul_f32_f32(half_x, tanh_plus_one); + + vdst[i] = hvx_vec_mul_f32_f32(gelu_x, g); } - const uint8_t * restrict data_src0 = actx->data_src0; - const uint8_t * restrict data_src1 = actx->data_src1; - uint8_t * restrict data_dst = actx->data_dst; + if (nloe) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; - const int nc = actx->nc; + HVX_Vector x2 = hvx_vec_mul_f32_f32(x, x); + HVX_Vector coef = hvx_vec_mul_f32_f32(x2, v_coef_a_times_sqrt); + coef = hvx_vec_add_f32_f32(coef, v_sqrt_2_pi); + HVX_Vector inner = hvx_vec_mul_f32_f32(x, coef); - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; - const size_t src1_row_size_aligned = actx->src1_row_size_aligned; - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; + HVX_Vector y2 = hvx_vec_mul_f32_f32(inner, v_two); - uint8_t * restrict src0_spad_data = actx->octx->src0_spad.data + (ith * actx->octx->src0_spad.size_per_thread); - uint8_t * restrict src1_spad_data = actx->octx->src1_spad.data + (ith * actx->octx->src1_spad.size_per_thread); - uint8_t * restrict dst_spad_data = actx->octx->dst_spad.data + (ith * actx->octx->dst_spad.size_per_thread); + HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(v_max_exp, y2); + HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(y2, v_min_exp); - size_t src0_spad_half_size = actx->src0_spad_half_size; - size_t src1_spad_half_size = actx->src1_spad_half_size; - size_t dst_spad_half_size = actx->dst_spad_half_size; + HVX_Vector v = Q6_Vqf32_vmpy_VsfVsf(y2, v_log2f); + v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), v_half); - const int BLOCK = actx->block; - if (BLOCK == 0) { - FARF(ERROR, - "geglu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", - actx->octx->src0_spad.size_per_thread, src0_row_size_aligned); - return; + HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v)); + HVX_Vector x_sig = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); + HVX_Vector xx_sig = Q6_Vqf32_vmpy_Vqf32Vqf32(x_sig, x_sig); + + HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx_sig), v_c2); + v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, v_log2f); + + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x_sig), v_c1); + v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx_sig); + v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x_sig); + + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); + v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); + + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); + HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); + + HVX_Vector i_sf = Q6_Vw_vsub_VwVw(v_inv_aprox, v5); + HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf( + i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5))))); + r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32( + r_qf, Q6_Vqf32_vsub_VsfVsf(v_two, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5)))); + HVX_Vector res_inv = Q6_Vsf_equals_Vqf32(r_qf); + + HVX_Vector sig2y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, res_inv)); + + sig2y = Q6_V_vmux_QVV(pred_max, sig2y, v_one); + sig2y = Q6_V_vmux_QVV(pred_min, sig2y, Q6_V_vzero()); + + HVX_Vector tanh_val = hvx_vec_mul_f32_f32(sig2y, v_two); + tanh_val = hvx_vec_sub_f32_f32(tanh_val, v_one); + + HVX_Vector tanh_plus_one = hvx_vec_add_f32_f32(tanh_val, v_one); + HVX_Vector half_x = hvx_vec_mul_f32_f32(x, v_half); + HVX_Vector gelu_x = hvx_vec_mul_f32_f32(half_x, tanh_plus_one); + + HVX_Vector res = hvx_vec_mul_f32_f32(gelu_x, g); + hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(float), res); } - - dma_queue * dma_queue = actx->octx->ctx->dma[ith]; - - // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - // Dummy DMA transation for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, - dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), - dst_row_size, dst_row_size_aligned, 0); - - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)), - src0_row_size_aligned, src0_row_size, block_size); - dma_queue_push_ddr_to_vtcm(dma_queue, - dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)), - src1_row_size_aligned, src1_row_size, block_size); - } - - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - float * dst_spad = (float *) dma_queue_pop(dma_queue).src; - float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; - float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; - - for (uint32_t ib = 0; ib < block_size; ib++) { - const uint8_t * src0_spad_ptr = (const uint8_t *)(src0_spad + ib * (src0_row_size_aligned / sizeof(float))); - const uint8_t * src1_spad_ptr = (const uint8_t *)(src1_spad + ib * (src1_row_size_aligned / sizeof(float))); - uint8_t * dst_spad_ptr = (uint8_t *)(dst_spad + ib * (dst_row_size_aligned / sizeof(float))); - - // geglu tanh implementation - // geglu(x, g) = gelu(x) * g - // gelu(x) = 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x))) - hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, src0_spad_ptr, nc); // res = x*x - hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, GELU_COEF_A, nc); // res = res * GELU_COEF_A - hvx_add_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, 1.0f, nc); // res = res + 1.0f - hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, (const uint8_t *)dst_spad_ptr, nc); // res = res * x - hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t*)dst_spad_ptr, SQRT_2_OVER_PI, nc); // res = result * SQRT_2_OVER_PI - hvx_tanh_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, nc); // res = tanh(res) - hvx_add_scalar_f32_aa(dst_spad_ptr, (const uint8_t*)dst_spad_ptr, 1.0f, nc); // res = res + 1.0f - hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, (const uint8_t *)dst_spad_ptr, nc); // res = res * x - hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, 0.5f, nc); // res = res + 0.5f - hvx_mul_f32_aaa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, src1_spad_ptr, nc); // res = res * g - } - - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size, - dst_row_size_aligned, block_size); - - // prefetch N+2 loop iteration if any - const uint32_t pref_block = (ir + BLOCK * 2); - if (pref_block < src0_end_row) { - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)), - src0_row_size_aligned, src0_row_size, pref_block_size); - dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)), - src1_row_size_aligned, src1_row_size, pref_block_size); - } - } - - dma_queue_flush(dma_queue); - - t2 = HAP_perf_get_qtimer_count(); - - FARF(HIGH, "geglu-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, - ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3, - (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1)); } +// geglu(x, g) = gelu(x) * g +static void geglu_f32(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_geglu_f32_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + +#define DEFINE_GLU_PER_THREAD(NAME, OP_STR, CORE_EXPR) \ + static void glu_##NAME##_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_act_context * actx = (struct htp_act_context *) data; \ + htp_act_preamble; \ + \ + struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \ + \ + size_t src0_row_size = actx->src0_row_size; \ + size_t src1_row_size = actx->src1_row_size; \ + size_t dst_row_size = actx->dst_row_size; \ + \ + size_t src0_row_stride = actx->src0_row_stride; \ + size_t src1_row_stride = actx->src1_row_stride; \ + \ + const uint32_t src0_nrows = actx->src0_nrows; \ + const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \ + \ + const uint32_t src0_start_row = src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \ + \ + /* no work for this thread */ \ + if (src0_start_row >= src0_end_row) { \ + return; \ + } \ + \ + const uint8_t * restrict data_src0 = actx->data_src0; \ + const uint8_t * restrict data_src1 = actx->data_src1; \ + uint8_t * restrict data_dst = actx->data_dst; \ + \ + const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \ + const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \ + const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \ + \ + uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \ + uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \ + uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \ + \ + size_t src0_spad_half_size = actx->src0_spad_half_size; \ + size_t src1_spad_half_size = actx->src1_spad_half_size; \ + size_t dst_spad_half_size = actx->dst_spad_half_size; \ + \ + const int BLOCK = actx->block; \ + if (BLOCK == 0) { \ + FARF(ERROR, \ + OP_STR \ + " : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", \ + actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \ + return; \ + } \ + \ + dma_queue * dma_queue = actx->octx->ctx->dma[ith]; \ + \ + /* See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 */ \ + for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \ + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ + \ + /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \ + dma_queue_push_vtcm_to_ddr(dma_queue, \ + dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \ + dst_row_size, dst_row_size_aligned, 0); \ + \ + dma_queue_push( \ + dma_queue, \ + dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \ + src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \ + dma_queue_push( \ + dma_queue, \ + dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \ + src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \ + } \ + \ + for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \ + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ + \ + float * dst_spad = (float *) dma_queue_pop(dma_queue).src; \ + float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; \ + float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; \ + \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ + CORE_EXPR; \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ + \ + dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), \ + dst_row_size, dst_row_size_aligned, block_size); \ + \ + /* prefetch N+2 loop iteration if any */ \ + const uint32_t pref_block = (ir + BLOCK * 2); \ + if (pref_block < src0_end_row) { \ + const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \ + dma_queue_push(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_stride)), \ + src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \ + dma_queue_push(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_stride)), \ + src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \ + } \ + } \ + \ + dma_queue_flush(dma_queue); \ + \ + } + +DEFINE_GLU_PER_THREAD(swiglu, "swiglu-f32", swiglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx)) +DEFINE_GLU_PER_THREAD(swiglu_oai, "swiglu-oai-f32", swiglu_oai_f32(src0_spad, src1_spad, dst_spad, block_size, actx)) +DEFINE_GLU_PER_THREAD(geglu, "geglu-f32", geglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx)) + static int execute_op_activations_f32(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; - if (((src0->ne[0] * SIZEOF_FP32) != src0->nb[1]) || ((dst->ne[0] * SIZEOF_FP32) != dst->nb[1])) { - FARF(ERROR, "Non-contiguous tensors are not supported at this time \n"); + if ((dst->ne[0] * SIZEOF_FP32) != dst->nb[1]) { + FARF(ERROR, "Non-contiguous dst is not supported at this time \n"); return HTP_STATUS_NO_SUPPORT; } @@ -636,11 +469,6 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { const char * op_type = NULL; switch (octx->op) { - case HTP_OP_UNARY_SILU: - act_op_func = (worker_callback_t)unary_silu_f32_per_thread; - op_type = "silu-f32"; - break; - case HTP_OP_GLU_SWIGLU: act_op_func = (worker_callback_t)glu_swiglu_f32_per_thread; op_type = "swiglu-f32"; @@ -650,10 +478,6 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { act_op_func = (worker_callback_t)glu_swiglu_oai_f32_per_thread; op_type = "swiglu-oai-f32"; break; - case HTP_OP_UNARY_GELU: - act_op_func = (worker_callback_t)unary_gelu_f32_per_thread; - op_type = "gelu-f32"; - break; case HTP_OP_GLU_GEGLU: act_op_func = (worker_callback_t)glu_geglu_f32_per_thread; @@ -667,51 +491,39 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); - size_t src0_row_size = src0->nb[1]; - size_t src1_row_size = src1 ? src1->nb[1] : src0->nb[1]; - size_t dst_row_size = dst->nb[1]; + // row_size = bytes of useful data per row (what the kernel touches / what DMA copies). + // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src). + const size_t nc_bytes = dst->ne[0] * SIZEOF_FP32; + const size_t src0_row_size = nc_bytes; + const size_t src1_row_size = nc_bytes; + const size_t dst_row_size = nc_bytes; + const size_t src0_row_stride = src0->nb[1]; + const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1]; const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN); const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); - // VTCM scratchpads for all tensors - // N rows per thread, padded to HVX vector size - size_t spad_size_per_row = (src0_row_size_aligned + src1_row_size_aligned) + dst_row_size_aligned; - size_t vtcm_row_per_thread = (octx->ctx->vtcm_size)/ (n_threads* spad_size_per_row); + struct htp_act_vtcm_layout L; + htp_act_vtcm_layout_build(&L, src0_row_size_aligned, src1_row_size_aligned, dst_row_size_aligned, n_threads, + octx->ctx->vtcm_size); // Make sure the reserved vtcm size is sufficient - if (vtcm_row_per_thread == 0) { + if (L.vtcm_row_per_thread == 0) { FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size, - spad_size_per_row * n_threads); + (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads); return HTP_STATUS_VTCM_TOO_SMALL; } - octx->src0_spad.size_per_thread = src0_row_size_aligned * vtcm_row_per_thread; - octx->src1_spad.size_per_thread = src1_row_size_aligned * vtcm_row_per_thread; - octx->dst_spad.size_per_thread = dst_row_size_aligned * vtcm_row_per_thread; - - octx->dst_spad.size = n_threads* octx->dst_spad.size_per_thread; - octx->src0_spad.size = n_threads* octx->src0_spad.size_per_thread; - octx->src1_spad.size = n_threads* octx->src1_spad.size_per_thread; - - octx->src0_spad.data = octx->ctx->vtcm_base; - octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; - octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size; - - octx->src0_spad.src = NULL; - octx->src1_spad.src = NULL; - octx->dst_spad.src = NULL; - if (src1) { - FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n", + FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], - src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], octx->src0_spad.size, octx->src1_spad.size, - octx->dst_spad.size); + src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads, + L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); } else { - FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n", op_type, + FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], - octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size); + L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); } if ((octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { @@ -732,9 +544,21 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { actx.src1_row_size_aligned = src1_row_size_aligned; actx.dst_row_size_aligned = dst_row_size_aligned; - actx.src0_spad_half_size = octx->src0_spad.size_per_thread / 2; - actx.src1_spad_half_size = octx->src1_spad.size_per_thread / 2; - actx.dst_spad_half_size = octx->dst_spad.size_per_thread / 2; + actx.src0_row_stride = src0_row_stride; + actx.src1_row_stride = src1_row_stride; + + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + actx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + actx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + actx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); + + actx.vtcm_src0_size_per_thread = L.src0_bytes_per_thread; + actx.vtcm_src1_size_per_thread = L.src1_bytes_per_thread; + actx.vtcm_dst_size_per_thread = L.dst_bytes_per_thread; + + actx.src0_spad_half_size = L.src0_bytes_per_thread / 2; + actx.src1_spad_half_size = L.src1_bytes_per_thread / 2; + actx.dst_spad_half_size = L.dst_bytes_per_thread / 2; actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned; actx.src0_nrows = src0_nrows; diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index e76ac7f488..321b832e14 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -723,14 +723,14 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_SQRT: case HTP_OP_UNARY_SOFTPLUS: case HTP_OP_UNARY_SIGMOID: + case HTP_OP_UNARY_SILU: + case HTP_OP_UNARY_GELU: case HTP_OP_UNARY_NEG: case HTP_OP_UNARY_EXP: case HTP_OP_UNARY_TANH: case HTP_OP_L2_NORM: return op_unary(octx); - case HTP_OP_UNARY_SILU: - case HTP_OP_UNARY_GELU: case HTP_OP_GLU_SWIGLU: case HTP_OP_GLU_SWIGLU_OAI: case HTP_OP_GLU_GEGLU: diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.c b/ggml/src/ggml-hexagon/htp/unary-ops.c index c6806efc37..b21415a67d 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.c +++ b/ggml/src/ggml-hexagon/htp/unary-ops.c @@ -276,6 +276,39 @@ static void sigmoid_f32(const float * restrict src, } } +// silu(x) = x * sigmoid(x) +static void silu_f32(const float * restrict src, + float * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_sigmoid_f32_aa(dst_local, src_local, ne0); + hvx_mul_f32_aaa(dst_local, src_local, dst_local, ne0); + } +} + +// gelu(x) = x * sigmoid(1.702 * x) (quick/sigmoid approximation, matches CPU GELU_QUICK reference) +static void gelu_f32(const float * restrict src, + float * restrict dst, + const uint32_t num_rows, + const struct htp_unary_context * uctx) { + htp_unary_op_preamble; + + for (uint32_t ir = 0; ir < num_rows; ir++) { + const uint8_t * restrict src_local = (const uint8_t *)src + (ir * src0_row_size_aligned); + uint8_t * restrict dst_local = (uint8_t *)dst + (ir * dst_row_size_aligned); + + hvx_mul_scalar_f32(dst_local, src_local, 1.702f, ne0); + hvx_sigmoid_f32_aa(dst_local, dst_local, ne0); + hvx_mul_f32_aaa(dst_local, src_local, dst_local, ne0); + } +} + static void tri_f32(const float * restrict src, float * restrict dst, const uint32_t num_rows, @@ -566,6 +599,8 @@ DEFINE_UNARY_TASK(sqrt, false, false, sqrt_f32(src0_vtcm, dst_vtcm, bl DEFINE_UNARY_TASK(unary_neg, false, false, neg_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(unary_exp, false, false, exp_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(unary_sigmoid, false, false, sigmoid_f32(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK(unary_silu, false, false, silu_f32(src0_vtcm, dst_vtcm, block_size, uctx)) +DEFINE_UNARY_TASK(unary_gelu, false, false, gelu_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(unary_softplus, false, false, softplus_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(unary_tanh, false, false, tanh_f32(src0_vtcm, dst_vtcm, block_size, uctx)) DEFINE_UNARY_TASK(l2_norm, false, false, l2_norm_f32(src0_vtcm, dst_vtcm, block_size, uctx)) @@ -717,6 +752,19 @@ static inline void tile_unary_softplus_f32(uint8_t * dst_vtcm, const uint8_t * s } } +// silu(x) = x * sigmoid(x) +static inline void tile_silu_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) { + hvx_sigmoid_f32_aa(dst_vtcm, src_vtcm, tw); + hvx_mul_f32_aaa(dst_vtcm, src_vtcm, dst_vtcm, tw); +} + +// gelu(x) = x * sigmoid(1.702 * x) (quick/sigmoid approximation, matches CPU GELU_QUICK reference) +static inline void tile_gelu_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw) { + hvx_mul_scalar_f32(dst_vtcm, src_vtcm, 1.702f, tw); + hvx_sigmoid_f32_aa(dst_vtcm, dst_vtcm, tw); + hvx_mul_f32_aaa(dst_vtcm, src_vtcm, dst_vtcm, tw); +} + // Triangular mask applied to one column tile. Boundary is an absolute column index, so // each vector compares against its absolute column position (col_start + i*VLEN_FP32). static inline void tri_apply_tile_f32(const uint8_t * restrict src, uint8_t * restrict dst, @@ -798,6 +846,8 @@ DEFINE_UNARY_TILED_TASK(sqrt, false, hvx_sqrt_f32_aa(dst_vtcm, src_vtc DEFINE_UNARY_TILED_TASK(unary_neg, false, hvx_scale_f32_aa(dst_vtcm, src_vtcm, tw, -1.0f)) DEFINE_UNARY_TILED_TASK(unary_exp, false, hvx_exp_f32(dst_vtcm, src_vtcm, tw, false)) DEFINE_UNARY_TILED_TASK(unary_sigmoid, false, hvx_sigmoid_f32_aa(dst_vtcm, src_vtcm, tw)) +DEFINE_UNARY_TILED_TASK(unary_silu, false, tile_silu_f32(dst_vtcm, src_vtcm, tw)) +DEFINE_UNARY_TILED_TASK(unary_gelu, false, tile_gelu_f32(dst_vtcm, src_vtcm, tw)) DEFINE_UNARY_TILED_TASK(unary_softplus, false, tile_unary_softplus_f32(dst_vtcm, src_vtcm, tw)) DEFINE_UNARY_TILED_TASK(unary_tanh, false, hvx_tanh_f32_aa(dst_vtcm, src_vtcm, tw)) DEFINE_UNARY_TILED_TASK(tri, true, tri_apply_tile_f32(src_vtcm, dst_vtcm, tw, col, i01, ne0, tri_ttype)) @@ -821,6 +871,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { case HTP_OP_UNARY_NEG: op_type = "neg-f32"; break; case HTP_OP_UNARY_EXP: op_type = "exp-f32"; break; case HTP_OP_UNARY_SIGMOID: op_type = "sigmoid-f32"; break; + case HTP_OP_UNARY_SILU: op_type = "silu-f32"; break; + case HTP_OP_UNARY_GELU: op_type = "gelu-f32"; break; case HTP_OP_UNARY_SOFTPLUS: op_type = "softplus-f32"; break; case HTP_OP_UNARY_TANH: op_type = "tanh-f32"; break; case HTP_OP_L2_NORM: op_type = "l2norm-f32"; break; @@ -917,6 +969,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { case HTP_OP_UNARY_NEG: task_func = unary_task_f32_tiled_unary_neg; break; case HTP_OP_UNARY_EXP: task_func = unary_task_f32_tiled_unary_exp; break; case HTP_OP_UNARY_SIGMOID: task_func = unary_task_f32_tiled_unary_sigmoid; break; + case HTP_OP_UNARY_SILU: task_func = unary_task_f32_tiled_unary_silu; break; + case HTP_OP_UNARY_GELU: task_func = unary_task_f32_tiled_unary_gelu; break; case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_tiled_unary_softplus; break; case HTP_OP_UNARY_TANH: task_func = unary_task_f32_tiled_unary_tanh; break; case HTP_OP_TRI: task_func = unary_task_f32_tiled_tri; break; @@ -934,6 +988,8 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) { case HTP_OP_UNARY_NEG: task_func = unary_task_f32_unary_neg; break; case HTP_OP_UNARY_EXP: task_func = unary_task_f32_unary_exp; break; case HTP_OP_UNARY_SIGMOID: task_func = unary_task_f32_unary_sigmoid; break; + case HTP_OP_UNARY_SILU: task_func = unary_task_f32_unary_silu; break; + case HTP_OP_UNARY_GELU: task_func = unary_task_f32_unary_gelu; break; case HTP_OP_UNARY_SOFTPLUS: task_func = unary_task_f32_unary_softplus; break; case HTP_OP_UNARY_TANH: task_func = unary_task_f32_unary_tanh; break; case HTP_OP_L2_NORM: task_func = unary_task_f32_l2_norm; break; diff --git a/ggml/src/ggml-hexagon/htp/unary-ops.h b/ggml/src/ggml-hexagon/htp/unary-ops.h index 4005e65151..1f4c3a5c4d 100644 --- a/ggml/src/ggml-hexagon/htp/unary-ops.h +++ b/ggml/src/ggml-hexagon/htp/unary-ops.h @@ -51,6 +51,8 @@ static inline bool htp_op_is_unary(uint32_t opcode) { case HTP_OP_UNARY_NEG: case HTP_OP_UNARY_EXP: case HTP_OP_UNARY_SIGMOID: + case HTP_OP_UNARY_SILU: + case HTP_OP_UNARY_GELU: case HTP_OP_UNARY_SOFTPLUS: case HTP_OP_UNARY_TANH: case HTP_OP_L2_NORM: