upstream-archive/whisper.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/whisper.cpp.git synced 2026-09-28 08:57:51 -05:00
Code Issues Packages Projects Releases Wiki Activity

hexagon: support for multi-device model split (aka row-split) (llama/28589)

Browse Source 
* hex-row-split: add support for multi-device row spliting

Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>

* hex-mdev: add work splitting to fused kernels

* hex-mdev: use mdev_ prefix for all multi-device state

* hex-mdev: make device configuration more expressive to support device groups

* hex-mdev: fix mdev session init

* hex-mdev: fused nx (2x,3x) matmuls must update row counts for each w/o

* hex-mdev: fix MUL_MAT work partitioning bugs introduced by mdev

* hex-cont: fix crashes with new tests due to wrong striding

* hex-mdev: move fences after l2flushes

* hex-cont: fix work splitting for mnpu -- align chunks to cachelines

* hex-mdev: fix CPY tests with multi-dev

* hex-mmid: fix work partitioning with mnpu

* hex-mm: fix test failures with mdev

* hex-binary: fix work partitioning for mdev

* hex-argsort: fix mdev partitioning

* hex-mdev: fix work partitioning and general updates for all simple ops

* hex-fa: fix mdev work splitting issues

* hex-mdev: fixing more failing ops test

* hex-mdev: update the rest of the ops

* hex-mdev: refactor all mdev splitting logic to be contained within if (mdev_count > 1) {...}

* hex-mdev: fix macros

* hex-mdev: simplify session flush logic

* hex-sync: fix recursion in session flush

* hex-mdev: factor out fence buffer and allocator

* hex-fence: make fence allocation more robust with reserved slots for mdev

* hex-mdev: keep all mdev state in htp_mdev_group

* hex-mdev: further cleanup mdev group handling at the host

* hex-mdev: update group idx in the opbatch before serializing

* hex-batch: remove separate op_pending and use batch_req/rsp_seq

* hex-async: workaround another missing tensor_init in ggml-meta

* hex-fence: cleanup and robustify fences and error handling in multi-device scenarios

* hex-ar: improve ALLREDUCE error handling

* hex-async: robust error handling for op_cpy_fence

* hex-async: use seq0 from allreduce context to allocate fence_seq

* hex-mdev: fix remaining issues with fence and barrier clearing in CPY_FENCE

* hex-misc: realign macros and fix misplaces trace events

* hex-misc: align macros

* hex-mdev: fix unclone buffer re-entrancy

* hex-glu: fix mdev partitioning logic

* hex-mdev: make buffer uncloning/cleanup work with tensor-split scenarios

* hex-mdev: tighten up the can_split check in act-ops

* hex-mdev: factor out common bits of the partitioning logic

* hex-mm: minor realignment of the macros

* hex-bufs: fix incorrectly placed assert for MAX_BUFS

* hex-pad: tighten up gating checks for PAD

* hex-kparams: make sure all kernels properly use kparams->n_threads

* hex-docs: update user and developer docs with new features and detailed guide for ops development

* hex-scripts: update run script to properly parse dev groups

* hex-misc: formatting

* hex-sess: minor cleanup for session init

* hex-ar: fix vtcm size calc in allreduce kparams

* hex-scripts: fix flake8 warnings

* hex-rope: update ROPE to support mdev work split

* hex-ops: remove redunant checks and minor reformat

* hex-dev-guide: update dev-guide to avoid redundant null checks

* hex-async: improve event_wait, event_sync and fence implementations

* hex-async: remove synchronous flush from event_sync

* hex-async: symplify fence recovery protocol and make sync more robust

* hex-async: futher simplify error recovery for fences

* hex-err: return status instead of just -1

* hex-async: print all seq nums in hex

* hex-async: make sure fences flush dirty ranges

* hex-async: add dirty ranges merging to reduce fence flushes

* hex-async: properly sync before freeing the event

* hex-async: make sure fence owner session is not overriden

* hex-async: more fence write order more robust

* hex-async: make sure not to fuse ALLREDUCE+ADD if their dsts overlap

* hex-fusion: cleanup redundant checks

---------

Co-authored-by: Alexander Lu <alexlu@qti.qualcomm.com>
This commit is contained in:
Max Krasnyansky
2026-09-14 20:45:06 +03:00
committed by Georgi Gerganov
co-authored by Alexander Lu
parent b17794130a
commit ea3ef8f581
41 changed files with 3546 additions and 1836 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-hexagon/ggml-hexagon.cpp
+820 -343
View File
File diff suppressed because it is too large Load Diff
ggml/src/ggml-hexagon/htp-opnode.h
+6
View File
@@ -344,6 +344,12 @@ struct htp_opformat {
} else if (htp_op_is_unary(node.opcode)) {
const auto * kparams = (const struct htp_unary_kernel_params *) node.kernel_params;
snprintf(str, max_size, "%s vtcm %d", kparams->col_tile ? "wide-row" : "row-block", (int) kparams->vtcm_size);
} else if (node.opcode == HTP_OP_MDEV_GROUP && node.node) {
snprintf(str, max_size, "idx %d count %d", (int) node.node->op_params[0], (int) node.dst()->ne[1]);
} else if ((node.opcode == HTP_OP_FENCE || node.opcode == HTP_OP_CPY_FENCE) && node.node) {
snprintf(str, max_size, "seq 0x%x", (uint32_t) node.node->op_params[0]);
} else if (node.opcode == HTP_OP_ALLREDUCE && node.node) {
snprintf(str, max_size, "seq 0x%x -> 0x%x", (uint32_t) node.node->op_params[0], (uint32_t) node.node->op_params[1]);
} else {
snprintf(str, max_size, "----");
}
ggml/src/ggml-hexagon/htp/act-ops.c
+125 -108
View File
@@ -3,7 +3,6 @@
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <HAP_perf.h>
#include <math.h>
#include <string.h>
@@ -15,7 +14,7 @@
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-ops.h"
#include "hex-common.h"
#include "htp-tensor.h"
#include "htp-vtcm.h"
@@ -80,6 +79,7 @@ struct htp_act_context {
uint32_t block;
uint32_t src0_nrows;
uint32_t src0_nrows_per_thread;
uint32_t row_start;
int nc;
uint8_t * vtcm_src0;
@@ -329,104 +329,104 @@ static void geglu_f32(const float * restrict src0,
}
}
#define DEFINE_GLU_PER_THREAD(NAME, OP_STR, CORE_EXPR) \
static void glu_##NAME##_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { \
struct htp_act_context * actx = (struct htp_act_context *) data; \
htp_act_preamble; \
\
struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \
\
size_t src0_row_size = actx->src0_row_size; \
size_t src1_row_size = actx->src1_row_size; \
size_t dst_row_size = actx->dst_row_size; \
\
size_t src0_row_stride = actx->src0_row_stride; \
size_t src1_row_stride = actx->src1_row_stride; \
\
const uint32_t src0_nrows = actx->src0_nrows; \
const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \
\
const uint32_t src0_start_row = src0_nrows_per_thread * ith; \
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \
\
/* no work for this thread */ \
if (src0_start_row >= src0_end_row) { \
return; \
} \
\
const uint8_t * restrict data_src0 = actx->data_src0; \
const uint8_t * restrict data_src1 = actx->data_src1; \
uint8_t * restrict data_dst = actx->data_dst; \
\
const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \
const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \
const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \
\
uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \
uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \
uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \
\
size_t src0_spad_half_size = actx->src0_spad_half_size; \
size_t src1_spad_half_size = actx->src1_spad_half_size; \
size_t dst_spad_half_size = actx->dst_spad_half_size; \
\
const int BLOCK = actx->block; \
if (BLOCK == 0) { \
FARF(ERROR, \
OP_STR \
" : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", \
actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \
return; \
} \
\
dma_queue * dma_queue = actx->octx->ctx->dma[ith]; \
\
/* See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 */ \
for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \
const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \
\
/* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \
dma_queue_push_vtcm_to_ddr(dma_queue, \
dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \
dst_row_size, dst_row_size_aligned, 0); \
\
dma_queue_push( \
dma_queue, \
dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \
src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \
dma_queue_push( \
dma_queue, \
dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \
src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \
} \
\
for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \
const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \
\
float * dst_spad = (float *) dma_queue_pop(dma_queue).src; \
float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; \
float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; \
\
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \
CORE_EXPR; \
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \
\
dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), \
dst_row_size, dst_row_size_aligned, block_size); \
\
/* prefetch N+2 loop iteration if any */ \
const uint32_t pref_block = (ir + BLOCK * 2); \
if (pref_block < src0_end_row) { \
const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \
dma_queue_push(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_stride)), \
src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \
dma_queue_push(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_stride)), \
src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \
} \
} \
\
dma_queue_flush(dma_queue); \
\
#define DEFINE_GLU_PER_THREAD(NAME, OP_STR, CORE_EXPR) \
static void glu_##NAME##_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { \
struct htp_act_context * actx = (struct htp_act_context *) data; \
htp_act_preamble; \
\
struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \
\
size_t src0_row_size = actx->src0_row_size; \
size_t src1_row_size = actx->src1_row_size; \
size_t dst_row_size = actx->dst_row_size; \
\
size_t src0_row_stride = actx->src0_row_stride; \
size_t src1_row_stride = actx->src1_row_stride; \
\
const uint32_t src0_nrows = actx->src0_nrows; \
const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \
\
const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith; \
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows); \
\
/* no work for this thread */ \
if (src0_start_row >= src0_end_row) { \
return; \
} \
\
const uint8_t * restrict data_src0 = actx->data_src0; \
const uint8_t * restrict data_src1 = actx->data_src1; \
uint8_t * restrict data_dst = actx->data_dst; \
\
const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \
const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \
const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \
\
uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \
uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \
uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \
\
size_t src0_spad_half_size = actx->src0_spad_half_size; \
size_t src1_spad_half_size = actx->src1_spad_half_size; \
size_t dst_spad_half_size = actx->dst_spad_half_size; \
\
const int BLOCK = actx->block; \
if (BLOCK == 0) { \
FARF(ERROR, \
OP_STR \
" : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", \
actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \
return; \
} \
\
dma_queue * dma_queue = actx->octx->ctx->dma[ith]; \
\
/* See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379 */ \
for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \
const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \
\
/* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \
dma_queue_push_vtcm_to_ddr(dma_queue, \
dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \
dst_row_size, dst_row_size_aligned, 0); \
\
dma_queue_push( \
dma_queue, \
dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \
src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \
dma_queue_push( \
dma_queue, \
dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \
src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \
} \
\
for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \
const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \
\
float * dst_spad = (float *) dma_queue_pop(dma_queue).src; \
float * src0_spad = (float *) dma_queue_pop(dma_queue).dst; \
float * src1_spad = (float *) dma_queue_pop(dma_queue).dst; \
\
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \
CORE_EXPR; \
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \
\
dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), \
dst_row_size, dst_row_size_aligned, block_size); \
\
/* prefetch N+2 loop iteration if any */ \
const uint32_t pref_block = (ir + BLOCK * 2); \
if (pref_block < src0_end_row) { \
const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \
dma_queue_push(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_stride)), \
src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \
dma_queue_push(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_stride)), \
src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \
} \
} \
\
dma_queue_flush(dma_queue); \
\
}
DEFINE_GLU_PER_THREAD(swiglu, "swiglu-f32", swiglu_f32(src0_spad, src1_spad, dst_spad, block_size, actx))
@@ -473,14 +473,30 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
}
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);
const size_t dst_row_size = dst->ne[0] * SIZEOF_FP32;
uint32_t row_start = 0;
uint32_t nrows = src0_nrows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
// row_size = bytes of useful data per row (what the kernel touches / what DMA copies).
// row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src).
const size_t nc_bytes = dst->ne[0] * SIZEOF_FP32;
const size_t src0_row_size = nc_bytes;
const size_t src1_row_size = nc_bytes;
const size_t dst_row_size = nc_bytes;
const size_t nc_bytes = dst_row_size;
const size_t src0_row_size = nc_bytes;
const size_t src1_row_size = nc_bytes;
const size_t src0_row_stride = src0->nb[1];
const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1];
@@ -518,7 +534,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
struct htp_act_context actx;
actx.octx = octx;
actx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
actx.src0_row_size = src0_row_size;
actx.src1_row_size = src1_row_size;
@@ -545,7 +561,8 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
actx.dst_spad_half_size = L.dst_bytes_per_thread / 2;
actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned;
actx.src0_nrows = src0_nrows;
actx.src0_nrows = nrows;
actx.row_start = row_start;
actx.nc = dst->ne[0];
@@ -570,7 +587,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
actx.data_src1 = data_src1;
actx.data_dst = (uint8_t *) dst->data;
worker_pool_run_func(octx->ctx->worker_pool, act_op_func, &actx, n_threads);
work_queue_run(octx->ctx->work_queue, act_op_func, &actx, n_threads);
return HTP_STATUS_OK;
}
ggml/src/ggml-hexagon/htp/allreduce-ops.c
+86 -27
View File
@@ -17,6 +17,7 @@
#include "hex-dma.h"
#include "hex-profile.h"
#include "allreduce-ops.h"
#include "htp-fence.h"
struct htp_allreduce_context {
struct htp_ops_context * octx;
@@ -242,7 +243,42 @@ DEFINE_ALLREDUCE_THREAD_DMA_2D(add_f32, float, hvx_add_f32_aaa, 1, 0)
DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f16, __fp16, hvx_add_f16_aaa, 1, 1)
DEFINE_ALLREDUCE_THREAD_DMA_2D(add_bcast_f32, float, hvx_add_f32_aaa, 1, 1)
static int validate_allreduce(
struct htp_ops_context * octx,
const struct htp_allreduce_kernel_params * kparams,
uint32_t n_ranks
) {
if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
if (kparams->vtcm_size_per_thread <= 0 || kparams->vtcm_size <= 0) {
return HTP_STATUS_INVAL_PARAMS;
}
const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD);
const size_t n_vtcm_buffers = htp_allreduce_vtcm_buffer_count(
n_ranks, octx->n_threads, has_add, kparams->is_row_bcast != 0);
const size_t vtcm_size = n_vtcm_buffers * (size_t) kparams->vtcm_size_per_thread;
if (vtcm_size != (size_t) kparams->vtcm_size) {
return HTP_STATUS_INVAL_PARAMS;
}
if (vtcm_size > octx->ctx->vtcm_size) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
if (octx->dst->type != HTP_TYPE_F16 && octx->dst->type != HTP_TYPE_F32) {
return HTP_STATUS_NO_SUPPORT;
}
return HTP_STATUS_OK;
}
int op_allreduce(struct htp_ops_context * octx) {
if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) {
return HTP_STATUS_OK;
}
const struct htp_allreduce_kernel_params * kparams = (const struct htp_allreduce_kernel_params *) octx->kernel_params;
const struct htp_tensor * dst = octx->dst;
@@ -253,38 +289,53 @@ int op_allreduce(struct htp_ops_context * octx) {
return HTP_STATUS_INVAL_PARAMS;
}
if (dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_F32) {
return HTP_STATUS_NO_SUPPORT;
}
const uint32_t nelem = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3];
const uint32_t fence_seq_entry = (uint32_t) octx->op_params[0];
const uint32_t fence_seq_exit = (uint32_t) octx->op_params[1];
const struct htp_tensor * my_sync = octx->src[n_ranks + rank];
atomic_uint * my_fence = (atomic_uint *) (uintptr_t) my_sync->data;
const int status = validate_allreduce(octx, kparams, n_ranks);
if (status != HTP_STATUS_OK) {
if (status == HTP_STATUS_NO_SUPPORT) {
FARF(ERROR, "ggml-hex: allreduce unsupported type %d : rank %u\n", dst->type, rank);
}
htp_fence_write(my_fence, fence_seq_exit, status);
return status;
}
const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD);
const uint32_t nelem = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3];
// 1. Entry Barrier: Synchronize all ranks before reading
struct htp_thread_trace * tr0 = &octx->ctx->trace[0];
htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry);
const struct htp_tensor * my_sync = octx->src[n_ranks + rank];
atomic_uint * my_fence = (atomic_uint *) my_sync->data;
atomic_store(&my_fence[0], fence_seq_entry);
asm volatile ("syncht" : : : "memory");
Q6_dccleaninva_A((void *) my_fence);
htp_fence_write(my_fence, fence_seq_entry, octx->status);
for (uint32_t j = 0; j < n_ranks; j++) {
if (j == rank) continue;
const struct htp_tensor * peer_sync = octx->src[n_ranks + j];
atomic_uint * peer_fence = (atomic_uint *) peer_sync->data;
atomic_uint * peer_fence = (atomic_uint *) (uintptr_t) peer_sync->data;
uint64_t spins = 0;
while (1) {
Q6_dccleaninva_A((void *) peer_fence);
uint32_t val = atomic_load(&peer_fence[0]);
if (val == fence_seq_entry || val == fence_seq_exit) {
uint32_t peer_seq;
uint32_t peer_status;
htp_fence_read(peer_fence, &peer_seq, &peer_status);
if ((int32_t)(peer_seq - fence_seq_entry) >= 0) {
if (peer_status > HTP_STATUS_OK) {
FARF(ERROR, "ggml-hex: allreduce entry peer %u failed with status %u\n", j, peer_status);
htp_fence_write(my_fence, fence_seq_exit, peer_status);
htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry);
return peer_status;
}
break;
}
if (++spins > HTP_FENCE_TIMEOUT) {
FARF(ERROR, "ggml-hex: allreduce entry fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_entry);
FARF(ERROR, "ggml-hex: allreduce entry fence-wait TIMEOUT : rank %u waiting on %u fence %p seq 0x%x peer-seq 0x%x\n",
rank, j, peer_fence, fence_seq_entry, peer_seq);
htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_INTERNAL_ERR);
htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_entry);
return HTP_STATUS_INTERNAL_ERR;
}
hex_pause();
@@ -301,8 +352,6 @@ int op_allreduce(struct htp_ops_context * octx) {
const uint32_t elems_per_thread = (uint32_t) kparams->elems_per_thread;
const uint32_t vtcm_size_per_thread = (uint32_t) kparams->vtcm_size_per_thread;
const bool has_add = (octx->op == HTP_OP_ALLREDUCE_ADD);
struct htp_allreduce_context actx;
actx.octx = octx;
actx.n_ranks = n_ranks;
@@ -339,6 +388,8 @@ int op_allreduce(struct htp_ops_context * octx) {
}
break;
default:
FARF(ERROR, "ggml-hex: allreduce unsupported kernel %d : rank %u\n", kparams->kernel_type, rank);
htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_NO_SUPPORT);
return HTP_STATUS_NO_SUPPORT;
}
@@ -368,23 +419,31 @@ int op_allreduce(struct htp_ops_context * octx) {
// 4. Exit Barrier: Synchronize all ranks after writing
htp_trace_event_start(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit);
atomic_store(&my_fence[0], fence_seq_exit);
asm volatile ("syncht" : : : "memory");
Q6_dccleaninva_A((void *) my_fence);
htp_fence_write(my_fence, fence_seq_exit, octx->status);
for (uint32_t j = 0; j < n_ranks; j++) {
if (j == rank) continue;
const struct htp_tensor * peer_sync = octx->src[n_ranks + j];
atomic_uint * peer_fence = (atomic_uint *) peer_sync->data;
atomic_uint * peer_fence = (atomic_uint *) (uintptr_t) peer_sync->data;
uint64_t spins = 0;
while (1) {
Q6_dccleaninva_A((void *) peer_fence);
uint32_t val = atomic_load(&peer_fence[0]);
if (val == fence_seq_exit) {
uint32_t peer_seq;
uint32_t peer_status;
htp_fence_read(peer_fence, &peer_seq, &peer_status);
if ((int32_t)(peer_seq - fence_seq_exit) >= 0) {
if (peer_status > HTP_STATUS_OK) {
FARF(ERROR, "ggml-hex: allreduce exit peer %u failed with status %u\n", j, peer_status);
htp_fence_write(my_fence, fence_seq_exit, peer_status);
htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit);
return peer_status;
}
break;
}
if (++spins > HTP_FENCE_TIMEOUT) {
FARF(ERROR, "ggml-hex: allreduce exit fence-wait TIMEOUT: rank %u waiting on %u (fence %p seq %u)\n", rank, j, peer_fence, fence_seq_exit);
FARF(ERROR, "ggml-hex: allreduce exit fence-wait TIMEOUT : rank %u waiting on %u fence %p seq 0x%x peer-seq 0x%x\n",
rank, j, peer_fence, fence_seq_exit, peer_seq);
htp_fence_write(my_fence, fence_seq_exit, HTP_STATUS_INTERNAL_ERR);
htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit);
return HTP_STATUS_INTERNAL_ERR;
}
hex_pause();
@@ -394,5 +453,5 @@ int op_allreduce(struct htp_ops_context * octx) {
htp_trace_event_stop(tr0, HTP_TRACE_EVT_FENCE, (uint16_t) fence_seq_exit);
return HTP_STATUS_OK;
return octx->status;
}
ggml/src/ggml-hexagon/htp/allreduce-ops.h
+11
View File
@@ -2,6 +2,8 @@
#define ALLREDUCE_OPS_H
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#define HTP_ALLREDUCE_MAX_RANKS 4
@@ -15,6 +17,15 @@ enum htp_allreduce_kernel_type {
HTP_ALLREDUCE_KERNEL_DMA_2D,
};
static inline size_t htp_allreduce_vtcm_buffer_count(
uint32_t n_ranks,
uint32_t n_threads,
bool has_add,
bool is_row_bcast
) {
return (size_t) (n_ranks + 1) * n_threads + (has_add ? (is_row_bcast ? 1 : n_threads) : 0);
}
struct htp_allreduce_kernel_params {
int32_t rank;
int32_t n_ranks;
ggml/src/ggml-hexagon/htp/argsort-ops.c
+36 -14
View File
@@ -11,9 +11,10 @@
#include "hvx-utils.h"
#include "hex-dma.h"
#include "hex-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#ifndef MIN
#define MIN(a, b) ((a) < (b) ? (a) : (b))
@@ -22,6 +23,9 @@
struct htp_argsort_context {
struct htp_ops_context * octx;
uint32_t nrows_per_thread;
uint32_t total_rows;
uint32_t row_start;
uint32_t row_end;
uint8_t * vtcm_base;
size_t vtcm_per_thread;
};
@@ -336,10 +340,9 @@ static void htp_argsort_f32_##ne00##_##order_name(unsigned int n, unsigned int i
const struct htp_tensor * src0 = octx->src[0]; \
const struct htp_tensor * dst = octx->dst; \
uint8_t * spad = actx->vtcm_base + actx->vtcm_per_thread * i; \
uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3]; \
uint32_t rows_per_thread = actx->nrows_per_thread; \
uint32_t start_row = rows_per_thread * i; \
uint32_t end_row = MIN(start_row + rows_per_thread, total_rows); \
uint32_t start_row = actx->row_start + rows_per_thread * i; \
uint32_t end_row = MIN(start_row + rows_per_thread, actx->row_end); \
size_t values_size = hex_round_up(ne00 * sizeof(float), 128); \
float * values_buf = (float *) spad; \
int32_t * indices_buf = (int32_t *) (spad + values_size); \
@@ -386,9 +389,6 @@ static void htp_argsort_f32_fallback(unsigned int n, unsigned int i, void * data
// Dimensions
uint32_t ne00 = src0->ne[0];
uint32_t ne01 = src0->ne[1];
uint32_t ne02 = src0->ne[2];
uint32_t ne03 = src0->ne[3];
uint32_t nb01 = src0->nb[1];
@@ -398,10 +398,9 @@ static void htp_argsort_f32_fallback(unsigned int n, unsigned int i, void * data
enum ggml_sort_order order = (enum ggml_sort_order) octx->op_params[0];
// Rows to process
uint32_t total_rows = ne01 * ne02 * ne03;
uint32_t rows_per_thread = actx->nrows_per_thread;
uint32_t start_row = rows_per_thread * i;
uint32_t end_row = MIN(start_row + rows_per_thread, total_rows);
uint32_t start_row = actx->row_start + rows_per_thread * i;
uint32_t end_row = MIN(start_row + rows_per_thread, actx->row_end);
size_t values_size = hex_round_up(ne00 * sizeof(float), 128);
uint32_t num_vec_ind_values = hmx_ceil_div(ne00, VLEN/(sizeof(int32_t)));
@@ -451,8 +450,28 @@ int op_argsort(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
const uint32_t total_rows = octx->src[0]->ne[1] * octx->src[0]->ne[2] * octx->src[0]->ne[3];
const uint32_t n_threads = MIN(total_rows, octx->n_threads);
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const size_t dst_row_size = dst->ne[0] * sizeof(int32_t);
uint32_t row_start = 0;
uint32_t row_end = total_rows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, sizeof(int32_t), (uint32_t) dst_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
row_end = range.start + range.count;
}
const uint32_t nrows = row_end - row_start;
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
// Allocate scratchpad
// We need 1 row of float + 1 row of int32 per thread.
@@ -478,7 +497,10 @@ int op_argsort(struct htp_ops_context * octx) {
struct htp_argsort_context actx;
actx.octx = octx;
actx.nrows_per_thread = (total_rows + n_threads - 1) / n_threads;
actx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
actx.total_rows = nrows;
actx.row_start = row_start;
actx.row_end = row_end;
actx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
actx.vtcm_per_thread = spad_per_thread;
@@ -508,7 +530,7 @@ int op_argsort(struct htp_ops_context * octx) {
}
// Run jobs
worker_pool_run_func(octx->ctx->worker_pool, job_func, &actx, n_threads);
work_queue_run(octx->ctx->work_queue, job_func, &actx, n_threads);
return HTP_STATUS_OK;
}
ggml/src/ggml-hexagon/htp/binary-ops.c
+136 -92
View File
@@ -13,9 +13,10 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
#include "hex-profile.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#ifndef MIN
@@ -36,6 +37,8 @@ struct htp_binary_context {
uint32_t block_max;
uint32_t nrows_per_thread;
uint32_t total_rows;
uint32_t row_start;
size_t src0_row_size_aligned;
size_t src1_row_size_aligned;
size_t dst_row_size_aligned;
@@ -48,27 +51,27 @@ struct htp_binary_context {
const struct htp_tensor * src0 = octx->src[0]; \
const struct htp_tensor * src1 = octx->src[1]; \
const struct htp_tensor * dst = octx->dst; \
\
const uint32_t ne00 = src0->ne[0]; \
const uint32_t ne01 = src0->ne[1]; \
const uint32_t ne02 = src0->ne[2]; \
const uint32_t ne03 = src0->ne[3]; \
\
const uint32_t ne10 = src1->ne[0]; \
const uint32_t ne11 = src1->ne[1]; \
const uint32_t ne12 = src1->ne[2]; \
const uint32_t ne13 = src1->ne[3]; \
\
const uint32_t nb01 = src0->nb[1]; \
const uint32_t nb02 = src0->nb[2]; \
const uint32_t nb03 = src0->nb[3]; \
\
const uint32_t nb11 = src1->nb[1]; \
const uint32_t nb12 = src1->nb[2]; \
const uint32_t nb13 = src1->nb[3]; \
\
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
\
const uint32_t ne00 = src0->ne[0]; \
const uint32_t ne01 = src0->ne[1]; \
const uint32_t ne02 = src0->ne[2]; \
const uint32_t ne03 = src0->ne[3]; \
\
const uint32_t ne10 = src1->ne[0]; \
const uint32_t ne11 = src1->ne[1]; \
const uint32_t ne12 = src1->ne[2]; \
const uint32_t ne13 = src1->ne[3]; \
\
const uint32_t nb01 = src0->nb[1]; \
const uint32_t nb02 = src0->nb[2]; \
const uint32_t nb03 = src0->nb[3]; \
\
const uint32_t nb11 = src1->nb[1]; \
const uint32_t nb12 = src1->nb[2]; \
const uint32_t nb13 = src1->nb[3]; \
\
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3];
static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_t ir, uint32_t end_row, uint32_t ne01, uint32_t ne02) {
@@ -93,87 +96,87 @@ static inline uint32_t calc_block_size(struct htp_binary_context * bctx, uint32_
}
// Macro for scalar op switch
#define COMPUTE_SCALAR_OP(DST, SRC, VAL, TYPE, N) \
if(TYPE == HTP_TYPE_F32) { \
switch (octx->op) { \
case HTP_OP_ADD: hvx_add_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \
case HTP_OP_SUB: hvx_sub_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \
case HTP_OP_MUL: hvx_mul_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \
#define COMPUTE_SCALAR_OP(DST, SRC, VAL, TYPE, N) \
if(TYPE == HTP_TYPE_F32) { \
switch (octx->op) { \
case HTP_OP_ADD: hvx_add_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \
case HTP_OP_SUB: hvx_sub_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \
case HTP_OP_MUL: hvx_mul_scalar_f32_aa(DST, SRC, *(float *)VAL, N); break; \
case HTP_OP_DIV: hvx_mul_scalar_f32_aa(DST, SRC, 1.0f / (*(float *)VAL), N); break; \
default: break; \
} \
} \
else { \
switch (octx->op) { \
case HTP_OP_ADD: hvx_add_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \
case HTP_OP_SUB: hvx_sub_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \
case HTP_OP_MUL: hvx_mul_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \
case HTP_OP_DIV: hvx_div_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \
default: break; \
} \
default: break; \
} \
} \
else { \
switch (octx->op) { \
case HTP_OP_ADD: hvx_add_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \
case HTP_OP_SUB: hvx_sub_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \
case HTP_OP_MUL: hvx_mul_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \
case HTP_OP_DIV: hvx_div_scalar_f16_aa(DST, SRC, *(_Float16 *)VAL, N); break; \
default: break; \
} \
}
// Macro for vector op switch (All Aligned)
#define COMPUTE_VECTOR_OP_AAA(DST, SRC0, SRC1, TYPE, N) \
if(TYPE == HTP_TYPE_F32) { \
switch (octx->op) { \
#define COMPUTE_VECTOR_OP_AAA(DST, SRC0, SRC1, TYPE, N) \
if(TYPE == HTP_TYPE_F32) { \
switch (octx->op) { \
case HTP_OP_ADD: hvx_add_f32_aaa(DST, SRC0, SRC1, N); break; \
case HTP_OP_SUB: hvx_sub_f32_aaa(DST, SRC0, SRC1, N); break; \
case HTP_OP_MUL: hvx_mul_f32_aaa(DST, SRC0, SRC1, N); break; \
case HTP_OP_DIV: hvx_div_f32_aaa(DST, SRC0, SRC1, N); break; \
default: break; \
} \
} \
else { \
switch (octx->op) { \
default: break; \
} \
} \
else { \
switch (octx->op) { \
case HTP_OP_ADD: hvx_add_f16_aaa(DST, SRC0, SRC1, N); break; \
case HTP_OP_SUB: hvx_sub_f16_aaa(DST, SRC0, SRC1, N); break; \
case HTP_OP_MUL: hvx_mul_f16_aaa(DST, SRC0, SRC1, N); break; \
case HTP_OP_DIV: hvx_div_f16_aaa(DST, SRC0, SRC1, N); break; \
default: break; \
} \
default: break; \
} \
}
// Macro for vector op switch (Dst Aligned, Src0 Aligned, Src1 Unaligned)
#define COMPUTE_VECTOR_OP_AAU(DST, SRC0, SRC1, TYPE, N) \
if(TYPE == HTP_TYPE_F32) { \
switch (octx->op) { \
#define COMPUTE_VECTOR_OP_AAU(DST, SRC0, SRC1, TYPE, N) \
if(TYPE == HTP_TYPE_F32) { \
switch (octx->op) { \
case HTP_OP_ADD: hvx_add_f32_aau(DST, SRC0, SRC1, N); break; \
case HTP_OP_SUB: hvx_sub_f32_aau(DST, SRC0, SRC1, N); break; \
case HTP_OP_MUL: hvx_mul_f32_aau(DST, SRC0, SRC1, N); break; \
case HTP_OP_DIV: hvx_div_f32_aau(DST, SRC0, SRC1, N); break; \
default: break; \
} \
} \
else { \
switch (octx->op) { \
default: break; \
} \
} \
else { \
switch (octx->op) { \
case HTP_OP_ADD: hvx_add_f16_aau(DST, SRC0, SRC1, N); break; \
case HTP_OP_SUB: hvx_sub_f16_aau(DST, SRC0, SRC1, N); break; \
case HTP_OP_MUL: hvx_mul_f16_aau(DST, SRC0, SRC1, N); break; \
case HTP_OP_DIV: hvx_div_f16_aau(DST, SRC0, SRC1, N); break; \
default: break; \
} \
default: break; \
} \
}
// Macro for vector op switch (All Unaligned - generic loop used in element repeat)
#define COMPUTE_VECTOR_OP_UUU(DST, SRC0, SRC1, TYPE, N) \
if(TYPE == HTP_TYPE_F32) { \
switch (octx->op) { \
#define COMPUTE_VECTOR_OP_UUU(DST, SRC0, SRC1, TYPE, N) \
if(TYPE == HTP_TYPE_F32) { \
switch (octx->op) { \
case HTP_OP_ADD: hvx_add_f32_uuu(DST, SRC0, SRC1, N); break; \
case HTP_OP_SUB: hvx_sub_f32_uuu(DST, SRC0, SRC1, N); break; \
case HTP_OP_MUL: hvx_mul_f32_uuu(DST, SRC0, SRC1, N); break; \
case HTP_OP_DIV: hvx_div_f32_uuu(DST, SRC0, SRC1, N); break; \
default: break; \
} \
} \
else { \
switch (octx->op) { \
default: break; \
} \
} \
else { \
switch (octx->op) { \
case HTP_OP_ADD: hvx_add_f16_uuu(DST, SRC0, SRC1, N); break; \
case HTP_OP_SUB: hvx_sub_f16_uuu(DST, SRC0, SRC1, N); break; \
case HTP_OP_MUL: hvx_mul_f16_uuu(DST, SRC0, SRC1, N); break; \
case HTP_OP_DIV: hvx_div_f16_uuu(DST, SRC0, SRC1, N); break; \
default: break; \
} \
default: break; \
} \
}
// 1. Scalar src1 (ne10 == 1)
@@ -184,9 +187,8 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) {
const uint32_t src0_type = octx->src[0]->type;
const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16);
const uint32_t total_rows = ne01 * ne02 * ne03;
const uint32_t start_row = bctx->nrows_per_thread * ith;
const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows);
const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
if (start_row >= end_row) return;
FARF(HIGH, "binary-scalar: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);
@@ -222,6 +224,8 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) {
}
// Main loop
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ir = start_row; ir < end_row; ) {
uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);
@@ -242,12 +246,14 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) {
uint8_t * src1_ptr = (uint8_t *)src1->data + i13 * nb13 + i12 * nb12 + i11 * nb11;
uint32_t s1_stride = (ne11 == 1) ? 0 : nb11;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
for (uint32_t r = 0; r < current_block_size; r++) {
uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned;
uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned;
COMPUTE_SCALAR_OP(r_dst, r_src0, src1_ptr, src0_type, ne00);
src1_ptr += s1_stride;
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;
dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size);
@@ -266,6 +272,7 @@ static void binary_job_scalar(unsigned int nth, unsigned int ith, void * data) {
}
ir += current_block_size;
}
dma_queue_flush(q);
}
@@ -277,9 +284,8 @@ static void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, voi
const uint32_t src0_type = octx->src[0]->type;
const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16);
const uint32_t total_rows = ne01 * ne02 * ne03;
const uint32_t start_row = bctx->nrows_per_thread * ith;
const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows);
const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
if (start_row >= end_row) return;
FARF(HIGH, "binary-same-shape: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);
@@ -323,18 +329,22 @@ static void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, voi
spad_idx ^= 1;
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ir = start_row; ir < end_row; ) {
uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);
uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src;
uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst;
uint8_t * s1_spad = (uint8_t *) dma_queue_pop(q).dst;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
for (uint32_t r = 0; r < current_block_size; r++) {
uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned;
uint8_t * r_src1 = s1_spad + r * bctx->src1_row_size_aligned;
uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned;
COMPUTE_VECTOR_OP_AAA(r_dst, r_src0, r_src1, src0_type, ne00);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
uint32_t i03, i02, i01, rem;
i03 = fastdiv(ir, &bctx->src0_dim12_div);
@@ -366,6 +376,7 @@ static void binary_job_vector_same_shape(unsigned int nth, unsigned int ith, voi
}
ir += current_block_size;
}
dma_queue_flush(q);
}
@@ -377,9 +388,8 @@ static void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith,
const uint32_t src0_type = octx->src[0]->type;
const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16);
const uint32_t total_rows = ne01 * ne02 * ne03;
const uint32_t start_row = bctx->nrows_per_thread * ith;
const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows);
const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
if (start_row >= end_row) return;
FARF(HIGH, "binary-row-bcast: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);
@@ -416,17 +426,21 @@ static void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith,
spad_idx ^= 1;
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ir = start_row; ir < end_row; ) {
uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);
uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src;
uint8_t * s0_spad = (uint8_t *) dma_queue_pop(q).dst;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
for (uint32_t r = 0; r < current_block_size; r++) {
uint8_t * r_src0 = s0_spad + r * bctx->src0_row_size_aligned;
uint8_t * r_src1 = (uint8_t *)s1_ptr; // Constant
uint8_t * r_dst = d_spad + r * bctx->dst_row_size_aligned;
COMPUTE_VECTOR_OP_AAA(r_dst, r_src0, r_src1, src0_type, ne00);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
uint32_t i03 = fastdiv(ir, &bctx->src0_dim12_div);
uint32_t rem = ir - i03 * (ne02 * ne01);
@@ -447,6 +461,7 @@ static void binary_job_vector_row_broadcast(unsigned int nth, unsigned int ith,
}
ir += current_block_size;
}
dma_queue_flush(q);
}
@@ -458,9 +473,8 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void *
const uint32_t src0_type = octx->src[0]->type;
const uint32_t row_size_bytes = (src0_type == HTP_TYPE_F32) ? ne00 * sizeof(float) : ne00 * sizeof(_Float16);
const uint32_t total_rows = ne01 * ne02 * ne03;
const uint32_t start_row = bctx->nrows_per_thread * ith;
const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows);
const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
if (start_row >= end_row) return;
FARF(HIGH, "binary-complex: %d/%d (%u:%u) row-size %u (%u)", ith, nth, start_row, end_row, nb01, bctx->dst_row_size_aligned);
@@ -493,6 +507,8 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void *
spad_idx ^= 1;
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ir = start_row; ir < end_row; ) {
uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);
uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src;
@@ -503,6 +519,7 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void *
uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div);
uint32_t i01 = rem - i02 * ne01;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
for (uint32_t r = 0; r < current_block_size; r++) {
uint32_t r_i01 = i01 + r;
uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div);
@@ -516,6 +533,7 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void *
// Read src1 from DDR (unaligned)
COMPUTE_VECTOR_OP_AAU(r_dst, r_src0, r_src1, src0_type, ne00);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;
dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size);
@@ -532,6 +550,7 @@ static void binary_job_vector_complex(unsigned int nth, unsigned int ith, void *
}
ir += current_block_size;
}
dma_queue_flush(q);
}
@@ -544,9 +563,8 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void *
const uint32_t src0_type = octx->src[0]->type;
const uint32_t elem_size_bytes = (src0_type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16);
const uint32_t row_size_bytes = ne00 * elem_size_bytes;;
const uint32_t total_rows = ne01 * ne02 * ne03;
const uint32_t start_row = bctx->nrows_per_thread * ith;
const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows);
const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
if (start_row >= end_row) return;
uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);
@@ -579,6 +597,8 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void *
spad_idx ^= 1;
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ir = start_row; ir < end_row; ) {
uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);
uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src;
@@ -589,6 +609,7 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void *
uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div);
uint32_t i01 = rem - i02 * ne01;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
for (uint32_t r = 0; r < current_block_size; r++) {
uint32_t r_i01 = i01 + r;
uint32_t i13 = fastmodulo(i03, ne13, &bctx->src1_dim3_div);
@@ -606,6 +627,7 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void *
COMPUTE_VECTOR_OP_UUU(r_dst + c * elem_size_bytes, r_src0 + c * elem_size_bytes, r_src1_row, src0_type, len);
}
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;
dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, row_size_bytes, current_block_size);
@@ -622,6 +644,7 @@ static void binary_job_element_repeat(unsigned int nth, unsigned int ith, void *
}
ir += current_block_size;
}
dma_queue_flush(q);
}
@@ -650,9 +673,8 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) {
const uint32_t nb2 = dst->nb[2];
const uint32_t nb3 = dst->nb[3];
const uint32_t total_rows = ne01 * ne02 * ne03;
const uint32_t start_row = bctx->nrows_per_thread * ith;
const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, total_rows);
const uint32_t start_row = bctx->row_start + bctx->nrows_per_thread * ith;
const uint32_t end_row = MIN(start_row + bctx->nrows_per_thread, bctx->row_start + bctx->total_rows);
if (start_row >= end_row) return;
uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread);
@@ -683,6 +705,8 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) {
spad_idx ^= 1;
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ir = start_row; ir < end_row; ) {
uint32_t current_block_size = calc_block_size(bctx, ir, end_row, ne01, ne02);
uint8_t * d_spad = (uint8_t *) dma_queue_pop(q).src;
@@ -693,6 +717,7 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) {
uint32_t i02 = fastdiv(rem, &bctx->src0_dim1_div);
uint32_t i01 = rem - i02 * ne01;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
for (uint32_t r = 0; r < current_block_size; r++) {
uint32_t r_i01 = i01 + r; // linear within block since we split at ne01
@@ -704,6 +729,7 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) {
hvx_add_f32_aau(r_dst, r_src0, r_src1, ne00);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
uint8_t * dst_curr = (uint8_t *)dst->data + i03 * nb3 + i02 * nb2 + i01 * nb1;
dma_queue_push(q, dma_make_ptr(dst_curr, d_spad), nb1, bctx->dst_row_size_aligned, ne00 * sizeof(float), current_block_size);
@@ -720,6 +746,7 @@ static void binary_job_add_id(unsigned int nth, unsigned int ith, void * data) {
}
ir += current_block_size;
}
dma_queue_flush(q);
}
@@ -729,15 +756,31 @@ static int execute_op_binary(struct htp_ops_context * octx) {
const struct htp_tensor * dst = octx->dst;
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);
// Use packed row sizes for VTCM allocation
// Use packed row sizes for VTCM allocation and alignment
const uint32_t src0_type = octx->src[0]->type;
const size_t elem_size = (src0_type == HTP_TYPE_F32) ? sizeof(float) : sizeof(_Float16);
const size_t src0_row_size = src0->ne[0] * elem_size;
const size_t src1_row_size = src1->ne[0] * elem_size;
const size_t dst_row_size = dst->ne[0] * elem_size;
uint32_t row_start = 0;
uint32_t nrows = src0_nrows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);
size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);
size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);
@@ -815,7 +858,9 @@ static int execute_op_binary(struct htp_ops_context * octx) {
struct htp_binary_context bctx;
bctx.octx = octx;
bctx.nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
bctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
bctx.total_rows = nrows;
bctx.row_start = row_start;
bctx.block_max = rows_per_buffer;
bctx.src0_row_size_aligned = src0_row_size_aligned;
bctx.src1_row_size_aligned = src1_row_size_aligned;
@@ -850,7 +895,7 @@ static int execute_op_binary(struct htp_ops_context * octx) {
dma_queue_pop(q);
}
worker_pool_run_func(octx->ctx->worker_pool, worker_func, &bctx, n_threads);
work_queue_run(octx->ctx->work_queue, worker_func, &bctx, n_threads);
return HTP_STATUS_OK;
}
@@ -870,4 +915,3 @@ int op_binary(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
ggml/src/ggml-hexagon/htp/concat-ops.c
+66 -15
View File
@@ -1,5 +1,8 @@
#include "hex-common.h"
#include "hex-profile.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hexagon_types.h"
#include "hexagon_protos.h"
#include "hvx_hexagon_protos.h"
@@ -13,6 +16,10 @@ struct htp_concat_context {
struct htp_ops_context * octx;
uint32_t dim;
uint32_t nrows_per_thread;
uint32_t row_start;
uint32_t nrows;
uint32_t elem_start;
uint32_t nelems;
struct fastdiv_values div_ne0;
struct fastdiv_values div_ne1;
struct fastdiv_values div_ne2;
@@ -28,10 +35,10 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void *
const uint32_t src0_ne0 = src0->ne[0];
const uint32_t src1_ne0 = src1->ne[0];
const uint32_t ne1 = dst->ne[1];
const uint32_t start_i = ith * cctx->nrows_per_thread;
const uint32_t end_i = (start_i + cctx->nrows_per_thread < ne1) ? (start_i + cctx->nrows_per_thread) : ne1;
const uint32_t row_end = cctx->row_start + cctx->nrows;
const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread;
const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end;
if (start_i >= end_i) return;
dma_queue * q = octx->ctx->dma[ith];
@@ -51,6 +58,8 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void *
const uint32_t spad0_row_bytes = hex_round_up((src0_ne0 + src1_ne0_padded) * sizeof(float), VLEN);
uint32_t mu = src1_ne0_padded * spad1_stride;
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t i = start_i; i < end_i; i += block_i) {
uint32_t current_block_i = (end_i - i < block_i) ? (end_i - i) : block_i;
@@ -66,6 +75,7 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void *
HVX_Vector * vtcm_tmp = (HVX_Vector *)(spad1_base + src1_ne0_padded * spad1_stride);
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
for (uint32_t j = 0; j < src1_ne0_padded; j += 32) {
#pragma unroll(4)
for (uint32_t ii = 0; ii < current_block_i; ii++) {
@@ -75,6 +85,7 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void *
hvx_vmemu(dst_ptr) = vtcm_tmp[ii];
}
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
dma_queue_pop(q); // src0
@@ -95,10 +106,10 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void *
const uint32_t src0_ne0 = src0->ne[0];
const uint32_t src1_ne0 = src1->ne[0];
const uint32_t ne1 = dst->ne[1];
const uint32_t start_i = ith * cctx->nrows_per_thread;
const uint32_t end_i = (start_i + cctx->nrows_per_thread < ne1) ? (start_i + cctx->nrows_per_thread) : ne1;
const uint32_t row_end = cctx->row_start + cctx->nrows;
const uint32_t start_i = cctx->row_start + ith * cctx->nrows_per_thread;
const uint32_t end_i = (start_i + cctx->nrows_per_thread < row_end) ? (start_i + cctx->nrows_per_thread) : row_end;
if (start_i >= end_i) return;
dma_queue * q = octx->ctx->dma[ith];
@@ -118,6 +129,8 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void *
const uint32_t spad0_row_bytes = hex_round_up((src0_ne0 + src1_ne0_padded) * sizeof(__fp16), VLEN);
uint32_t mu = src1_ne0_padded * spad1_stride;
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t i = start_i; i < end_i; i += block_i) {
uint32_t current_block_i = (end_i - i < block_i) ? (end_i - i) : block_i;
@@ -133,6 +146,7 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void *
HVX_Vector * vtcm_tmp = (HVX_Vector *)(spad1_base + src1_ne0_padded * spad1_stride);
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
for (uint32_t j = 0; j < src1_ne0_padded; j += 64) {
#pragma unroll(4)
for (uint32_t ii = 0; ii < current_block_i; ii++) {
@@ -142,6 +156,7 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void *
hvx_vmemu(dst_ptr) = vtcm_tmp[ii];
}
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
dma_queue_pop(q); // src0
@@ -164,11 +179,14 @@ static void concat_generic(unsigned int nth, unsigned int ith, void * data) {
const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
const uint32_t ne[4] = {dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]};
const uint32_t total_elements = ne[0] * ne[1] * ne[2] * ne[3];
const uint32_t chunk_size = (total_elements + nth - 1) / nth;
const uint32_t start_idx = MIN(ith * chunk_size, total_elements);
const uint32_t end_idx = MIN(start_idx + chunk_size, total_elements);
// Per-device element range aligned to prevent false sharing
const uint32_t elem_start = cctx->elem_start;
const uint32_t nelems = cctx->nelems;
const uint32_t chunk_size = (nelems + nth - 1) / nth;
const uint32_t start_idx = MIN(elem_start + ith * chunk_size, elem_start + nelems);
const uint32_t end_idx = MIN(start_idx + chunk_size, elem_start + nelems);
// Naive scalar element-wise copy
for (uint32_t idx = start_idx; idx < end_idx; idx++) {
@@ -236,13 +254,28 @@ int op_concat(struct htp_ops_context * octx) {
void (*worker_func)(unsigned int, unsigned int, void *) = concat_generic;
if (dim == 0 && is_2d && is_src1_transposed && !is_src0_transposed) {
n_threads = MIN(dst->ne[1], n_threads);
if (n_threads < 1) {
n_threads = 1;
const uint32_t total_rows = dst->ne[1];
const size_t dst_data_row_size = dst->ne[0] * type_size;
uint32_t row_start = 0;
uint32_t nrows = total_rows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_data_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
cctx.row_start = row_start;
cctx.nrows = nrows;
uint32_t block_i = (type_size == 4) ? 32 : 64;
cctx.nrows_per_thread = hmx_ceil_div(dst->ne[1], n_threads);
cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
// Allocate VTCM
uint32_t spad1_stride = block_i * type_size;
@@ -270,8 +303,26 @@ int op_concat(struct htp_ops_context * octx) {
} else {
worker_func = concat_2d_f16_transposed;
}
} else {
const uint32_t total_elements = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3];
uint32_t elem_start = 0;
uint32_t nelems = total_elements;
if (octx->ctx->mdev.count > 1) {
const uint32_t elems_per_chunk = HEX_L2_LINE_SIZE / type_size;
const bool can_split = htp_tensor_mdev_data_aligned(dst) && htp_tensor_is_contiguous(dst, type_size) && !htp_tensor_is_permuted(dst);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_elements, can_split ? elems_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
elem_start = range.start;
nelems = range.count;
}
if (nelems == 0) {
return HTP_STATUS_OK;
}
cctx.elem_start = elem_start;
cctx.nelems = nelems;
}
worker_pool_run_func(octx->ctx->worker_pool, worker_func, &cctx, n_threads);
work_queue_run(octx->ctx->work_queue, worker_func, &cctx, n_threads);
return HTP_STATUS_OK;
}
ggml/src/ggml-hexagon/htp/cpy-ops.c
+279 -197
View File
@@ -16,6 +16,7 @@
#include "htp-ops.h"
#include "hvx-utils.h"
#include "htp-tensor.h"
#include "htp-fence.h"
struct htp_copy_context {
struct htp_ops_context * octx;
@@ -29,7 +30,23 @@ struct htp_copy_context {
uint32_t src0_blocks_per_row;
uint32_t dst_blocks_per_row;
uint32_t elem_start;
uint32_t nelem;
uint32_t elem_per_thread;
uint32_t src0_nrows_per_thread;
uint32_t row_start;
uint32_t nrows;
struct fastdiv_values div_ne01;
struct fastdiv_values div_ne02_ne01;
struct fastdiv_values div_ne0;
struct fastdiv_values div_ne1_ne0;
struct fastdiv_values div_ne2_ne1_ne0;
struct fastdiv_values div_ne00;
struct fastdiv_values div_ne01_ne00;
struct fastdiv_values div_ne02_ne01_ne00;
};
#define cpy_preamble \
@@ -54,131 +71,113 @@ struct htp_copy_context {
const uint32_t nb0 = dst->nb[0]; \
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3]; \
\
const uint32_t nr = ne01;
const uint32_t nb3 = dst->nb[3];
#define DEFINE_CPY_SAMESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \
static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
struct htp_ops_context * octx = ct->octx; \
cpy_preamble; \
const uint32_t dr = ct->src0_nrows_per_thread; \
const uint32_t ir0 = dr * ith; \
const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; \
if (ir0 >= nr) return; \
for (uint32_t i03 = 0; i03 < ne03; i03++) { \
for (uint32_t i02 = 0; i02 < ne02; i02++) { \
_Pragma("unroll(4)") \
for (uint32_t i01 = ir0; i01 < ir1; i01++) { \
uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3; \
uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \
hex_l2fetch(src0_ptr, ne00 * ELEM_SIZE, nb01, 2); \
hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \
} \
} \
} \
#define DEFINE_CPY_SAMESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \
static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
struct htp_ops_context * octx = ct->octx; \
cpy_preamble; \
const uint32_t dr = ct->src0_nrows_per_thread; \
const uint32_t ir0 = ct->row_start + dr * ith; \
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \
if (ir0 >= ir1) return; \
const bool contiguous = (nb01 == ne00 * ELEM_SIZE) && (nb1 == nb01) && \
(nb02 == ne01 * nb01) && (nb2 == nb02) && \
(nb03 == ne02 * nb02) && (nb3 == nb03); \
const uint32_t ne02_ne01 = ne02 * ne01; \
uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); \
uint32_t rem = ir0 - i03 * ne02_ne01; \
uint32_t i02 = fastdiv(rem, &ct->div_ne01); \
uint32_t i01 = rem - i02 * ne01; \
uint8_t * dst_ptr = (uint8_t *) dst->data + i01*nb1 + i02*nb2 + i03*nb3; \
uint8_t * src0_ptr = (uint8_t *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \
if (contiguous) { \
hvx_copy_uu(dst_ptr, src0_ptr, (ir1 - ir0) * ne00, ELEM_SIZE); \
return; \
} \
for (uint32_t r = ir0; r < ir1; r++) { \
hex_l2fetch(src0_ptr, ne00 * ELEM_SIZE, nb01, 2); \
hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \
dst_ptr += nb1; \
src0_ptr += nb01; \
if (++i01 == ne01) { \
i01 = 0; \
if (++i02 == ne02) { \
i02 = 0; \
i03++; \
} \
dst_ptr = (uint8_t *) dst->data + i02*nb2 + i03*nb3; \
src0_ptr = (uint8_t *) src0->data + i02*nb02 + i03*nb03; \
} \
} \
}
DEFINE_CPY_SAMESHAPE(f32, float, 4)
DEFINE_CPY_SAMESHAPE(f16, __fp16, 2)
#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \
static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
struct htp_ops_context * octx = ct->octx; \
cpy_preamble; \
const uint32_t dr = ct->src0_nrows_per_thread; \
const uint32_t ir0 = dr * ith; \
const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr; \
if (ir0 >= nr) return; \
const bool src0_contig = (nb00 == ELEM_SIZE) && \
(nb01 == ne00 * nb00) && \
(nb02 == ne01 * nb01) && \
(nb03 == ne02 * nb02); \
const bool dst_contig = (nb0 == ELEM_SIZE) && \
(nb1 == ne0 * nb0) && \
(nb2 == ne1 * nb1) && \
(nb3 == ne2 * nb2); \
if (src0_contig && dst_contig) { \
for (int64_t i03 = 0; i03 < ne03; i03++) { \
for (int64_t i02 = 0; i02 < ne02; i02++) { \
uint8_t * src_ptr = (uint8_t *) src0->data + i03*nb03 + i02*nb02 + ir0*nb01; \
uint32_t flat = ((i03*ne02 + i02)*ne01 + ir0) * ne00; \
uint8_t * dst_ptr = (uint8_t *) dst->data + flat * ELEM_SIZE; \
hvx_copy_uu(dst_ptr, src_ptr, (ir1 - ir0) * ne00, ELEM_SIZE); \
} \
} \
return; \
} \
const bool reshape_flat_fast = (ne03 == 1 && ne2 == 1 && ne3 == 1) && \
(ne0 == ne00 * ne01) && (ne1 == ne02) && \
(nb00 == ELEM_SIZE) && (nb0 == ELEM_SIZE); \
if (reshape_flat_fast) { \
for (uint32_t i02 = 0; i02 < ne02; i02++) { \
for (uint32_t i01 = ir0; i01 < ir1; i01++) { \
uint8_t * src0_ptr = (uint8_t *) src0->data + i01 * nb01 + i02 * nb02; \
uint8_t * dst_ptr = (uint8_t *) dst->data + i01 * ne00 * ELEM_SIZE + i02 * nb1; \
hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \
} \
} \
return; \
} \
int64_t k10 = 0; \
int64_t i11 = 0; \
int64_t i12 = 0; \
int64_t i13 = 0; \
const int64_t nk00 = ct->src0_blocks_per_row; \
const int64_t nk0 = ct->dst_blocks_per_row; \
for (int64_t i03 = 0; i03 < ne03; i03++) { \
for (int64_t i02 = 0; i02 < ne02; i02++) { \
k10 += nk00 * ir0; \
while (k10 >= nk0) { \
k10 -= nk0; \
if (++i11 == ne1) { \
i11 = 0; \
if (++i12 == ne2) { \
i12 = 0; \
if (++i13 == ne3) { \
i13 = 0; \
} \
} \
} \
} \
for (int64_t i01 = ir0; i01 < ir1; i01++) { \
for (int64_t k00 = 0; k00 < nk00; k00++) { \
const char * src0_ptr = ((char *) src0->data + k00*nb00 + i01*nb01 + i02*nb02 + i03*nb03); \
char * dst_ptr = ((char *) dst->data + k10*nb0 + i11*nb1 + i12*nb2 + i13*nb3); \
memcpy(dst_ptr, src0_ptr, ELEM_SIZE); \
if (++k10 == nk0) { \
k10 = 0; \
if (++i11 == ne1) { \
i11 = 0; \
if (++i12 == ne2) { \
i12 = 0; \
if (++i13 == ne3) { \
i13 = 0; \
} \
} \
} \
} \
} \
} \
k10 += nk00 * (ne01 - ir1); \
while (k10 >= nk0) { \
k10 -= nk0; \
if (++i11 == ne1) { \
i11 = 0; \
if (++i12 == ne2) { \
i12 = 0; \
if (++i13 == ne3) { \
i13 = 0; \
} \
} \
} \
} \
} \
} \
#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \
static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
struct htp_ops_context * octx = ct->octx; \
cpy_preamble; \
const uint32_t th_nelem = ct->elem_per_thread; \
const uint32_t th_start = ct->elem_start + ith * th_nelem; \
const uint32_t th_end = MIN(th_start + th_nelem, ct->elem_start + ct->nelem); \
if (th_start >= th_end) return; \
\
const uint32_t ne01_ne00 = ne01 * ne00; \
const uint32_t ne02_ne01_ne00 = ne02 * ne01_ne00; \
const uint32_t ne1_ne0 = ne1 * ne0; \
const uint32_t ne2_ne1_ne0 = ne2 * ne1_ne0; \
\
uint32_t e = th_start; \
uint32_t i13 = fastdiv(e, &ct->div_ne2_ne1_ne0); \
uint32_t rem = e - i13 * ne2_ne1_ne0; \
uint32_t i12 = fastdiv(rem, &ct->div_ne1_ne0); \
uint32_t rem2 = rem - i12 * ne1_ne0; \
uint32_t i11 = fastdiv(rem2, &ct->div_ne0); \
uint32_t i10 = rem2 - i11 * ne0; \
\
uint32_t i03 = fastdiv(e, &ct->div_ne02_ne01_ne00); \
uint32_t rem_s = e - i03 * ne02_ne01_ne00; \
uint32_t i02 = fastdiv(rem_s, &ct->div_ne01_ne00); \
uint32_t rem2_s = rem_s - i02 * ne01_ne00; \
uint32_t i01 = fastdiv(rem2_s, &ct->div_ne00); \
uint32_t i00 = rem2_s - i01 * ne00; \
\
char * dst_ptr = (char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3; \
const char * src0_ptr = (const char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03; \
\
for (; e < th_end; e++) { \
*((ELEM_TYPE *) dst_ptr) = *((const ELEM_TYPE *) src0_ptr); \
\
dst_ptr += nb0; \
if (++i10 == ne0) { \
i10 = 0; \
if (++i11 == ne1) { \
i11 = 0; \
if (++i12 == ne2) { \
i12 = 0; \
i13++; \
} \
} \
dst_ptr = (char *) dst->data + i11*nb1 + i12*nb2 + i13*nb3; \
} \
\
src0_ptr += nb00; \
if (++i00 == ne00) { \
i00 = 0; \
if (++i01 == ne01) { \
i01 = 0; \
if (++i02 == ne02) { \
i02 = 0; \
i03++; \
} \
} \
src0_ptr = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \
} \
} \
}
DEFINE_CPY_RESHAPE(f32, float, 4)
@@ -189,22 +188,33 @@ static void cpy_thread_f16_f32_sameshape(unsigned int nth, unsigned int ith, voi
struct htp_ops_context * octx = ct->octx;
cpy_preamble;
// parallelize by src0 rows
const uint32_t dr = ct->src0_nrows_per_thread;
const uint32_t ir0 = dr * ith;
const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr;
if (ir0 >= nr) return;
const uint32_t ir0 = ct->row_start + dr * ith;
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows);
if (ir0 >= ir1) return;
// copy by rows
for (uint32_t i03 = 0; i03 < ne03; i03++) {
for (uint32_t i02 = 0; i02 < ne02; i02++) {
#pragma unroll(2)
for (uint32_t i01 = ir0; i01 < ir1; i01++) {
uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3;
uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2);
hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00);
const uint32_t ne02_ne01 = ne02 * ne01;
uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01);
uint32_t rem = ir0 - i03 * ne02_ne01;
uint32_t i02 = fastdiv(rem, &ct->div_ne01);
uint32_t i01 = rem - i02 * ne01;
uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3;
uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
for (uint32_t r = ir0; r < ir1; r++) {
hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2);
hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00);
dst_ptr += nb1;
src0_ptr += nb01;
if (++i01 == ne01) {
i01 = 0;
if (++i02 == ne02) {
i02 = 0;
i03++;
}
dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3;
src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03;
}
}
}
@@ -214,22 +224,33 @@ static void cpy_thread_f32_f16_sameshape(unsigned int nth, unsigned int ith, voi
struct htp_ops_context * octx = ct->octx;
cpy_preamble;
// parallelize by src0 rows
const uint32_t dr = ct->src0_nrows_per_thread;
const uint32_t ir0 = dr * ith;
const uint32_t ir1 = (ir0 + dr) < nr ? (ir0 + dr) : nr;
if (ir0 >= nr) return;
const uint32_t ir0 = ct->row_start + dr * ith;
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows);
if (ir0 >= ir1) return;
// copy by rows
for (uint32_t i03 = 0; i03 < ne03; i03++) {
for (uint32_t i02 = 0; i02 < ne02; i02++) {
#pragma unroll(2)
for (uint32_t i01 = ir0; i01 < ir1; i01++) {
uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3;
uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
hex_l2fetch(src0_ptr, ne00 * sizeof(__fp16), nb01, 2);
hvx_copy_f32_f16_uu(dst_ptr, src0_ptr, ne00);
const uint32_t ne02_ne01 = ne02 * ne01;
uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01);
uint32_t rem = ir0 - i03 * ne02_ne01;
uint32_t i02 = fastdiv(rem, &ct->div_ne01);
uint32_t i01 = rem - i02 * ne01;
uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3;
uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
for (uint32_t r = ir0; r < ir1; r++) {
hex_l2fetch(src0_ptr, ne00 * sizeof(__fp16), nb01, 2);
hvx_copy_f32_f16_uu(dst_ptr, src0_ptr, ne00);
dst_ptr += nb1;
src0_ptr += nb01;
if (++i01 == ne01) {
i01 = 0;
if (++i02 == ne02) {
i02 = 0;
i03++;
}
dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3;
src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03;
}
}
}
@@ -250,15 +271,19 @@ static inline void cpy_dma_sametype_sameshape(
dma_queue * q = octx->ctx->dma[0];
if (contiguous_outer) {
dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03);
dma_queue_pop(q);
if (!dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03)) {
dma_queue_flush(q);
dma_queue_push(q, dma_make_ptr((void *) dst->data, (const void *) src0->data), nb1, nb01, ne00 * elem_size, ne01 * ne02 * ne03);
}
dma_queue_flush(q);
return;
}
for (uint32_t i03 = 0; i03 < ne03; i03++) {
for (uint32_t i02 = 0; i02 < ne02; i02++) {
uint8_t* dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3;
uint8_t* src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03;
uint8_t * dst_ptr = (uint8_t *) dst->data + i02 * nb2 + i03 * nb3;
uint8_t * src0_ptr = (uint8_t *) src0->data + i02 * nb02 + i03 * nb03;
if (!dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01)) {
dma_queue_flush(q);
dma_queue_push(q, dma_make_ptr(dst_ptr, src0_ptr), nb1, nb01, ne00 * elem_size, ne01);
@@ -269,10 +294,9 @@ static inline void cpy_dma_sametype_sameshape(
dma_queue_flush(q);
}
int op_cpy(struct htp_ops_context * octx) {
static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
cpy_preamble;
const uint32_t n_threads = MIN(nr, octx->n_threads);
*use_dma = false;
struct htp_copy_context ct;
ct.octx = octx;
@@ -296,59 +320,117 @@ int op_cpy(struct htp_ops_context * octx) {
}
const bool sametype = (src0->type == dst->type);
const bool transposed = (nb00 > nb01) || (nb0 > nb1);
const bool transposed = (nb00 > nb01) || (nb0 > nb1) ||
(nb00 != ct.src0_type_size) || (nb0 != ct.dst_type_size) ||
(nb01 < ne00 * ct.src0_type_size) || (nb1 < ne0 * ct.dst_type_size);
const bool sameshape = !transposed && (ne00 == ne0 && ne01 == ne1 && ne02 == ne2 && ne03 == ne3);
ct.src0_nrows_per_thread = (nr + n_threads - 1) / n_threads;
const uint32_t n_threads = octx->n_threads;
worker_callback_t copy_fun = NULL;
bool use_dma = false;
const bool dst_is_contiguous = htp_tensor_is_contiguous(dst, ct.dst_type_size);
if (sametype && sameshape) {
use_dma = true;
} else if (sameshape) {
/**/ if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32)
copy_fun = cpy_thread_f16_f32_sameshape;
else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16)
copy_fun = cpy_thread_f32_f16_sameshape;
else
return HTP_STATUS_NO_SUPPORT;
} else if (sametype) {
if (src0->type == HTP_TYPE_F32) {
copy_fun = cpy_thread_f32_reshape;
} else {
copy_fun = cpy_thread_f16_reshape;
if (sameshape) {
const uint32_t total_rows = ne01 * ne02 * ne03;
const uint32_t row_size = ne00 * ct.dst_type_size;
ct.div_ne01 = init_fastdiv_values(ne01);
ct.div_ne02_ne01 = init_fastdiv_values(ne02 * ne01);
uint32_t row_start = 0;
uint32_t nrows = total_rows;
if (octx->ctx->mdev.count > 1) {
const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;
const bool can_split = htp_tensor_mdev_data_aligned(dst) && dst_is_contiguous;
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, can_split ? rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
ct.row_start = row_start;
ct.nrows = nrows;
ct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
if (sametype && octx->ctx->mdev.count <= 1) {
*use_dma = true;
cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3);
} else {
work_queue_func_t copy_fun = NULL;
if (sametype) {
copy_fun = (src0->type == HTP_TYPE_F32) ? cpy_thread_f32_sameshape : cpy_thread_f16_sameshape;
} else if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) {
copy_fun = cpy_thread_f16_f32_sameshape;
} else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) {
copy_fun = cpy_thread_f32_f16_sameshape;
} else {
return HTP_STATUS_NO_SUPPORT;
}
work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads);
}
} else if (sametype) {
const uint32_t total_elems = ne0 * ne1 * ne2 * ne3;
const uint32_t elems_per_line = (ct.dst_type_size == 4) ? 32 : 64;
ct.div_ne0 = init_fastdiv_values(ne0);
ct.div_ne1_ne0 = init_fastdiv_values(ne1 * ne0);
ct.div_ne2_ne1_ne0 = init_fastdiv_values(ne2 * ne1 * ne0);
ct.div_ne00 = init_fastdiv_values(ne00);
ct.div_ne01_ne00 = init_fastdiv_values(ne01 * ne00);
ct.div_ne02_ne01_ne00 = init_fastdiv_values(ne02 * ne01 * ne00);
uint32_t elem_start = 0;
uint32_t nelem = total_elems;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_mdev_data_aligned(dst) && dst_is_contiguous;
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_elems, can_split ? elems_per_line : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
elem_start = range.start;
nelem = range.count;
}
if (nelem == 0) {
return HTP_STATUS_OK;
}
ct.elem_start = elem_start;
ct.nelem = nelem;
ct.elem_per_thread = fastdiv(nelem + n_threads - 1, &octx->n_threads_div);
work_queue_func_t copy_fun = (src0->type == HTP_TYPE_F32) ? cpy_thread_f32_reshape : cpy_thread_f16_reshape;
work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads);
} else {
return HTP_STATUS_NO_SUPPORT;
}
FARF(HIGH, "cpy-%s-%s: (%ux%ux%ux%u) -> (%ux%ux%ux%u) : use_dma=%d n_threads %u\n",
src0->type == HTP_TYPE_F32 ? "f32" : "f16", dst->type == HTP_TYPE_F32 ? "f32" : "f16",
ne00, ne01, ne02, ne03, ne0, ne1, ne2, ne3, use_dma, n_threads);
if (use_dma) {
cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3);
} else {
worker_pool_run_func(octx->ctx->worker_pool, copy_fun, &ct, n_threads);
}
const struct htp_tensor *sync = octx->src[1];
if (sync && (sync->flags & HTP_TENSOR_FENCE)) {
if (!use_dma) {
// htp_tensor_flush_all(octx->ctx, octx->dsts, 1);
qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);
}
atomic_uint * sync_fence = (atomic_uint *) sync->data;
const uint32_t seq = (uint32_t) octx->op_params[0];
atomic_store(&sync_fence[0], seq);
asm volatile ("syncht" : : : "memory");
Q6_dccleaninva_A((void *) sync_fence);
FARF(HIGH, "ggml-hex: sync-release : fence %p seq %u\n", sync_fence, seq);
}
return HTP_STATUS_OK;
}
int op_cpy(struct htp_ops_context * octx) {
bool use_dma = false;
int status = exec_cpy(octx, &use_dma);
htp_ops_context_set_status(octx, status);
if (octx->op == HTP_OP_CPY_FENCE) {
if (!use_dma) {
htp_flush_dirty_ranges(octx->ctx);
}
htp_mdev_group_barrier(octx);
if (octx->ctx->mdev.idx == 0) {
const struct htp_tensor * sync = octx->src[1];
const uint32_t seq = (uint32_t) octx->op_params[0];
atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data;
htp_fence_write(sync_fence, seq, octx->status);
FARF(HIGH, "ggml-hex: sync-release : fence %p seq 0x%x status %d\n", sync_fence, seq, octx->status);
}
}
return octx->status;
}
ggml/src/ggml-hexagon/htp/cumsum-ops.c
+62 -43
View File
@@ -7,6 +7,8 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
#include "hex-profile.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
@@ -17,25 +19,25 @@
#define htp_cumsum_tensors_preamble \
const struct htp_tensor * restrict src0 = octx->src[0]; \
const struct htp_tensor * restrict dst = octx->dst; \
\
const uint32_t ne00 = src0->ne[0]; \
const uint32_t ne01 = src0->ne[1]; \
const uint32_t ne02 = src0->ne[2]; \
const uint32_t ne03 = src0->ne[3]; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
const uint32_t ne2 = dst->ne[2]; \
const uint32_t ne3 = dst->ne[3]; \
\
const uint32_t nb00 = src0->nb[0]; \
const uint32_t nb01 = src0->nb[1]; \
const uint32_t nb02 = src0->nb[2]; \
const uint32_t nb03 = src0->nb[3]; \
\
const uint32_t nb0 = dst->nb[0]; \
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
\
const uint32_t ne00 = src0->ne[0]; \
const uint32_t ne01 = src0->ne[1]; \
const uint32_t ne02 = src0->ne[2]; \
const uint32_t ne03 = src0->ne[3]; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
const uint32_t ne2 = dst->ne[2]; \
const uint32_t ne3 = dst->ne[3]; \
\
const uint32_t nb00 = src0->nb[0]; \
const uint32_t nb01 = src0->nb[1]; \
const uint32_t nb02 = src0->nb[2]; \
const uint32_t nb03 = src0->nb[3]; \
\
const uint32_t nb0 = dst->nb[0]; \
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3];
struct htp_cumsum_context {
@@ -46,6 +48,7 @@ struct htp_cumsum_context {
size_t dst_row_size_aligned;
uint32_t rows_per_thread;
uint32_t total_rows;
uint32_t row_start;
};
#define htp_cumsum_preamble \
@@ -116,11 +119,8 @@ static inline void hvx_cumsum_row_f32(const float * restrict src, float * restri
static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * data) {
htp_cumsum_preamble;
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
const uint32_t ir0 = cctx->rows_per_thread * ith;
const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->total_rows);
const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith;
const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows);
if (ir0 >= ir1) {
return;
@@ -149,11 +149,15 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat
src_row_size_aligned, src_row_size, 1);
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ir = ir0; ir < ir1; ir++) {
float * dst_spad_row = (float *) dma_queue_pop(dma_queue).src;
float * src_spad_row = (float *) dma_queue_pop(dma_queue).dst;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
hvx_cumsum_row_f32(src_spad_row, dst_spad_row, ne00);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
dma_queue_push_vtcm_to_ddr(dma_queue,
dma_make_ptr(dst_data + (ir * dst_row_size), (uint8_t *) dst_spad_row),
@@ -168,12 +172,10 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat
}
dma_queue_flush(dma_queue);
t2 = HAP_perf_get_qtimer_count();
FARF(HIGH, "cumsum-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n",
FARF(HIGH, "cumsum-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n",
ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1,
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
}
// ---------------------------------------------------------------------------
@@ -183,14 +185,14 @@ static void cumsum_thread_f32_dma(unsigned int nth, unsigned int ith, void * dat
static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) {
htp_cumsum_preamble;
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
const uint8_t * src_data = (const uint8_t *) src0->data;
uint8_t * dst_data = (uint8_t *) dst->data;
const uint32_t ir0 = cctx->rows_per_thread * ith;
const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->total_rows);
const uint32_t ir0 = cctx->row_start + cctx->rows_per_thread * ith;
const uint32_t ir1 = MIN(ir0 + cctx->rows_per_thread, cctx->row_start + cctx->total_rows);
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);
for (uint32_t ir = ir0; ir < ir1; ir++) {
const float * restrict src_row = (const float *) (src_data + ir * cctx->src_row_size);
@@ -198,12 +200,11 @@ static void cumsum_thread_f32(unsigned int nth, unsigned int ith, void * data) {
hvx_cumsum_row_f32(src_row, dst_row, ne00);
}
t2 = HAP_perf_get_qtimer_count();
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);
FARF(HIGH, "cumsum-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n",
FARF(HIGH, "cumsum-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n",
ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1,
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
}
int op_cumsum_f32(struct htp_ops_context * octx) {
@@ -214,8 +215,25 @@ int op_cumsum_f32(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const uint32_t n_threads = MIN(octx->n_threads, total_rows);
const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const size_t dst_data_row_size = dst->ne[0] * sizeof(float);
uint32_t row_start = 0;
uint32_t nrows = total_rows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
const size_t src_row_size = src0->nb[1];
const size_t dst_row_size = dst->nb[1];
@@ -240,14 +258,15 @@ int op_cumsum_f32(struct htp_ops_context * octx) {
.dst_row_size = dst_row_size,
.src_row_size_aligned = src_row_size_aligned,
.dst_row_size_aligned = dst_row_size_aligned,
.rows_per_thread = (total_rows + n_threads - 1) / n_threads,
.total_rows = total_rows,
.rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
.total_rows = nrows,
.row_start = row_start,
};
if (octx->ctx->vtcm_size < spad_per_thread * n_threads) {
worker_pool_run_func(octx->ctx->worker_pool, cumsum_thread_f32, &cctx, n_threads);
work_queue_run(octx->ctx->work_queue, cumsum_thread_f32, &cctx, n_threads);
} else {
worker_pool_run_func(octx->ctx->worker_pool, cumsum_thread_f32_dma, &cctx, n_threads);
work_queue_run(octx->ctx->work_queue, cumsum_thread_f32_dma, &cctx, n_threads);
}
return HTP_STATUS_OK;
ggml/src/ggml-hexagon/htp/diag-ops.c
+66 -35
View File
@@ -5,8 +5,11 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
#include "hex-profile.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hvx-types.h"
#include "hex-utils.h"
#include "hvx-copy.h"
@@ -15,17 +18,17 @@
#define htp_diag_tensors_preamble \
const struct htp_tensor * restrict src0 = octx->src[0]; \
const struct htp_tensor * restrict dst = octx->dst; \
\
const uint32_t ne02 = src0->ne[2]; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
\
const uint32_t nb02 = src0->nb[2]; \
const uint32_t nb03 = src0->nb[3]; \
\
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
\
const uint32_t ne02 = src0->ne[2]; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
\
const uint32_t nb02 = src0->nb[2]; \
const uint32_t nb03 = src0->nb[3]; \
\
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3];
struct htp_diag_context {
@@ -36,6 +39,7 @@ struct htp_diag_context {
size_t dst_row_size_aligned;
uint32_t batches_per_thread;
uint32_t total_batches;
uint32_t batch_start;
};
#define htp_diag_preamble \
@@ -57,11 +61,8 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data)
htp_diag_preamble;
dma_queue * dma_queue = octx->ctx->dma[ith];
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
const uint32_t ib0 = dctx->batches_per_thread * ith;
const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->total_batches);
const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith;
const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches);
if (ib0 >= ib1) {
return;
@@ -79,6 +80,8 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data)
uint8_t * src_spad = octx->src0_spad.data + (ith * src_batch_size_aligned);
uint8_t * dst_spad = octx->dst_spad.data + (ith * dst_row_size_aligned);
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ib = ib0; ib < ib1; ib++) {
const uint32_t i3 = ib / ne02;
const uint32_t i2 = ib % ne02;
@@ -96,7 +99,9 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data)
for (uint32_t i1 = 0; i1 < ne1; i1++) {
// Compute row in VTCM
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (ib * ne1 + i1));
hvx_diag_row_f32(src_spad_f32, dst_spad_f32, i1, ne0);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (ib * ne1 + i1));
// Write completed row back to DDR
uint8_t * dst_row = dst_data + i3 * nb3 + i2 * nb2 + i1 * nb1;
@@ -107,12 +112,9 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data)
}
}
t2 = HAP_perf_get_qtimer_count();
FARF(HIGH, "diag-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n",
FARF(HIGH, "diag-f32-dma %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n",
ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ib0, ib1,
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
}
// ---------------------------------------------------------------------------
@@ -122,14 +124,14 @@ static void diag_thread_f32_dma(unsigned int nth, unsigned int ith, void * data)
static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) {
htp_diag_preamble;
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
const uint8_t * src_data = (const uint8_t *) src0->data;
uint8_t * dst_data = (uint8_t *) dst->data;
const uint32_t ib0 = dctx->batches_per_thread * ith;
const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->total_batches);
const uint32_t ib0 = dctx->batch_start + dctx->batches_per_thread * ith;
const uint32_t ib1 = MIN(ib0 + dctx->batches_per_thread, dctx->batch_start + dctx->total_batches);
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0);
for (uint32_t ib = ib0; ib < ib1; ib++) {
const uint32_t i3 = ib / ne02;
@@ -143,12 +145,11 @@ static void diag_thread_f32(unsigned int nth, unsigned int ith, void * data) {
}
}
t2 = HAP_perf_get_qtimer_count();
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ib0);
FARF(HIGH, "diag-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n",
FARF(HIGH, "diag-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u\n",
ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ib0, ib1,
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
}
int op_diag_f32(struct htp_ops_context * octx) {
@@ -160,7 +161,36 @@ int op_diag_f32(struct htp_ops_context * octx) {
}
const uint32_t total_batches = src0->ne[2] * src0->ne[3];
const uint32_t n_threads = MIN(octx->n_threads, total_batches);
const size_t dst_batch_size = dst->ne[1] * dst->nb[1];
uint32_t batch_start = 0;
uint32_t nbatches = total_batches;
if (octx->ctx->mdev.count > 1) {
bool can_split = htp_tensor_mdev_data_aligned(dst) && (dst->ne[0] == 1 || dst->nb[0] == sizeof(float)) && !htp_tensor_is_permuted(dst);
uint32_t batches_per_chunk = 1;
if (can_split) {
if (dst->ne[2] > 1 && (dst->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 &&
(dst->ne[3] <= 1 || (dst->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0)) {
batches_per_chunk = 1;
} else if (dst->nb[2] == dst_batch_size &&
(dst->ne[3] <= 1 || dst->nb[3] == dst->nb[2] * dst->ne[2])) {
batches_per_chunk = (dst_batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(dst_batch_size, HEX_L2_LINE_SIZE)) : 1;
} else {
can_split = false;
}
}
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, can_split ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
batch_start = range.start;
nbatches = range.count;
}
if (nbatches == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
const size_t src_batch_size = src0->ne[0] * sizeof(float);
const size_t dst_row_size = dst->ne[0] * sizeof(float);
@@ -185,14 +215,15 @@ int op_diag_f32(struct htp_ops_context * octx) {
.dst_row_size = dst_row_size,
.src_batch_size_aligned = src_batch_size_aligned,
.dst_row_size_aligned = dst_row_size_aligned,
.batches_per_thread = (total_batches + n_threads - 1) / n_threads,
.total_batches = total_batches,
.batches_per_thread = fastdiv(nbatches + n_threads - 1, &octx->n_threads_div),
.total_batches = nbatches,
.batch_start = batch_start,
};
if (octx->ctx->vtcm_size < spad_per_thread * n_threads) {
worker_pool_run_func(octx->ctx->worker_pool, diag_thread_f32, &dctx, n_threads);
work_queue_run(octx->ctx->work_queue, diag_thread_f32, &dctx, n_threads);
} else {
worker_pool_run_func(octx->ctx->worker_pool, diag_thread_f32_dma, &dctx, n_threads);
work_queue_run(octx->ctx->work_queue, diag_thread_f32_dma, &dctx, n_threads);
}
return HTP_STATUS_OK;
ggml/src/ggml-hexagon/htp/fill-ops.c
+46 -23
View File
@@ -3,10 +3,11 @@
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <HAP_perf.h>
#include <string.h>
#include "hex-common.h"
#include "hex-profile.h"
#include "hvx-copy.h"
#include "hvx-utils.h"
@@ -14,28 +15,30 @@
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
// ggml op_params layout for FILL:
// op_params[0] (as float) - the scalar fill value
#define fill_preamble \
#define fill_preamble \
const struct htp_tensor * dst = octx->dst; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
const uint32_t ne2 = dst->ne[2]; \
const uint32_t ne3 = dst->ne[3]; \
\
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3]; \
\
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
const uint32_t ne2 = dst->ne[2]; \
const uint32_t ne3 = dst->ne[3]; \
\
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3]; \
\
const uint32_t nr = ne1 * ne2 * ne3;
struct htp_fill_context {
struct htp_ops_context * octx;
uint32_t nrows_per_thread;
uint32_t total_rows; // ne1 * ne2 * ne3
uint32_t row_start;
bool opt_path;
HVX_Vector splat_vec;
uint32_t elem_size;
@@ -47,10 +50,15 @@ static void fill_thread(unsigned int nth, unsigned int ith, void * data) {
fill_preamble;
// Parallelise over the flat row index spanning ne1*ne2*ne3
const uint32_t ir0 = fctx->nrows_per_thread * ith;
const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->total_rows);
const uint32_t ir0 = fctx->row_start + fctx->nrows_per_thread * ith;
const uint32_t ir1 = MIN(ir0 + fctx->nrows_per_thread, fctx->row_start + fctx->total_rows);
uint64_t t1 = HAP_perf_get_qtimer_count();
if (ir0 >= ir1) {
return;
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);
if (fctx->opt_path) {
// Opt path: tensor is fully contiguous, treat as flat array
@@ -69,9 +77,8 @@ static void fill_thread(unsigned int nth, unsigned int ith, void * data) {
}
}
uint64_t t2 = HAP_perf_get_qtimer_count();
FARF(HIGH, "fill %u/%u: rows %u:%u usec %u\n",
ith, nth, ir0, ir1, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1);
FARF(HIGH, "fill %u/%u: rows %u:%u\n", ith, nth, ir0, ir1);
}
int op_fill(struct htp_ops_context * octx) {
@@ -85,8 +92,23 @@ int op_fill(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
uint32_t row_start = 0;
uint32_t nrows = nr;
if (octx->ctx->mdev.count > 1) {
const uint32_t row_size = nb1;
const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(nr, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
// nr = ne1*ne2*ne3 (flat row count across all outer dims); parallelise over it.
const uint32_t n_threads = MIN(nr, octx->n_threads);
const uint32_t n_threads = octx->n_threads;
// Optimize if fully contiguous: skip stride arithmetic, treat as flat array
const bool opt_path = (nb2 == nb1 * ne1) && (nb3 == nb2 * ne2);
@@ -99,8 +121,9 @@ int op_fill(struct htp_ops_context * octx) {
struct htp_fill_context fctx = {
.octx = octx,
.nrows_per_thread = (nr + n_threads - 1) / n_threads,
.total_rows = nr,
.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
.total_rows = nrows,
.row_start = row_start,
.opt_path = opt_path,
};
@@ -117,7 +140,7 @@ int op_fill(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
worker_pool_run_func(octx->ctx->worker_pool, fill_thread, &fctx, n_threads);
work_queue_run(octx->ctx->work_queue, fill_thread, &fctx, n_threads);
return HTP_STATUS_OK;
}
ggml/src/ggml-hexagon/htp/flash-attn-ops.c
+62 -22
View File
@@ -5,7 +5,6 @@
#include <assert.h>
#include <HAP_compute_res.h>
#include <HAP_farf.h>
#include <HAP_perf.h>
#include <math.h>
#include <stdbool.h>
#include <stdatomic.h>
@@ -75,6 +74,7 @@ struct htp_fa_context {
uint32_t qrows;
uint32_t qrows_per_thread;
uint32_t qrow_start;
bool is_q_fp32;
@@ -89,8 +89,6 @@ struct htp_fa_context {
const struct htp_tensor * k;
const struct htp_tensor * v;
uint64_t t_start;
};
struct hmx_fa_context {
@@ -206,10 +204,9 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
const uint32_t nb3 = dst->nb[3];
// total rows in q
const uint32_t nr = factx->qrows;
const uint32_t dr = factx->qrows_per_thread;
const uint32_t ir0 = dr * ith;
const uint32_t ir1 = MIN(ir0 + dr, nr);
const uint32_t dr = factx->qrows_per_thread;
const uint32_t ir0 = factx->qrow_start + dr * ith;
const uint32_t ir1 = MIN(ir0 + dr, factx->qrow_start + factx->qrows);
if (ir0 >= ir1) return;
@@ -1888,6 +1885,24 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
const uint32_t n_threads = factx.n_threads;
const uint32_t G = factx.G;
// Multi-device: split Q blocks across devices
const uint32_t n_q_blocks = (neq1 + Br - 1) / Br;
uint32_t q_start_min = 0;
uint32_t q_start_max = neq1;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_q_blocks, htp_tensor_mdev_data_aligned(dst) ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
const uint32_t block_start = range.start;
const uint32_t block_end = range.start + range.count;
if (block_start >= block_end) {
return HTP_STATUS_OK;
}
q_start_min = block_start * Br;
q_start_max = MIN(block_end * Br, neq1);
}
// ======== VTCM allocation (GQA-aware) ========
// K/V row sizes drive the DMA descriptors (not the VTCM layout) and are used
// throughout the KV loop below.
@@ -1977,7 +1992,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
// ======== Main loop ========
for (uint32_t ib3 = 0; ib3 < neq3; ++ib3) {
const uint32_t im3 = mask ? fastmodulo(ib3, mask->ne[3], &factx.src3_div3) : 0;
for (uint32_t q_start = 0; q_start < neq1; q_start += Br) {
for (uint32_t q_start = q_start_min; q_start < q_start_max; q_start += Br) {
const uint32_t n_rows_q = hex_smin(Br, neq1 - q_start);
const size_t n_rows_g = n_rows_q * G;
const size_t g_br_actual = hex_align_up(n_rows_g, HMX_FP16_TILE_N_ROWS);
@@ -1991,8 +2006,9 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
// 1. Push Q and KV DMAs for the very first iteration.
// Subsequent iterations are enqueued early at the end of the previous iteration.
if (ib3 == 0 && q_start == 0 && kv_head == 0) {
const uint8_t * q_ptr = (const uint8_t *) q->data;
if (ib3 == 0 && q_start == q_start_min && kv_head == 0) {
const uint8_t * q_ptr = (const uint8_t *) q->data + q_start * q->nb[1] +
(kv_head * factx.G) * q->nb[2] + ib3 * q->nb[3];
const size_t q_row_bytes = q_transposed ? n_rows_q * q_row_bytes_trans_factor : q_row_bytes_untransposed;
const size_t n_rows = q_transposed ? factx.G : n_rows_q;
dma_queue_push(dma, dma_make_ptr(factx.vtcm_q_dma, q_ptr), q_row_bytes, hex_smax(q_src_stride, q_row_bytes), q_row_bytes, n_rows);
@@ -2311,8 +2327,8 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
if (next_kv_head >= n_kv_heads) {
next_kv_head = 0;
next_q_start = q_start + Br;
if (next_q_start >= neq1) {
next_q_start = 0;
if (next_q_start >= q_start_max) {
next_q_start = q_start_min;
next_ib3 = ib3 + 1;
}
}
@@ -2398,6 +2414,10 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
if (kparams->kernel_type == HTP_FA_KERNEL_HMX) {
return hmx_flash_attn_ext(octx);
}
@@ -2407,8 +2427,6 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
factx.k = k;
factx.v = v;
factx.t_start = HAP_perf_get_qtimer_count();
factx.src0_div21 = kparams->u.hvx.src0_div21;
factx.src0_div1 = kparams->u.hvx.src0_div1;
@@ -2451,8 +2469,30 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
}
// total rows in q
factx.qrows = kparams->qrows;
factx.qrows_per_thread = kparams->qrows_per_thread;
const uint32_t neq1 = q->ne[1];
const uint32_t neq2 = q->ne[2];
const uint32_t neq3 = q->ne[3];
const uint32_t total_qrows = neq1 * neq2 * neq3;
uint32_t qrow_start = 0;
uint32_t qrows = total_qrows;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_qrows, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
qrow_start = range.start;
qrows = range.count;
}
if (qrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
factx.qrows = qrows;
factx.qrow_start = qrow_start;
factx.qrows_per_thread = fastdiv(qrows + n_threads - 1, &octx->n_threads_div);
size_t size_vkq_acc = hex_round_up(v->ne[0] * sizeof(float), 128); // VKQ32
@@ -2461,18 +2501,18 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
uint8_t * vtcm_cur = octx->ctx->vtcm_base;
factx.spad_q = vtcm_seq_alloc(&vtcm_cur, size_q_block * octx->n_threads);
factx.spad_k = vtcm_seq_alloc(&vtcm_cur, factx.size_k_block * 2 * octx->n_threads);
factx.spad_v = vtcm_seq_alloc(&vtcm_cur, factx.size_v_block * 2 * octx->n_threads);
factx.spad_m = vtcm_seq_alloc(&vtcm_cur, (mask ? factx.size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * octx->n_threads);
factx.spad_a = vtcm_seq_alloc(&vtcm_cur, size_vkq_acc * octx->n_threads);
factx.spad_q = vtcm_seq_alloc(&vtcm_cur, size_q_block * n_threads);
factx.spad_k = vtcm_seq_alloc(&vtcm_cur, factx.size_k_block * 2 * n_threads);
factx.spad_v = vtcm_seq_alloc(&vtcm_cur, factx.size_v_block * 2 * n_threads);
factx.spad_m = vtcm_seq_alloc(&vtcm_cur, (mask ? factx.size_m_block * HVX_FA_DMA_CACHE_SIZE : 0) * n_threads);
factx.spad_a = vtcm_seq_alloc(&vtcm_cur, size_vkq_acc * n_threads);
if ((size_t) (vtcm_cur - octx->ctx->vtcm_base) > octx->ctx->vtcm_size) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {
work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, octx->n_threads);
work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, n_threads);
}
return HTP_STATUS_OK;
ggml/src/ggml-hexagon/htp/flash-attn-ops.h
+1
View File
@@ -51,6 +51,7 @@ struct htp_fa_kernel_params {
uint32_t qrows;
uint32_t qrows_per_thread;
uint32_t qrow_start;
float m0;
float m1;
uint32_t n_head_log2;
ggml/src/ggml-hexagon/htp/gated-delta-net-ops.c
+53 -21
View File
@@ -4,10 +4,13 @@
#include "hvx-utils.h"
#include "hex-fastdiv.h"
#include "hex-common.h"
#include "hex-profile.h"
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-tensor.h"
#ifndef MIN
#define MIN(a, b) ((a) < (b) ? (a) : (b))
@@ -22,6 +25,8 @@ struct htp_gdn_context {
size_t state_bytes;
uint8_t * vtcm_base;
size_t vtcm_per_thread;
uint32_t row_start;
uint32_t nrows;
};
static inline HVX_Vector gdn_mul_dot_f32(float * restrict dst, const float * restrict mul, const float * restrict dot, uint32_t n) {
@@ -586,8 +591,9 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo
const uint32_t n_seqs = v->ne[3];
const uint32_t K = octx->op_params[0];
const uint32_t total_rows = H * n_seqs;
if (ith >= total_rows) {
const uint32_t row_end = gctx->row_start + gctx->nrows;
if (ith >= gctx->nrows) {
return;
}
@@ -621,11 +627,11 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo
const uint64_t state_seq_stride = state->nb[3] / sizeof(float);
const uint64_t state_size_per_snap = (uint64_t) S_v * S_v * H * n_seqs;
uint32_t ir_prefetch = ith;
uint32_t ir_prefetch = gctx->row_start + ith;
int spad_idx = 0;
// Prefetch preamble (up to 2 steps)
for (int k = 0; k < 2 && ir_prefetch < total_rows; k++) {
for (int k = 0; k < 2 && ir_prefetch < row_end; k++) {
const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H);
const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H);
const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v;
@@ -646,8 +652,11 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo
spad_idx ^= 1;
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (gctx->row_start + ith));
int curr_spad_idx = 0;
for (uint32_t ir = ith; ir < total_rows; ir += nth) {
for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) {
dma_queue_pop(dma);
dma_queue_pop(dma);
@@ -812,7 +821,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo
S_v * sizeof(float), S_v);
// Prefetch next block (if any)
if (ir_prefetch < total_rows) {
if (ir_prefetch < row_end) {
const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H);
const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H);
const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v;
@@ -828,6 +837,7 @@ static void gated_delta_net_f32_pp_thread(unsigned int nth, unsigned int ith, vo
curr_spad_idx ^= 1;
}
dma_queue_flush(dma);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row_end);
}
@@ -847,8 +857,9 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo
const uint32_t H = v->ne[1];
const uint32_t n_seqs = v->ne[3];
const uint32_t total_rows = H * n_seqs;
if (ith >= total_rows) {
const uint32_t row_end = gctx->row_start + gctx->nrows;
if (ith >= gctx->nrows) {
return;
}
@@ -881,11 +892,11 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo
const uint64_t state_seq_stride = state->nb[3] / sizeof(float);
uint32_t ir_prefetch = ith;
uint32_t ir_prefetch = gctx->row_start + ith;
int spad_idx = 0;
// Prefetch preamble (up to 2 steps)
for (int k = 0; k < 2 && ir_prefetch < total_rows; k++) {
for (int k = 0; k < 2 && ir_prefetch < row_end; k++) {
const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H);
const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H);
const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v;
@@ -906,8 +917,11 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo
spad_idx ^= 1;
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) (gctx->row_start + ith));
int curr_spad_idx = 0;
for (uint32_t ir = ith; ir < total_rows; ir += nth) {
for (uint32_t ir = gctx->row_start + ith; ir < row_end; ir += nth) {
dma_queue_pop(dma);
dma_queue_pop(dma);
@@ -1057,7 +1071,7 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo
S_v * sizeof(float), S_v);
// Prefetch next block (if any)
if (ir_prefetch < total_rows) {
if (ir_prefetch < row_end) {
const uint32_t piv1 = fastmodulo(ir_prefetch, H, &fd_H);
const uint32_t piv3 = fastdiv(ir_prefetch, &fd_H);
const float * ps_in = state_in_base + (uint64_t) piv3 * state_seq_stride + (uint64_t) piv1 * S_v * S_v;
@@ -1073,6 +1087,7 @@ static void gated_delta_net_f32_tg_thread(unsigned int nth, unsigned int ith, vo
curr_spad_idx ^= 1;
}
dma_queue_flush(dma);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) row_end);
}
@@ -1085,10 +1100,6 @@ int op_gated_delta_net(struct htp_ops_context * octx) {
const struct htp_tensor * state = octx->src[5];
const struct htp_tensor * dst = octx->dst;
if (!q || !k || !v || !g || !beta || !state || !dst) {
return HTP_STATUS_INVAL_PARAMS;
}
if (q->type != HTP_TYPE_F32 || k->type != HTP_TYPE_F32 || v->type != HTP_TYPE_F32 ||
g->type != HTP_TYPE_F32 || beta->type != HTP_TYPE_F32 || state->type != HTP_TYPE_F32 ||
dst->type != HTP_TYPE_F32) {
@@ -1124,16 +1135,37 @@ int op_gated_delta_net(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
const uint32_t total_rows = H * n_seqs;
uint32_t row_start = 0;
uint32_t nrows = total_rows;
if (octx->ctx->mdev.count > 1) {
const uint32_t head_bytes = S_v * sizeof(float);
const uint32_t rows_per_chunk = (head_bytes > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(head_bytes, HEX_L2_LINE_SIZE)) : 1;
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0,
octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
struct htp_gdn_context gctx;
gctx.octx = octx;
gctx.rows_per_thread = (H * n_seqs + octx->n_threads - 1) / octx->n_threads;
gctx.row_start = row_start;
gctx.nrows = nrows;
gctx.rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
gctx.state_bytes = (size_t) S_v * S_v * sizeof(float);
size_t state_aligned = (size_t) S_v * S_v * sizeof(float);
state_aligned = (state_aligned + 127) & ~(size_t)127;
assert(octx->ctx->vtcm_base != NULL);
assert(octx->ctx->vtcm_size >= 2 * state_aligned * octx->n_threads);
assert(octx->ctx->vtcm_size >= 2 * state_aligned * n_threads);
gctx.vtcm_base = octx->ctx->vtcm_base;
gctx.vtcm_per_thread = 2 * state_aligned;
@@ -1148,9 +1180,9 @@ int op_gated_delta_net(struct htp_ops_context * octx) {
gctx.vtcm_per_thread * octx->n_threads, octx->n_threads);
if (n_tokens == 1) {
worker_pool_run_func(octx->ctx->worker_pool, gated_delta_net_f32_tg_thread, &gctx, octx->n_threads);
work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_tg_thread, &gctx, n_threads);
} else {
worker_pool_run_func(octx->ctx->worker_pool, gated_delta_net_f32_pp_thread, &gctx, octx->n_threads);
work_queue_run(octx->ctx->work_queue, gated_delta_net_f32_pp_thread, &gctx, n_threads);
}
return HTP_STATUS_OK;
ggml/src/ggml-hexagon/htp/get-rows-ops.c
+47 -15
View File
@@ -10,6 +10,7 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
@@ -23,9 +24,12 @@ struct get_rows_context {
const struct htp_get_rows_kernel_params * kparams;
struct htp_get_rows_vtcm_layout vtcm_layout;
uint8_t * vtcm_base;
uint32_t task_start;
uint32_t tasks;
uint32_t tasks_per_thread;
};
#define get_rows_preamble \
#define get_rows_preamble \
const uint32_t ne00 = octx->src[0]->ne[0]; \
const uint32_t ne01 = octx->src[0]->ne[1]; \
const uint32_t ne02 = octx->src[0]->ne[2]; \
@@ -61,12 +65,12 @@ static void get_rows_thread_st_##IDX_TYPE(unsigned int nth, unsigned int ith, vo
struct htp_ops_context * octx = grctx->octx; \
const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \
get_rows_preamble; \
const uint32_t dr = kparams->tasks_per_thread; \
const uint32_t ir0 = dr * ith; \
if (ir0 >= kparams->total_tasks) { \
const uint32_t dr = grctx->tasks_per_thread; \
const uint32_t ir0 = grctx->task_start + dr * ith; \
if (ir0 >= grctx->task_start + grctx->tasks) { \
return; \
} \
const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \
const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); \
const uint32_t row_size_bytes = htp_tensor_get_row_size(octx->src[0]->type, ne00); \
dma_queue * dma_queue = octx->ctx->dma[ith]; \
for (uint32_t i = ir0; i < ir1; ++i) { \
@@ -101,12 +105,12 @@ static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned
const struct htp_get_rows_kernel_params * kparams = grctx->kparams; \
get_rows_preamble; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
const uint32_t dr = kparams->tasks_per_thread; \
const uint32_t ir0 = dr * ith; \
if (ir0 >= kparams->total_tasks) { \
const uint32_t dr = grctx->tasks_per_thread; \
const uint32_t ir0 = grctx->task_start + dr * ith; \
if (ir0 >= grctx->task_start + grctx->tasks) { \
return; \
} \
const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \
const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); \
const uint32_t chunks_per_row = kparams->chunks_per_row; \
const uint32_t chunk_size = kparams->chunk_size; \
dma_queue * dma_queue = octx->ctx->dma[ith]; \
@@ -225,13 +229,41 @@ int op_get_rows(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
const struct htp_tensor * dst = octx->dst;
const uint32_t total_tasks = kparams->total_tasks;
const size_t dst_row_size = htp_tensor_get_row_size(dst->type, dst->ne[0]);
uint32_t task_start = 0;
uint32_t tasks = total_tasks;
if (octx->ctx->mdev.count > 1) {
uint32_t tasks_per_chunk = 1;
htp_tensor_mdev_rows_per_chunk(dst, dst_row_size / dst->ne[0], (uint32_t) dst_row_size, &tasks_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, tasks_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
task_start = range.start;
tasks = range.count;
}
if (tasks == 0) {
return HTP_STATUS_OK;
}
if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
const uint32_t n_threads = octx->n_threads;
struct get_rows_context grctx;
grctx.octx = octx;
grctx.kparams = kparams;
grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;
grctx.task_start = task_start;
grctx.tasks = tasks;
grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
const uint32_t ne00 = octx->src[0]->ne[0];
htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, kparams->n_threads);
htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, n_threads);
const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
@@ -247,14 +279,14 @@ int op_get_rows(struct htp_ops_context * octx) {
}
}
FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu use_dma=%d n_threads %d\n",
FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu use-dma %d n-threads %d\n",
octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3],
octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3],
octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3],
grctx.vtcm_layout.src0_bytes_per_thread * kparams->n_threads,
grctx.vtcm_layout.dst_bytes_per_thread * kparams->n_threads,
kparams->use_dma, kparams->n_threads);
grctx.vtcm_layout.src0_bytes_per_thread * n_threads,
grctx.vtcm_layout.dst_bytes_per_thread * n_threads,
kparams->use_dma, n_threads);
work_queue_run(octx->ctx->work_queue, q_func, &grctx, kparams->n_threads);
work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads);
return HTP_STATUS_OK;
}
ggml/src/ggml-hexagon/htp/hex-common.h
+9
View File
@@ -77,4 +77,13 @@ static inline bool hex_add_overflow(size_t a, size_t b, size_t *out) {
return false;
}
static inline uint32_t hex_gcd_u32(uint32_t a, uint32_t b) {
while (b != 0) {
uint32_t t = b;
b = a % b;
a = t;
}
return a;
}
#endif // HEX_COMMON_H
ggml/src/ggml-hexagon/htp/hex-utils.h
-1
View File
@@ -39,7 +39,6 @@ static inline void hex_l2fetch_block(const void * addr, size_t size) {
#define HEX_L2_LINE_SIZE 128
#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (lines per loop iteration)
#define HEX_L2_FLUSH_IL_THRESHOLD 1024 // inline flush threshold
#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024)
#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024)
ggml/src/ggml-hexagon/htp/hmx-utils.h
+7 -7
View File
@@ -27,7 +27,7 @@ static inline void hmx_init_column_scales(void *out_scales, HVX_Vector v_scale)
// vscatter offsets for fused dequant+transpose: write K-values directly to [K][N] tile.
// word[i] = i*128 maps K-row-pair i to byte offset i*128.
// Column offset (n*4) is added at runtime. Entries 0..15 cover one tile (region 2047);
// entries 16..31 cover the next adjacent tile (region 4095) — pick region size at the
// entries 16..31 cover the next adjacent tile (region 4095) - pick region size at the
// call site to scatter into one tile (masked) or two contiguous tiles (unmasked).
static const int32_t hmx_transpose_scatter_offsets[32] __attribute__((aligned(VLEN))) = {
0 * 128, 1 * 128, 2 * 128, 3 * 128, 4 * 128, 5 * 128, 6 * 128, 7 * 128, 8 * 128, 9 * 128, 10 * 128,
@@ -198,16 +198,16 @@ static inline void hmx_interleave_cols_to_tiles(__fp16 * restrict tiles_out,
}
// --- HMX inline asm macros for load-store packetization ---
#define HMX_LOAD_MPY_F16(act, wt, range) \
"{\n" \
#define HMX_LOAD_MPY_F16(act, wt, range) \
"{\n" \
" activation.hf = mxmem(" act ", " range ")\n" \
" weight.hf = mxmem(" wt ", " range ")\n" \
" weight.hf = mxmem(" wt ", " range ")\n" \
"}\n"
#define HMX_LOAD_MPY_DEEP_F16(act, wt, range) \
"{\n" \
#define HMX_LOAD_MPY_DEEP_F16(act, wt, range) \
"{\n" \
" activation.hf = mxmem(" act ", " range "):deep\n" \
" weight.hf = mxmem(" wt ", " range ")\n" \
" weight.hf = mxmem(" wt ", " range ")\n" \
"}\n"
#define HMX_STORE_AFTER_F16(out, scale_reg) \
ggml/src/ggml-hexagon/htp/htp-ctx.h
+42 -9
View File
@@ -19,7 +19,7 @@
#endif
#define HTP_MAX_MMAPS 16
#define HTP_MAX_DIRTY_RANGES 16
#define HTP_MAX_DIRTY_RANGES 32
// Memory mapping
struct htp_mmap {
@@ -29,6 +29,11 @@ struct htp_mmap {
uint32_t reserved;
};
struct htp_dirty_range {
uint32_t start;
uint32_t end;
};
// Scratchpad state
struct htp_spad {
const struct htp_tensor * src; // original src of the data (for reuse)
@@ -38,6 +43,14 @@ struct htp_spad {
uint32_t size_per_thread; // size per thread
};
struct htp_mdev_group {
uint16_t idx;
uint16_t count;
struct fastdiv_values count_div;
uint8_t * fence_base;
uint32_t fence_seq;
};
struct htp_context;
// Context while processing an Op
@@ -65,8 +78,10 @@ struct htp_ops_context {
struct htp_spad src3_spad;
struct htp_spad dst_spad;
uint32_t n_threads;
uint32_t flags;
uint32_t flags;
uint32_t n_threads;
struct fastdiv_values n_threads_div;
int status;
};
// Main context for htp DSP backend
@@ -76,6 +91,7 @@ struct htp_context {
struct htp_mmap mmap[HTP_MAX_MMAPS];
dma_queue_t dma[HTP_MAX_NTHREADS];
dma_queue_t dma_cached[HTP_MAX_NTHREADS];
struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1];
work_queue_t work_queue;
hmx_queue_t hmx_queue;
@@ -88,7 +104,6 @@ struct htp_context {
bool hmx_enabled;
bool etm;
uint32_t profiler;
struct htp_thread_trace trace[HTP_MAX_NTHREADS + 1];
uint8_t * vtcm_base;
size_t vtcm_size;
@@ -97,16 +112,13 @@ struct htp_context {
atomic_bool vtcm_needs_release;
uint64_t max_vmem;
struct htp_dirty_range {
uint32_t start;
uint32_t end;
uint32_t bi;
} dirty_ranges[HTP_MAX_DIRTY_RANGES];
struct htp_dirty_range dirty_ranges[HTP_MAX_DIRTY_RANGES];
// Persistent DDR scratchpad for MUL_MAT_ID mappings
void * ddr_spad_base;
size_t ddr_spad_size;
struct htp_mdev_group mdev;
struct htp_ops_context octx;
qurt_thread_t main_thread;
@@ -115,6 +127,27 @@ struct htp_context {
size_t footprint;
};
static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) {
if (n_threads == 0 || n_threads > octx->ctx->n_threads) {
return false;
}
if (n_threads != octx->n_threads) {
octx->n_threads = n_threads;
octx->n_threads_div = n_threads == octx->ctx->n_threads
? octx->ctx->n_threads_div
: init_fastdiv_values(n_threads);
}
return true;
}
static inline void htp_ops_context_set_status(struct htp_ops_context * octx, int status) {
if (status > HTP_STATUS_OK && octx->status == HTP_STATUS_OK) {
octx->status = status;
}
}
int op_matmul(struct htp_ops_context * octx);
int op_matmul_id(struct htp_ops_context * octx);
int op_matmul_nx(struct htp_ops_context * octx);
ggml/src/ggml-hexagon/htp/htp-fence.h
+89
View File
@@ -0,0 +1,89 @@
#ifndef HTP_FENCE_H
#define HTP_FENCE_H
#include <stdatomic.h>
#include <stdint.h>
#include <HAP_farf.h>
#include "hex-utils.h"
#include "htp-ops.h"
#include "htp-ctx.h"
static inline atomic_uint * htp_mdev_fence_slot(const void * fence_base, uint32_t idx) {
return (atomic_uint *) ((const uint8_t *) fence_base + (size_t) idx * HTP_FENCE_SLOT_SIZE);
}
static inline void htp_fence_write(void * fence_ptr, uint32_t seq, uint32_t status) {
atomic_uint * fence = (atomic_uint *) fence_ptr;
atomic_store(&fence[1], status);
atomic_store(&fence[0], seq);
asm volatile ("syncht" : : : "memory");
Q6_dccleaninva_A((void *) fence);
}
static inline void htp_fence_read(const void * fence_ptr, uint32_t * seq, uint32_t * status) {
const atomic_uint * fence = (const atomic_uint *) fence_ptr;
Q6_dccleaninva_A((void *) fence);
asm volatile ("syncht" : : : "memory");
*seq = atomic_load(&fence[0]);
*status = atomic_load(&fence[1]);
}
static inline void htp_mdev_group_barrier(struct htp_ops_context * octx) {
struct htp_context * ctx = octx->ctx;
if (ctx->mdev.count <= 1) {
return;
}
const uint32_t seq = ++ctx->mdev.fence_seq;
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
const uint32_t mdev_idx = ctx->mdev.idx;
const uint32_t mdev_count = ctx->mdev.count;
uint8_t * fence_base = ctx->mdev.fence_base;
atomic_uint * my_fence = htp_mdev_fence_slot(fence_base, mdev_idx);
htp_fence_write(my_fence, seq, octx->status);
for (uint32_t d = 0; d < mdev_count; d++) {
if (d == mdev_idx) continue;
atomic_uint * peer_fence = htp_mdev_fence_slot(fence_base, d);
uint64_t spins = 0;
while (1) {
uint32_t peer_seq;
uint32_t peer_status;
htp_fence_read(peer_fence, &peer_seq, &peer_status);
if ((int32_t)(peer_seq - seq) >= 0) {
if (peer_status > HTP_STATUS_OK) {
FARF(ERROR, "ggml-hex: mdev %u peer %u failed with status %u : seq 0x%08x\n",
mdev_idx, d, peer_status, seq);
htp_ops_context_set_status(octx, peer_status);
}
break;
}
if (++spins == 10000) {
FARF(ALWAYS, "ggml-hex: mdev %u waiting for mdev %u : seq 0x%08x (b %u op %u) my-fence %p peer-fence %p peer-seq 0x%08x (diff %d)\n",
mdev_idx, d, seq, seq >> 12, seq & 0xfff, my_fence, peer_fence, peer_seq, (int32_t)(peer_seq - seq));
}
if (spins > HTP_FENCE_TIMEOUT) {
FARF(ERROR, "ggml-hex: mdev %u timeout waiting for mdev %u : seq 0x%08x (b %u op %u) peer-fence %p peer-seq 0x%08x\n",
mdev_idx, d, seq, seq >> 12, seq & 0xfff, peer_fence, peer_seq);
htp_ops_context_set_status(octx, HTP_STATUS_INTERNAL_ERR);
break;
}
hex_pause();
}
}
asm volatile ("syncht" : : : "memory");
if (octx->status > HTP_STATUS_OK) {
htp_fence_write(my_fence, seq, octx->status);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
}
#endif // HTP_FENCE_H
ggml/src/ggml-hexagon/htp/htp-ops.h
+11 -12
View File
@@ -77,6 +77,7 @@ enum htp_op_code {
HTP_OP_GET_ROWS,
HTP_OP_SCALE,
HTP_OP_CPY,
HTP_OP_CPY_FENCE,
HTP_OP_ARGSORT,
HTP_OP_SQR,
HTP_OP_SQRT,
@@ -100,6 +101,7 @@ enum htp_op_code {
HTP_OP_ALLREDUCE,
HTP_OP_ALLREDUCE_ADD,
HTP_OP_GLU_SWIGLU_CLAMP,
HTP_OP_MDEV_GROUP,
HTP_OP_INVALID
};
@@ -114,6 +116,7 @@ enum htp_op_code {
#define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16)
#define HTP_FENCE_TIMEOUT (1000000000ULL)
#define HTP_FENCE_SLOT_SIZE 128
#define HTP_OP_MAX_VMEM_DEFAULT (3355443200u)
@@ -214,30 +217,26 @@ struct htp_prof_desc {
};
struct htp_opbatch_req {
uint32_t id; // Batch id
uint64_t seq; // Sequence number
uint32_t n_bufs; // Number of buffers
uint32_t n_tensors; // Number of tensors
uint32_t n_ops; // Number of ops
uint32_t n_traces; // Number of trace descriptors per thread
uint32_t pad; // unused
uint64_t seq; // Sequence number
// struct htp_buf_desc bufs[]; -- dspqueue buf 0
// struct htp_tensor tensors[]; -- dspqueue buf 0
// struct htp_op_desc ops[]; -- dspqueue buf 0
};
struct htp_opbatch_rsp {
uint32_t id; // Batch id
uint32_t status; // HTP_STATUS_...
uint32_t n_bufs; // Number of buffers
uint32_t n_tensors; // Number of tensors
uint32_t n_ops; // Number of op profile descriptors
uint32_t n_traces[HTP_MAX_NTHREADS + 1];
uint32_t usecs; // Number of usec
uint32_t pad; // align to 8 bytes
uint64_t seq; // Sequence number
uint64_t cycles_start; // Start cycle counter
uint64_t cycles_stop; // Stop cycle counter
uint64_t seq; // Sequence number
uint32_t status; // HTP_STATUS_...
uint32_t n_bufs; // Number of buffers
uint32_t n_tensors; // Number of tensors
uint32_t n_ops; // Number of op profile descriptors
uint32_t usecs; // Number of usec
uint32_t n_traces[HTP_MAX_NTHREADS + 1];
// struct htp_prof_desc profs[]; -- dspqueue buf 0
};
ggml/src/ggml-hexagon/htp/htp-tensor.c
+94 -43
View File
@@ -20,7 +20,7 @@ struct l2flush_range {
struct l2flush_multi_task {
struct htp_thread_trace * trace;
struct l2flush_range ranges[HTP_OP_MAX_INPUTS];
struct l2flush_range ranges[HTP_MAX_DIRTY_RANGES];
uint32_t n_ranges;
uint32_t total_blocks;
uint32_t blocks_per_thread;
@@ -73,6 +73,27 @@ static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) {
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, gb_first);
}
static void merge_dirty_ranges(struct htp_context * ctx) {
for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) {
struct htp_dirty_range * r = &ctx->dirty_ranges[i];
if (!r->start) continue;
for (uint32_t j = 0; j < HTP_MAX_DIRTY_RANGES;) {
struct htp_dirty_range * s = &ctx->dirty_ranges[j];
if (i == j || !s->start || r->end < s->start || s->end < r->start) {
j++;
continue;
}
r->start = MIN(r->start, s->start);
r->end = MAX(r->end, s->end);
s->start = 0;
s->end = 0;
j = 0;
}
}
}
void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) {
const struct htp_tensor * pending[HTP_OP_MAX_OUTPUTS];
uint32_t n_pending = 0;
@@ -83,11 +104,6 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co
continue;
}
if (t->size <= HEX_L2_FLUSH_IL_THRESHOLD) {
hex_l2flush((void *) (uintptr_t) t->data, t->size);
continue;
}
uint32_t t_start = t->data;
uint32_t t_end = t_start + t->size;
@@ -110,6 +126,8 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co
}
}
merge_dirty_ranges(ctx);
if (n_pending == 0) {
return;
}
@@ -132,8 +150,8 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co
struct htp_dirty_range * r = &ctx->dirty_ranges[idx];
r->start = pending[i]->data;
r->end = pending[i]->data + pending[i]->size;
r->bi = pending[i]->bi;
}
merge_dirty_ranges(ctx);
return;
}
@@ -151,12 +169,12 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co
struct htp_dirty_range * r = &ctx->dirty_ranges[i];
r->start = pending[i]->data;
r->end = pending[i]->data + pending[i]->size;
r->bi = pending[i]->bi;
}
merge_dirty_ranges(ctx);
return;
}
if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_OP_MAX_INPUTS) {
if (total_evict_size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1 && n_evict <= HTP_MAX_DIRTY_RANGES) {
struct l2flush_multi_task task;
task.trace = ctx->trace;
task.n_ranges = n_evict;
@@ -195,7 +213,6 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co
struct htp_dirty_range * r = &ctx->dirty_ranges[idx];
r->start = pending[i]->data;
r->end = pending[i]->data + pending[i]->size;
r->bi = pending[i]->bi;
}
for (uint32_t i = 0; i < n_empty; i++) {
@@ -203,8 +220,9 @@ void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * co
struct htp_dirty_range * r = &ctx->dirty_ranges[idx];
r->start = pending[n_evict + i]->data;
r->end = pending[n_evict + i]->data + pending[n_evict + i]->size;
r->bi = pending[n_evict + i]->bi;
}
merge_dirty_ranges(ctx);
}
static void make_tensor_clean(struct htp_context * ctx, const struct htp_tensor * t) {
@@ -242,15 +260,77 @@ static inline bool is_tensor_dirty(struct htp_context * ctx, const struct htp_te
return false;
}
static void flush_dirty_ranges(struct htp_context * ctx, const struct htp_dirty_range * ranges, uint32_t n_ranges, uint64_t total_dirty) {
if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) {
struct l2flush_multi_task task;
task.trace = ctx->trace;
task.n_ranges = n_ranges;
uint32_t block_acc = 0;
for (uint32_t i = 0; i < n_ranges; i++) {
const struct htp_dirty_range * r = &ranges[i];
struct l2flush_range * rg = &task.ranges[i];
rg->start = hex_align_down((size_t) r->start, HEX_L2_LINE_SIZE);
rg->end = hex_align_up((size_t) r->end, HEX_L2_LINE_SIZE);
rg->block_first = block_acc;
rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE;
block_acc += rg->n_blocks;
}
task.total_blocks = block_acc;
task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div);
work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads);
} else {
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0);
for (uint32_t i = 0; i < n_ranges; i++) {
const struct htp_dirty_range * r = &ranges[i];
hex_l2flush((void *) (uintptr_t) r->start, r->end - r->start);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0);
}
}
void htp_flush_dirty_ranges(struct htp_context * ctx) {
struct htp_dirty_range ranges[HTP_MAX_DIRTY_RANGES];
uint32_t n_ranges = 0;
uint64_t total_dirty = 0;
for (uint32_t i = 0; i < HTP_MAX_DIRTY_RANGES; i++) {
const struct htp_dirty_range * r = &ctx->dirty_ranges[i];
if (!r->start) {
continue;
}
ranges[n_ranges++] = *r;
total_dirty += r->end - r->start;
}
if (total_dirty == 0) {
return;
}
if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) {
flush_all_dcache(ctx);
return;
}
flush_dirty_ranges(ctx, ranges, n_ranges, total_dirty);
memset(ctx->dirty_ranges, 0, sizeof(ctx->dirty_ranges));
}
void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) {
const struct htp_tensor * dirty_tensors[HTP_OP_MAX_INPUTS];
struct htp_dirty_range ranges[HTP_OP_MAX_INPUTS];
uint32_t n_dirty = 0;
uint64_t total_dirty = 0;
for (uint32_t i = 0; i < n; i++) {
const struct htp_tensor * t = tensors[i];
if (t && !(t->flags & (HTP_TENSOR_WEIGHT | HTP_TENSOR_FENCE)) && is_tensor_dirty(ctx, t)) {
if (t && is_tensor_dirty(ctx, t)) {
dirty_tensors[n_dirty++] = t;
ranges[n_dirty - 1].start = t->data;
ranges[n_dirty - 1].end = t->data + t->size;
total_dirty += t->size;
}
}
@@ -264,37 +344,8 @@ void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * co
return;
}
if (total_dirty >= HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) {
struct l2flush_multi_task task;
task.trace = ctx->trace;
task.n_ranges = 0;
uint32_t block_acc = 0;
for (uint32_t i = 0; i < n_dirty; i++) {
const struct htp_tensor * t = dirty_tensors[i];
make_tensor_clean(ctx, t);
struct l2flush_range * rg = &task.ranges[task.n_ranges++];
rg->start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE);
rg->end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE);
rg->block_first = block_acc;
rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE;
block_acc += rg->n_blocks;
}
task.total_blocks = block_acc;
task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div);
work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads);
return;
}
struct htp_thread_trace * tr = &ctx->trace[0];
flush_dirty_ranges(ctx, ranges, n_dirty, total_dirty);
for (uint32_t i = 0; i < n_dirty; i++) {
const struct htp_tensor * t = dirty_tensors[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);
hex_l2flush((void *) (uintptr_t) t->data, t->size);
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);
make_tensor_clean(ctx, t);
make_tensor_clean(ctx, dirty_tensors[i]);
}
}
ggml/src/ggml-hexagon/htp/htp-tensor.h
+109
View File
@@ -2,8 +2,20 @@
#define HTP_TENSOR_H
#include <stdint.h>
#include <stdbool.h>
#include "htp-ops.h"
#include "hex-bitmap.h"
#include "hex-common.h"
#include "hex-fastdiv.h"
enum {
HTP_TENSOR_MDEV_LINE_SIZE = 128,
};
struct htp_tensor_mdev_range {
uint32_t start;
uint32_t count;
};
static inline void * htp_tensor_data(const struct htp_tensor * t) {
return (void *) (uintptr_t) t->data;
@@ -13,6 +25,102 @@ static inline uint32_t * htp_tensor_flags(const struct htp_tensor * t) {
return (uint32_t *) &t->flags;
}
static inline bool htp_tensor_is_contiguous(const struct htp_tensor * t, uint32_t type_size) {
uint32_t next_nb = type_size;
if (t->ne[0] != 1 && t->nb[0] != next_nb) {
return false;
}
next_nb *= t->ne[0];
for (int i = 1; i < HTP_OP_MAX_DIMS; i++) {
if (t->ne[i] != 1 && t->nb[i] != next_nb) {
return false;
}
next_nb *= t->ne[i];
}
return true;
}
static inline bool htp_tensor_is_permuted(const struct htp_tensor * t) {
return t->nb[0] > t->nb[1] || t->nb[1] > t->nb[2] || t->nb[2] > t->nb[3];
}
static inline bool htp_tensor_mdev_data_aligned(const struct htp_tensor * t) {
return ((uintptr_t) t->data & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0;
}
static inline bool htp_tensor_can_row_partition(const struct htp_tensor * t, uint32_t elem_size) {
if (!htp_tensor_mdev_data_aligned(t)) {
return false;
}
if (t->ne[0] != 1 && t->nb[0] != elem_size) {
return false;
}
if (htp_tensor_is_permuted(t)) {
return false;
}
if (t->ne[1] > 1 && (t->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false;
if (t->ne[2] > 1 && (t->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false;
if (t->ne[3] > 1 && (t->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) != 0) return false;
return true;
}
static inline bool htp_tensor_mdev_rows_per_chunk(const struct htp_tensor * t, uint32_t elem_size, uint32_t row_size, uint32_t * rows_per_chunk) {
*rows_per_chunk = 0;
if (!htp_tensor_mdev_data_aligned(t)) {
return false;
}
if (t->ne[0] != 1 && t->nb[0] != elem_size) {
return false;
}
if (htp_tensor_is_permuted(t)) {
return false;
}
if (t->ne[1] > 1 && (t->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 &&
(t->ne[2] <= 1 || (t->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0) &&
(t->ne[3] <= 1 || (t->nb[3] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0)) {
*rows_per_chunk = 1;
return true;
}
if (t->nb[1] == row_size &&
(t->ne[2] <= 1 || t->nb[2] == t->nb[1] * t->ne[1]) &&
(t->ne[3] <= 1 || t->nb[3] == t->nb[2] * t->ne[2])) {
*rows_per_chunk = (row_size > 0) ? (HTP_TENSOR_MDEV_LINE_SIZE / hex_gcd_u32(row_size, HTP_TENSOR_MDEV_LINE_SIZE)) : 1;
return true;
}
return false;
}
static inline struct htp_tensor_mdev_range htp_tensor_mdev_partition(uint32_t total_units, uint32_t units_per_chunk, uint32_t mdev_idx, uint32_t mdev_count, const struct fastdiv_values * mdev_count_div) {
struct htp_tensor_mdev_range range = { 0, total_units };
if (mdev_count <= 1) {
return range;
}
if (units_per_chunk == 0) {
range.start = (mdev_idx == 0) ? 0 : total_units;
range.count = (mdev_idx == 0) ? total_units : 0;
return range;
}
const uint32_t total_chunks = total_units / units_per_chunk;
if (total_chunks < mdev_count) {
range.start = (mdev_idx == 0) ? 0 : total_units;
range.count = (mdev_idx == 0) ? total_units : 0;
return range;
}
const uint32_t chunks_per_mdev = fastdiv(total_chunks + mdev_count - 1, mdev_count_div);
range.start = MIN(mdev_idx * chunks_per_mdev * units_per_chunk, total_units);
if (mdev_idx == mdev_count - 1) {
range.count = total_units - range.start;
} else {
range.count = MIN(chunks_per_mdev * units_per_chunk, total_units - range.start);
}
return range;
}
static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) {
switch (type) {
case HTP_TYPE_F32: return ne00 * 4;
@@ -23,6 +131,7 @@ static inline uint32_t htp_tensor_get_row_size(int type, uint32_t ne00) {
}
struct htp_context;
void htp_flush_dirty_ranges(struct htp_context * ctx);
void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);
void htp_tensor_dirty_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);
ggml/src/ggml-hexagon/htp/hvx-arith.h
+137 -137
View File
@@ -16,25 +16,25 @@
#define UNUSED(x) (void)(x)
#define hvx_arith_loop_body(dst_type, src0_type, src1_type, elem_size, vec_store, vec_op) \
do { \
dst_type * vdst = (dst_type *) dst; \
src0_type * vsrc0 = (src0_type *) src0; \
src1_type * vsrc1 = (src1_type *) src1; \
\
const uint32_t epv = 128 / (elem_size); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = vec_op(vsrc0[i], vsrc1[i]); \
} \
if (nloe) { \
HVX_Vector v = vec_op(vsrc0[i], vsrc1[i]); \
vec_store((void *) &vdst[i], nloe * (elem_size), v); \
} \
do { \
dst_type * vdst = (dst_type *) dst; \
src0_type * vsrc0 = (src0_type *) src0; \
src1_type * vsrc1 = (src1_type *) src1; \
\
const uint32_t epv = 128 / (elem_size); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = vec_op(vsrc0[i], vsrc1[i]); \
} \
if (nloe) { \
HVX_Vector v = vec_op(vsrc0[i], vsrc1[i]); \
vec_store((void *) &vdst[i], nloe * (elem_size), v); \
} \
} while(0)
#if __HVX_ARCH__ < 79
@@ -56,43 +56,43 @@
#define HVX_OP_MUL_F16(a, b) hvx_vec_mul_f16_f16(a, b)
// Generic macro to define alignment permutations for an op
#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \
static inline void OP_NAME##_aaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_aau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_aua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_auu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_uaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
static inline void OP_NAME##_uau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) src0 % 128 == 0); \
hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
static inline void OP_NAME##_uua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
static inline void OP_NAME##_uuu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
#define DEFINE_HVX_BINARY_OP_VARIANTS(OP_NAME, OP_MACRO, ELEM_TYPE) \
static inline void OP_NAME##_aaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_aau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_aua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_auu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
hvx_arith_loop_body(HVX_Vector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_uaa(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
static inline void OP_NAME##_uau(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) src0 % 128 == 0); \
hvx_arith_loop_body(HVX_UVector, HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
static inline void OP_NAME##_uua(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
assert((uintptr_t) src1 % 128 == 0); \
hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
static inline void OP_NAME##_uuu(uint8_t * dst, const uint8_t * src0, const uint8_t * src1, uint32_t n) { \
hvx_arith_loop_body(HVX_UVector, HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
} \
DEFINE_HVX_BINARY_OP_VARIANTS(hvx_add_f32, HVX_OP_ADD_F32, float)
DEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f32, HVX_OP_SUB_F32, float)
@@ -103,25 +103,25 @@ DEFINE_HVX_BINARY_OP_VARIANTS(hvx_sub_f16, HVX_OP_SUB_F16, _Float16)
DEFINE_HVX_BINARY_OP_VARIANTS(hvx_mul_f16, HVX_OP_MUL_F16, _Float16)
// Dispatcher logic
#define HVX_BINARY_DISPATCHER(OP_NAME) \
#define HVX_BINARY_DISPATCHER(OP_NAME) \
static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \
if (hex_is_aligned((void *) dst, 128)) { \
if (hex_is_aligned((void *) src0, 128)) { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \
else OP_NAME##_aau(dst, src0, src1, num_elems); \
} else { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \
else OP_NAME##_auu(dst, src0, src1, num_elems); \
} \
} else { \
if (hex_is_aligned((void *) src0, 128)) { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \
else OP_NAME##_uau(dst, src0, src1, num_elems); \
} else { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \
else OP_NAME##_uuu(dst, src0, src1, num_elems); \
} \
} \
if (hex_is_aligned((void *) dst, 128)) { \
if (hex_is_aligned((void *) src0, 128)) { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \
else OP_NAME##_aau(dst, src0, src1, num_elems); \
} else { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \
else OP_NAME##_auu(dst, src0, src1, num_elems); \
} \
} else { \
if (hex_is_aligned((void *) src0, 128)) { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \
else OP_NAME##_uau(dst, src0, src1, num_elems); \
} else { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \
else OP_NAME##_uuu(dst, src0, src1, num_elems); \
} \
} \
}
HVX_BINARY_DISPATCHER(hvx_add_f32)
@@ -166,44 +166,44 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re
// Scalar Operations
#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t epv = 128 / (elem_size); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
HVX_Vector v = vsrc[i]; \
vdst[i] = scalar_op_macro(v); \
} \
if (nloe) { \
HVX_Vector v = vsrc[i]; \
v = scalar_op_macro(v); \
vec_store((void *) &vdst[i], nloe * (elem_size), v); \
} \
#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t epv = 128 / (elem_size); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
HVX_Vector v = vsrc[i]; \
vdst[i] = scalar_op_macro(v); \
} \
if (nloe) { \
HVX_Vector v = vsrc[i]; \
v = scalar_op_macro(v); \
vec_store((void *) &vdst[i], nloe * (elem_size), v); \
} \
} while(0)
#define HVX_OP_ADD_SCALAR_F32(v) \
({ \
#define HVX_OP_ADD_SCALAR_F32(v) \
({ \
const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VwVw(inf, v); \
HVX_Vector out = HVX_OP_ADD_F32(v, val_vec); \
Q6_V_vmux_QVV(pred_inf, inf, out); \
HVX_Vector out = HVX_OP_ADD_F32(v, val_vec); \
Q6_V_vmux_QVV(pred_inf, inf, out); \
})
#define HVX_OP_MUL_SCALAR_F32(v) HVX_OP_MUL_F32(v, val_vec)
#define HVX_OP_SUB_SCALAR_F32(v) HVX_OP_SUB_F32(v, val_vec)
#define HVX_OP_ADD_SCALAR_F16(v) \
({ \
#define HVX_OP_ADD_SCALAR_F16(v) \
({ \
const HVX_VectorPred pred_inf = Q6_Q_vcmp_eq_VhVh(inf, v); \
HVX_Vector out = HVX_OP_ADD_F16(v, val_vec); \
Q6_V_vmux_QVV(pred_inf, inf, out); \
HVX_Vector out = HVX_OP_ADD_F16(v, val_vec); \
Q6_V_vmux_QVV(pred_inf, inf, out); \
})
#define HVX_OP_MUL_SCALAR_F16(v) HVX_OP_MUL_F16(v, val_vec)
@@ -212,31 +212,31 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re
// Scalar Variants
// Generic macro to define alignment permutations for an op
#define DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(OP_NAME, OP_MACRO, SPLAT_MACRO, ELEM_TYPE) \
#define DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(OP_NAME, OP_MACRO, SPLAT_MACRO, ELEM_TYPE) \
static inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \
const HVX_Vector val_vec = SPLAT_MACRO(val); \
const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src % 128 == 0); \
hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
const HVX_Vector val_vec = SPLAT_MACRO(val); \
const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src % 128 == 0); \
hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \
const HVX_Vector val_vec = SPLAT_MACRO(val); \
const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \
assert((uintptr_t) dst % 128 == 0); \
hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
const HVX_Vector val_vec = SPLAT_MACRO(val); \
const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \
assert((uintptr_t) dst % 128 == 0); \
hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_a, OP_MACRO); \
} \
static inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \
const HVX_Vector val_vec = SPLAT_MACRO(val); \
const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \
assert((uintptr_t) src % 128 == 0); \
hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
const HVX_Vector val_vec = SPLAT_MACRO(val); \
const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \
assert((uintptr_t) src % 128 == 0); \
hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
static inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, uint32_t n) { \
const HVX_Vector val_vec = SPLAT_MACRO(val); \
const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \
hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
const HVX_Vector val_vec = SPLAT_MACRO(val); \
const HVX_Vector inf = SPLAT_MACRO((ELEM_TYPE)INFINITY); UNUSED(inf); \
hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(ELEM_TYPE), hvx_vec_store_u, OP_MACRO); \
} \
DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_add_scalar_f32, HVX_OP_ADD_SCALAR_F32, hvx_vec_splat_f32, float)
DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f32, HVX_OP_SUB_SCALAR_F32, hvx_vec_splat_f32, float)
@@ -247,17 +247,17 @@ DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_sub_scalar_f16, HVX_OP_SUB_SCALAR_F16,
DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS(hvx_mul_scalar_f16, HVX_OP_MUL_SCALAR_F16, hvx_vec_splat_f16, _Float16)
// Dispatcher logic
#define HVX_BINARY_SCALAR_DISPATCHER(OP_NAME, ELEM_TYPE) \
#define HVX_BINARY_SCALAR_DISPATCHER(OP_NAME, ELEM_TYPE) \
static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const ELEM_TYPE val, const uint32_t num_elems) { \
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \
OP_NAME##_aa(dst, src, val, num_elems); \
} else if (hex_is_aligned((void *) dst, 128)) { \
OP_NAME##_au(dst, src, val, num_elems); \
} else if (hex_is_aligned((void *) src, 128)) { \
OP_NAME##_ua(dst, src, val, num_elems); \
} else { \
OP_NAME##_uu(dst, src, val, num_elems); \
} \
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \
OP_NAME##_aa(dst, src, val, num_elems); \
} else if (hex_is_aligned((void *) dst, 128)) { \
OP_NAME##_au(dst, src, val, num_elems); \
} else if (hex_is_aligned((void *) src, 128)) { \
OP_NAME##_ua(dst, src, val, num_elems); \
} else { \
OP_NAME##_uu(dst, src, val, num_elems); \
} \
}
HVX_BINARY_SCALAR_DISPATCHER(hvx_add_scalar_f32, float)
@@ -350,12 +350,12 @@ static inline void hvx_max_scalar_f32(uint8_t * restrict dst, const uint8_t * re
// CLAMP Scalar variants
#define HVX_OP_CLAMP_SCALAR(v) \
({ \
#define HVX_OP_CLAMP_SCALAR(v) \
({ \
HVX_VectorPred pred_cap_right = Q6_Q_vcmp_gt_VsfVsf(v, max_vec); \
HVX_VectorPred pred_cap_left = Q6_Q_vcmp_gt_VsfVsf(min_vec, v); \
HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \
Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \
HVX_Vector tmp = Q6_V_vmux_QVV(pred_cap_right, max_vec, v); \
Q6_V_vmux_QVV(pred_cap_left, min_vec, tmp); \
})
static inline void hvx_clamp_scalar_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const float min, const float max, uint32_t n) {
ggml/src/ggml-hexagon/htp/hvx-div.h
+47 -47
View File
@@ -219,64 +219,64 @@ static inline HVX_Vector hvx_vec_hybrid_div_f16(HVX_Vector vec1, HVX_Vector vec2
} while(0)
// Generic macro to define alignment permutations for an op
#define DEFINE_HVX_DIV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \
#define DEFINE_HVX_DIV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \
static inline void OP_NAME##_aaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_Vector, hvx_vec_store_a); \
} \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_Vector, hvx_vec_store_a); \
} \
static inline void OP_NAME##_aau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_UVector, hvx_vec_store_a); \
} \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src0 % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_Vector, HVX_UVector, hvx_vec_store_a); \
} \
static inline void OP_NAME##_aua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_Vector, hvx_vec_store_a); \
} \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_Vector, hvx_vec_store_a); \
} \
static inline void OP_NAME##_auu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_UVector, hvx_vec_store_a); \
} \
assert((uintptr_t) dst % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_UVector, HVX_UVector, hvx_vec_store_a); \
} \
static inline void OP_NAME##_uaa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_Vector, hvx_vec_store_u); \
} \
assert((uintptr_t) src0 % 128 == 0); \
assert((uintptr_t) src1 % 128 == 0); \
OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_Vector, hvx_vec_store_u); \
} \
static inline void OP_NAME##_uau(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) src0 % 128 == 0); \
OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_UVector, hvx_vec_store_u); \
} \
assert((uintptr_t) src0 % 128 == 0); \
OP_LOOP_BODY(HVX_UVector, HVX_Vector, HVX_UVector, hvx_vec_store_u); \
} \
static inline void OP_NAME##_uua(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
assert((uintptr_t) src1 % 128 == 0); \
OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_Vector, hvx_vec_store_u); \
} \
assert((uintptr_t) src1 % 128 == 0); \
OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_Vector, hvx_vec_store_u); \
} \
static inline void OP_NAME##_uuu(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { \
OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_UVector, hvx_vec_store_u); \
} \
OP_LOOP_BODY(HVX_UVector, HVX_UVector, HVX_UVector, hvx_vec_store_u); \
} \
// Dispatcher logic
#define HVX_DIV_DISPATCHER(OP_NAME) \
#define HVX_DIV_DISPATCHER(OP_NAME) \
static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint32_t num_elems) { \
if (hex_is_aligned((void *) dst, 128)) { \
if (hex_is_aligned((void *) src0, 128)) { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \
else OP_NAME##_aau(dst, src0, src1, num_elems); \
} else { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \
else OP_NAME##_auu(dst, src0, src1, num_elems); \
} \
} else { \
if (hex_is_aligned((void *) src0, 128)) { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \
else OP_NAME##_uau(dst, src0, src1, num_elems); \
} else { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \
else OP_NAME##_uuu(dst, src0, src1, num_elems); \
} \
} \
if (hex_is_aligned((void *) dst, 128)) { \
if (hex_is_aligned((void *) src0, 128)) { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aaa(dst, src0, src1, num_elems); \
else OP_NAME##_aau(dst, src0, src1, num_elems); \
} else { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_aua(dst, src0, src1, num_elems); \
else OP_NAME##_auu(dst, src0, src1, num_elems); \
} \
} else { \
if (hex_is_aligned((void *) src0, 128)) { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uaa(dst, src0, src1, num_elems); \
else OP_NAME##_uau(dst, src0, src1, num_elems); \
} else { \
if (hex_is_aligned((void *) src1, 128)) OP_NAME##_uua(dst, src0, src1, num_elems); \
else OP_NAME##_uuu(dst, src0, src1, num_elems); \
} \
} \
}
DEFINE_HVX_DIV_OP_VARIANTS(hvx_div_f32, hvx_div_f32_loop_body)
ggml/src/ggml-hexagon/htp/hvx-inverse.h
+23 -23
View File
@@ -169,36 +169,36 @@ static inline HVX_Vector hvx_vec_inverse_f16_guard(HVX_Vector v_sf, HVX_Vector n
} while(0)
// Generic macro to define alignment permutations for an op
#define DEFINE_HVX_INV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \
#define DEFINE_HVX_INV_OP_VARIANTS(OP_NAME, OP_LOOP_BODY) \
static inline void OP_NAME##_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_Vector, hvx_vec_store_a); \
} \
assert((uintptr_t) dst % 128 == 0); \
assert((uintptr_t) src % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_Vector, hvx_vec_store_a); \
} \
static inline void OP_NAME##_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \
assert((uintptr_t) dst % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_UVector, hvx_vec_store_a); \
} \
assert((uintptr_t) dst % 128 == 0); \
OP_LOOP_BODY(HVX_Vector, HVX_UVector, hvx_vec_store_a); \
} \
static inline void OP_NAME##_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \
assert((uintptr_t) src % 128 == 0); \
OP_LOOP_BODY(HVX_UVector, HVX_Vector, hvx_vec_store_u); \
} \
assert((uintptr_t) src % 128 == 0); \
OP_LOOP_BODY(HVX_UVector, HVX_Vector, hvx_vec_store_u); \
} \
static inline void OP_NAME##_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { \
OP_LOOP_BODY(HVX_UVector, HVX_UVector, hvx_vec_store_u); \
} \
OP_LOOP_BODY(HVX_UVector, HVX_UVector, hvx_vec_store_u); \
} \
// Dispatcher logic
#define HVX_INV_DISPATCHER(OP_NAME) \
#define HVX_INV_DISPATCHER(OP_NAME) \
static inline void OP_NAME(uint8_t * restrict dst, const uint8_t * restrict src, const uint32_t num_elems) { \
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \
OP_NAME##_aa(dst, src, num_elems); \
} else if (hex_is_aligned((void *) dst, 128)) { \
OP_NAME##_au(dst, src, num_elems); \
} else if (hex_is_aligned((void *) src, 128)) { \
OP_NAME##_ua(dst, src, num_elems); \
} else { \
OP_NAME##_uu(dst, src, num_elems); \
} \
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { \
OP_NAME##_aa(dst, src, num_elems); \
} else if (hex_is_aligned((void *) dst, 128)) { \
OP_NAME##_au(dst, src, num_elems); \
} else if (hex_is_aligned((void *) src, 128)) { \
OP_NAME##_ua(dst, src, num_elems); \
} else { \
OP_NAME##_uu(dst, src, num_elems); \
} \
}
DEFINE_HVX_INV_OP_VARIANTS(hvx_inverse_f32, hvx_inverse_f32_loop_body)
ggml/src/ggml-hexagon/htp/hvx-scale.h
+22 -22
View File
@@ -68,30 +68,30 @@ static inline void hvx_scale_f32(uint8_t * restrict dst, const uint8_t * restric
}
}
#define hvx_scale_offset_f32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
HVX_Vector vs = hvx_vec_splat_f32(scale); \
HVX_Vector vo = hvx_vec_splat_f32(offset); \
\
const uint32_t elem_size = sizeof(float); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; ++i) { \
#define hvx_scale_offset_f32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
HVX_Vector vs = hvx_vec_splat_f32(scale); \
HVX_Vector vo = hvx_vec_splat_f32(offset); \
\
const uint32_t elem_size = sizeof(float); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; ++i) { \
HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \
vdst[i] = Q6_Vsf_equals_Vqf32(v); \
} \
if (nloe) { \
vdst[i] = Q6_Vsf_equals_Vqf32(v); \
} \
if (nloe) { \
HVX_Vector v = Q6_Vqf32_vadd_Vqf32Vsf(Q6_Vqf32_vmpy_VsfVsf(vsrc[i], vs), vo); \
vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \
} \
vec_store((void *) &vdst[i], nloe * elem_size, Q6_Vsf_equals_Vqf32(v)); \
} \
} while(0)
static inline void hvx_scale_offset_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, const int n, const float scale, const float offset) {
ggml/src/ggml-hexagon/htp/hvx-sigmoid.h
+40 -40
View File
@@ -68,50 +68,50 @@ static inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) {
return Q6_Vsf_equals_Vqf32(res);
}
#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const HVX_Vector one = hvx_vec_splat_f32(1.f); \
const HVX_Vector max_exp = hvx_vec_splat_f32(87.f); \
const HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); \
\
const uint32_t epv = 128 / sizeof(float); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \
} \
if (nloe) { \
#define hvx_sigmoid_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const HVX_Vector one = hvx_vec_splat_f32(1.f); \
const HVX_Vector max_exp = hvx_vec_splat_f32(87.f); \
const HVX_Vector min_exp = hvx_vec_splat_f32(-87.f); \
\
const uint32_t epv = 128 / sizeof(float); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \
} \
if (nloe) { \
HVX_Vector tmp = hvx_vec_fast_sigmoid_f32_guard(vsrc[i], one, max_exp, min_exp); \
vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \
} \
vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \
} \
} while(0)
#define hvx_tanh_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t epv = 128 / sizeof(float); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_tanh_f32(vsrc[i]); \
} \
if (nloe) { \
HVX_Vector tmp = hvx_vec_tanh_f32(vsrc[i]); \
#define hvx_tanh_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t epv = 128 / sizeof(float); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_tanh_f32(vsrc[i]); \
} \
if (nloe) { \
HVX_Vector tmp = hvx_vec_tanh_f32(vsrc[i]); \
vec_store((void *) &vdst[i], nloe * sizeof(float), tmp); \
} \
} \
} while(0)
static inline void hvx_sigmoid_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
ggml/src/ggml-hexagon/htp/im2col-ops.c
+69 -36
View File
@@ -3,11 +3,12 @@
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <HAP_perf.h>
#include <hexagon_protos.h>
#include <hexagon_types.h>
#include <string.h>
#include "hex-common.h"
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "htp-ctx.h"
@@ -16,14 +17,19 @@
#include "hex-dma.h"
#include "hex-profile.h"
#include "htp-vtcm.h"
#include "htp-tensor.h"
struct htp_im2col_context {
struct htp_ops_context * octx;
uint32_t patch_base; // first patch index assigned to this dev
uint32_t npatches; // number of patches assigned to this dev
uint32_t npatches_per_thread; // patches = N*OH*OW (pure-DDR kernel)
uint32_t pe_rows_per_thread; // N*OH rows per worker
uint32_t pe_src_row_bytes; // one output row's source: IC*KH*IW*4, rounded 256
uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256
uint32_t pe_row_base; // first N*OH row index assigned to this dev (DMA path)
uint32_t pe_nrows; // number of N*OH rows assigned to this dev (DMA path)
uint32_t pe_rows_per_thread; // N*OH rows per worker
uint32_t pe_src_row_bytes; // one output row's source: IC*KH*IW*4, rounded 256
uint32_t pe_dst_row_bytes; // one output row's dst: OW*patch_stride*2, rounded 256
// Patch-embed DMA path VTCM ping-pong.
uint8_t * pe_vtcm_src; // base of the 2x src buffers region
@@ -58,33 +64,27 @@ static inline void htp_im2col_vtcm_layout_build(struct htp_im2col_vtcm_layout *
struct htp_im2col_context * ictx = (struct htp_im2col_context *) data; \
struct htp_ops_context * octx = ictx->octx; \
struct htp_thread_trace * restrict tr = &octx->ctx->trace[ith]; \
const struct htp_tensor * restrict src0 = octx->src[0]; \
const struct htp_tensor * restrict src1 = octx->src[1]; \
const struct htp_tensor * restrict dst = octx->dst; \
const int32_t s0 = octx->op_params[0]; \
const int32_t s1 = octx->op_params[1]; \
const int32_t p0 = octx->op_params[2]; \
const int32_t p1 = octx->op_params[3]; \
const int32_t d0 = octx->op_params[4]; \
const int32_t d1 = octx->op_params[5]; \
const uint32_t N = src1->ne[3]; \
const uint32_t IC = src1->ne[2]; \
const uint32_t IH = src1->ne[1]; \
const uint32_t IW = src1->ne[0]; \
const uint32_t KH = octx->src[0]->ne[1]; \
const uint32_t KW = octx->src[0]->ne[0]; \
const int32_t s0 = octx->op_params[0], s1 = octx->op_params[1]; \
const int32_t p0 = octx->op_params[2], p1 = octx->op_params[3]; \
const int32_t d0 = octx->op_params[4], d1 = octx->op_params[5]; \
const uint32_t N = src1->ne[3], IC = src1->ne[2], IH = src1->ne[1], IW = src1->ne[0]; \
const uint32_t KH = src0->ne[1], KW = src0->ne[0]; \
const uint32_t OH = dst->ne[2]; \
const uint32_t OW = dst->ne[1]; \
const uint32_t patch_stride = IC * KH * KW; \
const float * restrict src_data = (const float *) src1->data; \
DST_CTYPE * restrict dst_data = (DST_CTYPE *) dst->data; \
const uint32_t npatches = N * OH * OW; \
const uint32_t patch_start = ictx->npatches_per_thread * ith; \
const uint32_t patch_end = MIN(patch_start + ictx->npatches_per_thread, npatches); \
if (patch_start >= patch_end) { \
const uint32_t patch_end = ictx->patch_base + ictx->npatches; \
const uint32_t patch_start = ictx->patch_base + ictx->npatches_per_thread * ith; \
const uint32_t patch_stop = MIN(patch_start + ictx->npatches_per_thread, patch_end);\
if (patch_start >= patch_stop) { \
return; \
} \
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, patch_start); \
for (uint32_t p = patch_start; p < patch_end; p++) { \
for (uint32_t p = patch_start; p < patch_stop; p++) { \
const uint32_t iow = p % OW; \
const uint32_t ioh = (p / OW) % OH; \
const uint32_t in = p / (OW * OH); \
@@ -154,10 +154,10 @@ IM2COL_PATCHEMBED_BODY(im2col_patchembed_f32_thread, float, hvx_copy_f32_uu, hvx
uint8_t * dst_base = ictx->pe_vtcm_dst + ith * ictx->pe_dst_size_per_thread; \
float * srcb = (float *) src_base; \
DST_CTYPE * dstb = (DST_CTYPE *) dst_base; \
const uint32_t nrows = N * OH; \
const uint32_t row_end_max = ictx->pe_row_base + ictx->pe_nrows; \
const uint32_t per_thread = ictx->pe_rows_per_thread; \
const uint32_t row_start = per_thread * ith; \
const uint32_t row_end = MIN(row_start + per_thread, nrows); \
const uint32_t row_start = ictx->pe_row_base + per_thread * ith; \
const uint32_t row_end = MIN(row_start + per_thread, row_end_max); \
if (row_start >= row_end) \
return; \
for (uint32_t r = row_start; r < row_end; r++) { \
@@ -266,26 +266,55 @@ int op_im2col(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
const uint32_t N = src1->ne[3];
const uint32_t OH = dst->ne[2];
const uint32_t OW = dst->ne[1];
const uint32_t npatches = N * OH * OW;
const uint32_t n_threads = MIN(octx->n_threads, npatches);
if ((octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) || n_threads == 0) {
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
return HTP_STATUS_OK;
}
const uint32_t N = src1->ne[3];
const uint32_t OH = dst->ne[2];
const uint32_t OW = dst->ne[1];
const uint32_t total_patches = N * OH * OW;
const uint32_t total_rows = N * OH;
uint32_t patch_base = 0;
uint32_t npatches = total_patches;
if (octx->ctx->mdev.count > 1) {
const uint32_t patch_size = dst->nb[1];
const uint32_t patches_per_chunk = (patch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(patch_size, HEX_L2_LINE_SIZE)) : 1;
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_patches, htp_tensor_mdev_data_aligned(dst) ? patches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
patch_base = range.start;
npatches = range.count;
}
uint32_t row_base = 0;
uint32_t nrows = total_rows;
if (octx->ctx->mdev.count > 1) {
const uint32_t row_size = dst->nb[2];
const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, htp_tensor_mdev_data_aligned(dst) ? rows_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_base = range.start;
nrows = range.count;
}
if (npatches == 0 && nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = MIN(octx->n_threads, MAX(npatches, 1));
struct htp_im2col_context ictx = { 0 };
ictx.octx = octx;
ictx.npatches_per_thread = (npatches + n_threads - 1) / n_threads;
ictx.octx = octx;
ictx.patch_base = patch_base;
ictx.npatches = npatches;
ictx.npatches_per_thread = (npatches + n_threads - 1) / n_threads;
// Clean non-overlapping patch-embed -> DMA kernel (if it fits VTCM);
// everything else (padding/dilation/stride edges) -> pure-DDR kernel.
if (im2col_use_patchembed_dma(octx)) {
const uint32_t nrows = N * OH;
const uint32_t pth = MIN(octx->n_threads, nrows);
if (im2col_use_patchembed_dma(octx) && nrows > 0) {
const uint32_t pth = MIN(octx->n_threads, nrows);
if (pth > 0 && im2col_patchembed_dma_fits(octx, &ictx, pth)) {
ictx.pe_row_base = row_base;
ictx.pe_nrows = nrows;
ictx.pe_rows_per_thread = (nrows + pth - 1) / pth;
if (dst->type == HTP_TYPE_F16) {
work_queue_run(octx->ctx->work_queue, im2col_patchembed_dma_thread, &ictx, pth);
@@ -297,6 +326,10 @@ int op_im2col(struct htp_ops_context * octx) {
// else: doesn't fit -> fall through to the pure-DDR kernel below.
}
if (npatches == 0) {
return HTP_STATUS_OK;
}
if (dst->type == HTP_TYPE_F16) {
work_queue_run(octx->ctx->work_queue, im2col_patchembed_thread, &ictx, n_threads);
} else {
ggml/src/ggml-hexagon/htp/main.c
+80 -23
View File
@@ -34,6 +34,7 @@
#include "work-queue.h"
#include "hex-profile.h"
#include "allreduce-ops.h"
#include "htp-fence.h"
#define HMX_QUEUE_CAPACITY 16
#define HMX_QUEUE_STACK_SIZE 16384
@@ -710,22 +711,43 @@ static inline void profile_stop(uint32_t mode, struct profile_data * d) {
static int op_fence(struct htp_ops_context * octx) {
struct htp_context *ctx = octx->ctx;
struct htp_thread_trace * tr = &ctx->trace[0];
const uint32_t seq = (uint32_t) octx->op_params[0];
const uint32_t seq = (uint32_t) octx->op_params[0];
const uint32_t mode = (uint32_t) octx->op_params[1];
htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
const struct htp_tensor * sync = octx->src[0];
atomic_uint * sync_fence = (atomic_uint *) sync->data;
atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data;
if (mode == 1) {
htp_flush_dirty_ranges(ctx);
htp_mdev_group_barrier(octx);
if (ctx->mdev.idx == 0) {
htp_fence_write(sync_fence, seq, octx->status);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
FARF(HIGH, "ggml-hex: sync-signal : fence %p seq 0x%x status %d\n", sync_fence, seq, octx->status);
return octx->status;
}
int status = HTP_STATUS_OK;
uint64_t spins = 0;
while (1) {
Q6_dccleaninva_A((void *) sync_fence);
asm volatile ("syncht" : : : "memory");
uint32_t val = atomic_load(&sync_fence[0]);
if ((int32_t)(val - seq) >= 0) {
uint32_t sync_seq;
uint32_t sync_status;
htp_fence_read(sync_fence, &sync_seq, &sync_status);
if ((int32_t)(sync_seq - seq) >= 0) {
if (sync_status > HTP_STATUS_OK) {
FARF(ERROR, "ggml-hex: sync-wait peer failed with status %u : fence %p seq 0x%x\n", sync_status, sync_fence, seq);
status = sync_status;
}
break;
}
if (++spins > HTP_FENCE_TIMEOUT) {
FARF(ERROR, "ggml-hex: sync-wait TIMEOUT : fence %p spins %llu seq %u\n", sync_fence, spins, seq);
FARF(ERROR, "ggml-hex: sync-wait TIMEOUT : fence %p spins %llu seq 0x%x\n", sync_fence, spins, seq);
status = HTP_STATUS_INTERNAL_ERR;
break;
}
hex_pause();
@@ -733,12 +755,27 @@ static int op_fence(struct htp_ops_context * octx) {
htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, (uint16_t) seq);
FARF(HIGH, "ggml-hex: sync-done : fence %p spins %llu seq %u\n", sync_fence, spins, seq);
FARF(HIGH, "ggml-hex: sync-done : fence %p spins %llu seq 0x%x\n", sync_fence, spins, seq);
return status;
}
static int op_mdev_group(struct htp_ops_context * octx) {
struct htp_context * ctx = octx->ctx;
const struct htp_tensor * sync = octx->src[0];
ctx->mdev.idx = (uint16_t) octx->op_params[0];
ctx->mdev.count = (uint16_t) sync->ne[1];
if (ctx->mdev.count > 1) {
ctx->mdev.count_div = init_fastdiv_values(ctx->mdev.count);
ctx->mdev.fence_base = (uint8_t *) sync->data;
}
return HTP_STATUS_OK;
}
static int execute_op(struct htp_ops_context * octx) {
switch (octx->op) {
case HTP_OP_MDEV_GROUP:
return op_mdev_group(octx);
case HTP_OP_FENCE:
return op_fence(octx);
@@ -812,6 +849,7 @@ static int execute_op(struct htp_ops_context * octx) {
return op_sum_rows(octx);
case HTP_OP_CPY:
case HTP_OP_CPY_FENCE:
return op_cpy(octx);
case HTP_OP_REPEAT:
@@ -855,7 +893,7 @@ static int execute_op(struct htp_ops_context * octx) {
}
FARF(ERROR, "Unknown Op %u", octx->op);
return -1;
return HTP_STATUS_NO_SUPPORT;
}
static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct htp_buf_desc *b) {
@@ -984,11 +1022,19 @@ static void prep_tensors(struct htp_context *ctx, struct htp_buf_desc *bufs, str
}
}
static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, uint32_t idx, struct htp_op_desc * op) {
memcpy(octx->op_params, op->params, sizeof(octx->op_params));
static void mdev_group_init(struct htp_context * ctx, const struct htp_opbatch_req * req) {
memset(&ctx->mdev, 0, sizeof(ctx->mdev));
ctx->mdev.fence_seq = (uint32_t)((req->seq & 0xfffff) << 12);
}
static int proc_op_req(struct htp_ops_context * octx, struct htp_buf_desc * bufs, uint32_t n_bufs,
struct htp_tensor * tens, uint32_t idx, struct htp_op_desc * op) {
memcpy(octx->op_params, op->params, sizeof(octx->op_params));
memcpy(octx->kernel_params, op->kernel_params, sizeof(octx->kernel_params));
octx->flags = op->flags;
octx->op = op->opcode;
octx->flags = op->flags;
octx->op = op->opcode;
octx->n_threads = octx->ctx->n_threads;
octx->n_threads_div = octx->ctx->n_threads_div;
FARF(HIGH, "proc-op #%u: opcode %u flags 0x%x", idx, octx->op, octx->flags);
@@ -1027,9 +1073,13 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
}
htp_tensor_dirty_all(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS);
htp_mdev_group_barrier(octx);
int status = execute_op(octx);
htp_tensor_dirty_all(octx->ctx, octx->dsts, HTP_OP_MAX_OUTPUTS);
htp_ops_context_set_status(octx, status);
octx->src0_spad.src = NULL;
octx->src1_spad.src = NULL;
@@ -1037,7 +1087,7 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u
octx->src3_spad.src = NULL;
octx->dst_spad.src = NULL;
return status;
return octx->status;
}
static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_req * req, const struct dspqueue_buffer * dbuf) {
@@ -1059,7 +1109,7 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
return;
}
FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", req->id,
FARF(HIGH, "processing opbatch #%llu: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", (unsigned long long) req->seq,
n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size);
// Setup descriptor pointers
@@ -1096,8 +1146,11 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
struct htp_ops_context *octx = &ctx->octx;
memset(octx, 0, sizeof(*octx));
octx->n_threads = ctx->n_threads;
octx->ctx = ctx;
octx->n_threads = ctx->n_threads;
octx->n_threads_div = ctx->n_threads_div;
octx->ctx = ctx;
mdev_group_init(ctx, req);
work_queue_wakeup(ctx->work_queue);
if (ctx->hmx_queue) {
@@ -1105,15 +1158,18 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
}
int op_status = HTP_STATUS_OK;
for (uint32_t i = 0; i < n_ops && op_status == HTP_STATUS_OK; i++) {
octx->status = HTP_STATUS_OK;
for (uint32_t i = 0; i < n_ops; i++) {
struct profile_data prof;
profile_start(ctx->profiler, &prof);
op_status = proc_op_req(octx, tens, i, &ops[i]);
op_status = proc_op_req(octx, bufs, n_bufs, tens, i, &ops[i]);
profile_stop(ctx->profiler, &prof);
htp_ops_context_set_status(octx, op_status);
if (ctx->profiler) {
pds[i].opcode = ops[i].opcode;
pds[i].usecs = prof.usecs;
@@ -1136,19 +1192,20 @@ static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_r
qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);
htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0);
htp_mdev_group_barrier(octx);
profile_stop(HTP_PROF_BASIC, &batch_prof);
struct htp_opbatch_rsp rsp;
memset(&rsp, 0, sizeof(rsp));
rsp.id = req->id;
rsp.status = op_status;
rsp.seq = req->seq;
rsp.status = octx->status;
rsp.n_bufs = n_bufs;
rsp.n_tensors = n_tens;
rsp.n_ops = n_ops;
rsp.usecs = batch_prof.usecs;
rsp.cycles_start = batch_prof.cycles_start;
rsp.cycles_stop = batch_prof.cycles_stop;
rsp.seq = req->seq;
if (ctx->profiler == HTP_PROF_TRACE) {
for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
ggml/src/ggml-hexagon/htp/matmul-ops.c
+270 -101
View File
@@ -21,6 +21,7 @@
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "matmul-ops.h"
#include "htp-vtcm.h"
@@ -89,6 +90,8 @@ struct htp_mm_context {
// Precomputed values
uint32_t src0_nrows_per_thread;
uint32_t src0_row_start;
uint32_t src0_row_end;
uint32_t src0_row_size_padded;
uint32_t src1_nrows;
@@ -135,6 +138,23 @@ struct htp_mm_context {
uint32_t vtcm_dst_size_per_thread;
};
static int htp_mm_init_context(
struct htp_ops_context * octx,
const struct htp_mm_kernel_params * kparams
) {
if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
if (kparams->n_hmx) {
if (kparams->n_act_threads <= 0 || kparams->n_act_threads > (int32_t) octx->n_threads) {
return HTP_STATUS_INVAL_PARAMS;
}
}
return HTP_STATUS_OK;
}
// vdelta control to expand first 32 e8m0 values into 32 uint32 elements
static const uint8_t __attribute__((aligned(128))) expand_x32_e8m0[128] = {
0x00, 0x00, 0x00, 0x00, 0x01, 0x04, 0x00, 0x00, 0x02, 0x00, 0x08, 0x08, 0x01, 0x02, 0x00, 0x04, 0x04, 0x00, 0x00,
@@ -238,22 +258,24 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) {
// This is the size of the rest of the dimensions of the result
const uint32_t nr1 = ne1 * ne2 * ne3;
const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start;
// distribute the thread work across the inner or outer loop based on which one is larger
uint32_t dr0, dr1, ith0, ith1;
if (nr0 > nr1) {
dr0 = fastdiv(nr0 + nth - 1, &octx->ctx->n_threads_div);
dr0 = fastdiv(src0_nrows + nth - 1, &octx->n_threads_div);
dr1 = nr1;
ith0 = ith;
ith1 = 0;
} else {
dr0 = nr0;
dr1 = fastdiv(nr1 + nth - 1, &octx->ctx->n_threads_div);
dr0 = src0_nrows;
dr1 = fastdiv(nr1 + nth - 1, &octx->n_threads_div);
ith0 = 0;
ith1 = ith;
}
const uint32_t ir0_start = dr0 * ith0;
const uint32_t ir0_end = MIN(ir0_start + dr0, nr0);
const uint32_t ir0_start = mmctx->src0_row_start + dr0 * ith0;
const uint32_t ir0_end = MIN(ir0_start + dr0, mmctx->src0_row_end);
const uint32_t ir1_start = dr1 * ith1;
const uint32_t ir1_end = MIN(ir1_start + dr1, nr1);
@@ -312,11 +334,11 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) {
static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
htp_matmul_preamble; \
\
const uint32_t src0_nrows = ne01 * ne02 * ne03; \
const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; \
const uint32_t src1_nrows = ne11 * ne12 * ne13; \
\
const uint32_t src0_start_row = src0_nrows_per_thread * ith; \
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \
const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; \
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); \
\
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
\
@@ -414,10 +436,10 @@ static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void
static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) { \
htp_matmul_preamble; \
\
const uint32_t src0_nrows = ne01; \
const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; \
\
const uint32_t src0_start_row = src0_nrows_per_thread * ith; \
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \
const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith; \
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end); \
\
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
\
@@ -549,12 +571,22 @@ static void hvx_mm_nx_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, v
uint32_t n_k_tiles_w = ne00 / 32; \
uint32_t tile_row_stride = n_k_tiles_w * tile_size; \
\
const uint32_t src0_nrows = ne01 * src_w->ne[2] * src_w->ne[3]; \
uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div); \
uint32_t src0_start_row = 0; \
uint32_t src0_end_row = ne01; \
if (octx->ctx->mdev.count > 1) { \
const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float)); \
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, \
octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); \
src0_start_row = range.start; \
src0_end_row = range.start + range.count; \
} \
\
const uint32_t nrows = src0_end_row - src0_start_row; \
uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div); \
src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32); \
\
const uint32_t start_row = src0_nrows_per_thread * ith; \
const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_nrows); \
const uint32_t start_row = src0_start_row + src0_nrows_per_thread * ith; \
const uint32_t end_row = MIN(start_row + src0_nrows_per_thread, src0_end_row); \
if (start_row >= end_row) continue; \
\
uint32_t ct_start = start_row / 32; \
@@ -735,11 +767,11 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) {
assert(n_prefetch >= 2 && n_prefetch <= HTP_MM_MAX_PREFETCH && (n_prefetch & (n_prefetch - 1)) == 0);
const uint32_t prefetch_mask = n_prefetch - 1;
const uint32_t src0_nrows = ne01 * ne02 * ne03; // src0 rows
const uint32_t src1_nrows = ne11 * ne12 * ne13; // src1 rows
const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows
const uint32_t src1_nrows = ne11 * ne12 * ne13; // src1 rows
const uint32_t src0_start_row = src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end);
const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U);
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
@@ -781,7 +813,7 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) {
const uint8_t * ss0 = dma_queue_pop(dma_queue).dst;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0);
// Process src1 columns in pairs (2×2 tiling)
// Process src1 columns in pairs (2x2 tiling)
uint32_t ir1 = 0;
for (; ir1 + 1 < src1_nrows; ir1 += 2) {
const uint8_t * restrict src1_col0 = (const uint8_t *) (src1_data + (ir1+0) * src1_stride);
@@ -791,7 +823,7 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) {
mmctx->vec_dot_2x2(ne00, &dst_row0[ir0], &dst_row1[ir0], ss0, ss0 + src0_stride, src1_col0, src1_col1);
}
// Handle remaining src1 rows (fallback to 2×1)
// Handle remaining src1 rows (fallback to 2x1)
for (; ir1 < src1_nrows; ++ir1) {
const uint8_t * restrict src1_col = (const uint8_t *) (src1_data + ir1 * src1_stride);
float * restrict dst_row = (float *) (dst->data + (ir1 * dst_row_size));
@@ -833,10 +865,10 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) {
static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) {
htp_matmul_preamble;
const uint32_t src0_nrows = ne01;
const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start;
const uint32_t src0_start_row = src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end);
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
@@ -943,13 +975,10 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) {
const struct htp_tensor * restrict ids = octx->src[2];
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
const uint32_t src0_nrows = ne01; // src0 rows per expert
const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert
const uint32_t src1_nrows = ne11;
const uint32_t src0_start_row = src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end);
hvx_mm_run_quant_task(mmctx, ith);
@@ -1036,9 +1065,9 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) {
const struct htp_tensor * restrict ids = octx->src[2];
const uint32_t src0_nrows = ne01; // src0 rows per expert
const uint32_t src0_start_row = src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
const uint32_t src0_nrows = mmctx->src0_row_end - mmctx->src0_row_start; // src0 rows per expert
const uint32_t src0_start_row = mmctx->src0_row_start + src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, mmctx->src0_row_end);
hvx_mm_run_quant_task(mmctx, ith);
@@ -1143,12 +1172,22 @@ static void hvx_mv_id_nx(unsigned int nth, unsigned int ith, void * data) {
const struct htp_tensor * restrict dst = octx->dsts[p];
if (!src_w || !dst) continue;
const uint32_t src0_nrows = src_w->ne[1];
uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div);
const uint32_t ne01 = src_w->ne[1];
uint32_t start_row = 0;
uint32_t end_row = ne01;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float));
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
start_row = range.start;
end_row = range.start + range.count;
}
const uint32_t nrows = end_row - start_row;
uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div);
src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32);
const uint32_t src0_start_row = src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row);
if (src0_start_row >= src0_end_row) continue;
const uint8_t * restrict src0_row = (const uint8_t *) src_w->data + eid * src_w->nb[2];
@@ -1227,12 +1266,22 @@ static void hvx_mm_id_nx(unsigned int nth, unsigned int ith, void * data) {
const struct htp_tensor * restrict dst = octx->dsts[p];
if (!src_w || !dst) continue;
const uint32_t src0_nrows = src_w->ne[1];
uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div);
const uint32_t ne01 = src_w->ne[1];
uint32_t start_row = 0;
uint32_t end_row = ne01;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float));
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
start_row = range.start;
end_row = range.start + range.count;
}
const uint32_t nrows = end_row - start_row;
uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div);
src0_nrows_per_thread = hex_round_up(src0_nrows_per_thread, 32);
const uint32_t src0_start_row = src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row);
if (src0_start_row >= src0_end_row) continue;
const uint8_t * src0_row = (const uint8_t *) src_w->data + cur_a * src_w->nb[2];
@@ -1323,15 +1372,33 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
const uint32_t src0_nrows = ne01 * ne02 * ne03;
const uint32_t src0_nrows = ne01;
const uint32_t src1_nrows = ne11 * ne12 * ne13;
uint32_t src0_row_start = 0;
uint32_t src0_row_end = src0_nrows;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float));
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
src0_row_start = range.start;
src0_row_end = range.start + range.count;
}
if (src0_row_start >= src0_row_end) {
return HTP_STATUS_OK;
}
const uint32_t nrows = src0_row_end - src0_row_start;
mmctx->src0_row_start = src0_row_start;
mmctx->src0_row_end = src0_row_end;
bool is_repacked = (src0->type == HTP_TYPE_Q4_0 || src0->type == HTP_TYPE_Q4_1 ||
src0->type == HTP_TYPE_Q8_0 || src0->type == HTP_TYPE_IQ4_NL ||
src0->type == HTP_TYPE_MXFP4);
// Compute src0_nrows_per_thread
mmctx->src0_nrows_per_thread = fastdiv(src0_nrows + octx->n_threads - 1, &octx->ctx->n_threads_div);
mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div);
if (is_repacked) {
mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32);
} else {
@@ -1503,13 +1570,13 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
kparams->kernel_type == HTP_MM_KERNEL_HVX_QUANT_BLOCK) {
mmctx->vtcm_src1_size_per_thread = L.src1_bytes;
} else {
mmctx->vtcm_src1_size_per_thread = fastdiv(L.src1_bytes, &octx->ctx->n_threads_div);
mmctx->vtcm_src1_size_per_thread = fastdiv(L.src1_bytes, &octx->n_threads_div);
}
mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div);
mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div);
mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);
mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div);
size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes;
const size_t vtcm_size = L.total_bytes;
FARF(HIGH, "matmul-%s : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu (%zu)\n", mmctx->type,
L.src0_bytes, L.src1_bytes, L.dst_bytes, vtcm_size);
@@ -1583,13 +1650,21 @@ static void hvx_mm_nx_2d(unsigned int nth, unsigned int ith, void * data) {
const uint32_t ne00 = src_w->ne[0];
const uint32_t ne01 = src_w->ne[1];
const uint32_t src0_nrows = ne01 * src_w->ne[2] * src_w->ne[3];
uint32_t start_row = 0;
uint32_t end_row = ne01;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float));
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(ne01, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
start_row = range.start;
end_row = range.start + range.count;
}
uint32_t src0_nrows_per_thread = fastdiv(src0_nrows + nth - 1, &octx->ctx->n_threads_div);
const uint32_t nrows = end_row - start_row;
uint32_t src0_nrows_per_thread = fastdiv(nrows + nth - 1, &octx->n_threads_div);
src0_nrows_per_thread += (src0_nrows_per_thread & 1);
const uint32_t src0_start_row = src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
const uint32_t src0_start_row = start_row + src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, end_row);
const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U);
if (src0_start_row >= src0_end_row) continue;
@@ -2638,10 +2713,6 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
const struct htp_tensor * restrict src0 = octx->src[0];
const struct htp_tensor * restrict act = octx->src[n_weights];
if (!src0 || !act) {
return HTP_STATUS_INVAL_PARAMS;
}
const int weight_type = (int) src0->type;
const int k = (int) act->ne[0];
const int k_valid = (int) act->ne[0];
@@ -2714,16 +2785,31 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16
FARF(HIGH, "hmx-mm-nx-2d: n_weights %u m %d k %d wtype %d mc %d nc %d vtcm %zu/%zu",
n_weights, m, k, weight_type, m_chunk_n_rows, n_chunk_n_cols, L.total_bytes, vtcm_budget);
int m_start = 0;
int m_rows = m;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_can_row_partition(octx->dsts[0], sizeof(float));
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
m_start = (int) range.start;
m_rows = (int) range.count;
}
if (m_rows == 0) {
return HTP_STATUS_OK;
}
FARF(HIGH, "hmx-mm-nx-2d: n_weights %u m %d (%d..%d) k %d wtype %d mc %d nc %d vtcm %zu/%zu",
n_weights, m, m_start, m_start + m_rows, k, weight_type, m_chunk_n_rows, n_chunk_n_cols, L.total_bytes, vtcm_budget);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
const size_t mr_end = (size_t)(m_start + m_rows);
if (pipeline) {
hmx_matmul_job_t job_slots[2];
for (size_t mr = 0; mr < (size_t) m; mr += m_chunk_n_rows) {
const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows);
for (size_t mr = (size_t) m_start; mr < mr_end; mr += m_chunk_n_rows) {
const size_t n_rows = hex_smin(mr_end - mr, m_chunk_n_rows);
void *vtcm_weight_bufs[2] = { vtcm_scratch0, vtcm_scratch1 };
void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 };
@@ -2822,8 +2908,8 @@ static int hmx_mm_nx_2d_f32(struct htp_ops_context * octx, const struct htp_mm_k
}
} else {
hmx_matmul_job_t job;
for (size_t mr = 0; mr < (size_t) m; mr += m_chunk_n_rows) {
const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows);
for (size_t mr = (size_t) m_start; mr < mr_end; mr += m_chunk_n_rows) {
const size_t n_rows = hex_smin(mr_end - mr, m_chunk_n_rows);
struct activation_transfer_params act_params = {
.ctx = ctx,
@@ -3095,7 +3181,7 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_
int chunk_dst_cols = params->n - (int)nc;
if (chunk_dst_cols > 0) {
transfer_output_chunk_threaded(ctx, output, src2_chunk, vtcm_output, (int) n_rows, (int) n_cols,
params->dst_stride, params->src2_stride, chunk_dst_cols, ctx->n_threads);
params->dst_stride, params->src2_stride, chunk_dst_cols, n_threads);
}
}
}
@@ -3216,7 +3302,10 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
int weight_type,
const struct mmid_row_mapping *matrix_rows,
int cur_a,
int mapping_stride) {
int mapping_stride,
int m_start,
int m_end,
int n_threads) {
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
@@ -3247,7 +3336,6 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
const int n_k_tiles = k / HTP_MM_HMX_TILE_N_COLS;
const struct fastdiv_values n_k_tiles_div = init_fastdiv_values(n_k_tiles);
const int n_threads = ctx->n_threads;
const bool is_quant = (weight_type != HTP_TYPE_F16 && weight_type != HTP_TYPE_F32);
const size_t vec_dot_size = k * sizeof(__fp16);
@@ -3303,8 +3391,8 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
hmx_matmul_job_t job;
for (size_t mr = 0; mr < (size_t) m_padded; mr += m_chunk_n_rows) {
const size_t n_rows = hex_smin(m_padded - mr, m_chunk_n_rows);
for (size_t mr = (size_t) m_start; mr < (size_t) m_end; mr += m_chunk_n_rows) {
const size_t n_rows = hex_smin((size_t) m_end - mr, m_chunk_n_rows);
const size_t n_row_tiles = hmx_ceil_div(n_rows, HTP_MM_HMX_TILE_N_ROWS);
transfer_activation_chunk_gathered_threaded(
@@ -3368,31 +3456,48 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k
const int act_stride = (int)(src1->nb[1] / sizeof(float));
const int wgt_stride = (int)(src0->nb[1] / sizeof(__fp16));
int m_start = 0;
int m_rows = m_total;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float));
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_total, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
m_start = (int) range.start;
m_rows = (int) range.count;
}
if (m_rows == 0) {
return HTP_STATUS_OK;
}
const float * src2_ptr = NULL;
uint32_t src2_stride = 0;
size_t src2_nb2 = 0;
size_t src2_nb3 = 0;
if (src2) {
src2_ptr = (const float *) src2->data;
src2_stride = (src2->ne[1] == 1) ? 0 : (uint32_t) (src2->nb[1] / sizeof(float));
src2_ptr = (const float *) src2->data + m_start * src2_stride;
src2_nb2 = (src2->ne[2] == 1) ? 0 : src2->nb[2];
src2_nb3 = (src2->ne[3] == 1) ? 0 : src2->nb[3];
}
const int dst_stride = (int)(dst->nb[1] / sizeof(float));
float * dst_ptr = (float *) dst->data + m_start * dst_stride;
const float * act_ptr = (const float *) src1->data + m_start * act_stride;
int ret = -1;
const int n_threads = MIN(kparams->n_threads, (int) octx->n_threads);
const int n_threads = kparams->n_threads;
if (kparams->kernel_type == HTP_MM_KERNEL_HMX_F16_BATCHED) {
hmx_mm_f16_f32_batched_params_t batch_params = {
.dst = (float *) dst->data,
.dst = dst_ptr,
.src2 = src2_ptr,
.activation = (float *) src1->data,
.activation = act_ptr,
.weight = (const __fp16 *) src0->data,
.m = m_total,
.m = m_rows,
.k = k,
.n = n,
.act_stride = act_stride,
.weight_stride = wgt_stride,
.dst_stride = (int) (dst->nb[1] / sizeof(float)),
.dst_stride = dst_stride,
.src2_stride = src2_stride,
.ne02 = ne02,
.ne03 = ne03,
@@ -3420,9 +3525,9 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k
kparams->vtcm_size);
} else {
ret = hmx_mm_2d_f32(
octx->ctx, (float*) dst->data, src2_ptr, (float*) src1->data, (const uint8_t *) src0->data,
m_total, k, n, act_stride, (int) src0->nb[1], (int) src0->type, (int) src1->ne[0],
(int)(dst->nb[1] / sizeof(float)), src2_stride, (int)dst->ne[0],
octx->ctx, dst_ptr, src2_ptr, act_ptr, (const uint8_t *) src0->data,
m_rows, k, n, act_stride, (int) src0->nb[1], (int) src0->type, (int) src1->ne[0],
dst_stride, src2_stride, (int)dst->ne[0],
kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads,
kparams->n_act_threads,
&kparams->div_n_act_threads,
@@ -3441,6 +3546,11 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k
int op_matmul(struct htp_ops_context * octx) {
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
const int status = htp_mm_init_context(octx, kparams);
if (status != HTP_STATUS_OK) {
return status;
}
if (kparams->n_hmx) {
return hmx_mm_op_matmul(octx, kparams);
}
@@ -3463,6 +3573,16 @@ static int hmx_mm_op_matmul_id(
const int32_t cne1 = matrix_row_counts[cur_a];
if (cne1 == 0) continue;
const int m_padded = hex_align_up(cne1, 32);
int m_start = 0, m_end = m_padded;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_mdev_data_aligned(dst) && (uint32_t) cne1 >= octx->ctx->mdev.count;
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_padded, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
m_start = (int) range.start;
m_end = (int) (range.start + range.count);
}
if (m_start >= m_end) continue;
int ret = hmx_mm_id_2d_f32(octx->ctx, (float*) dst->data, (float*) src1->data,
(const uint8_t *) src0->data + cur_a * nb02,
cne1, ne00, ne01,
@@ -3471,7 +3591,8 @@ static int hmx_mm_op_matmul_id(
nb11, nb12,
nb1, nb2,
(int) src0->nb[1], (int) src0->type,
matrix_rows, cur_a, mmctx->mapping_stride);
matrix_rows, cur_a, mmctx->mapping_stride,
m_start, m_end, (int) octx->n_threads);
if (ret != 0) {
FARF(ERROR, "HMX matmul failed for expert %u, error %d\n", cur_a, ret);
return HTP_STATUS_NO_SUPPORT;
@@ -3524,7 +3645,7 @@ static int hvx_mm_matmul_id(
htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, ne10, src1_nrows, octx->n_threads,
0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false);
size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes;
const size_t vtcm_size = L.total_bytes;
FARF(HIGH, "matmul-id-%s : src0-spad-size %zu src1-spad-size %zu src2-spad-size 0 dst-spad-size %zu (%zu)\n", mmctx->type,
L.src0_bytes, L.src1_bytes, L.dst_bytes, vtcm_size);
@@ -3554,10 +3675,10 @@ static int hvx_mm_matmul_id(
mmctx->vtcm_src0_stride = src0_row_size_padded;
mmctx->vtcm_src1_stride = src1_row_size;
mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div);
mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);
mmctx->vtcm_src1_size_per_thread = L.src1_bytes;
mmctx->vtcm_src2_size_per_thread = 0;
mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div);
mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div);
mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks;
mmctx->quant_task_func = quant_task_func;
@@ -3587,6 +3708,20 @@ static int hmx_mm_op_matmul_id_nx(
const int32_t cne1 = matrix_row_counts[cur_a];
if (cne1 == 0) continue;
const int m_padded = hex_align_up(cne1, 32);
int m_start = 0, m_end = m_padded;
if (octx->ctx->mdev.count > 1) {
bool can_split = (uint32_t) cne1 >= octx->ctx->mdev.count;
for (uint32_t p = 0; p < n_weights && can_split; ++p) {
const struct htp_tensor * restrict dst = octx->dsts[p];
can_split = !dst || htp_tensor_mdev_data_aligned(dst);
}
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition((uint32_t) m_padded, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
m_start = (int) range.start;
m_end = (int) (range.start + range.count);
}
if (m_start >= m_end) continue;
for (uint32_t p = 0; p < n_weights; ++p) {
const struct htp_tensor * restrict src_w = octx->src[p];
const struct htp_tensor * restrict dst = octx->dsts[p];
@@ -3600,7 +3735,8 @@ static int hmx_mm_op_matmul_id_nx(
act->nb[1], act->nb[2],
dst->nb[1], dst->nb[2],
(int) src_w->nb[1], (int) src_w->type,
matrix_rows, cur_a, mmctx->mapping_stride);
matrix_rows, cur_a, mmctx->mapping_stride,
m_start, m_end, (int) octx->n_threads);
if (ret != 0) {
FARF(ERROR, "HMX matmul ID NX failed for expert %u weight %u, error %d\n", cur_a, p, ret);
return HTP_STATUS_NO_SUPPORT;
@@ -3656,7 +3792,7 @@ static int hvx_mm_matmul_id_nx(
htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], src1_nrows, octx->n_threads,
0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, true, false);
size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes;
const size_t vtcm_size = L.total_bytes;
if (octx->ctx->vtcm_size < vtcm_size) {
FARF(ERROR, "matmul-id-nx: current VTCM reservation %zu is too small, needed %zu\n",
@@ -3678,9 +3814,9 @@ static int hvx_mm_matmul_id_nx(
mmctx->vtcm_src0_stride = 0;
mmctx->vtcm_src1_stride = src1_row_size;
mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div);
mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);
mmctx->vtcm_src1_size_per_thread = L.src1_bytes;
mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div);
mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div);
mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks;
mmctx->quant_task_func = quant_task_func;
@@ -3769,16 +3905,21 @@ static inline void scan_expert_ids(
int op_matmul_id(struct htp_ops_context * octx) {
htp_matmul_tensors_preamble;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
struct htp_mm_context mmctx_struct = {0};
struct htp_mm_context * mmctx = &mmctx_struct;
const int status = htp_mm_init_context(octx, kparams);
if (status != HTP_STATUS_OK) {
return status;
}
struct htp_thread_trace * tr = &octx->ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
struct htp_mm_context mmctx_struct = {0};
struct htp_mm_context * mmctx = &mmctx_struct;
mmctx->octx = octx;
mmctx->act = src1;
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
const struct htp_tensor * restrict ids = octx->src[2];
const size_t src0_row_size = nb01;
@@ -3789,9 +3930,6 @@ int op_matmul_id(struct htp_ops_context * octx) {
const uint32_t src0_nrows = ne01; // per expert
const uint32_t src1_nrows = ne11 * ne12 * ne13;
mmctx->src0_nrows_per_thread = fastdiv(src0_nrows + octx->n_threads - 1, &octx->ctx->n_threads_div);
mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32);
// row groups
const int n_ids = ids->ne[0]; // n_expert_used
const int n_as = ne02; // n_expert
@@ -3843,6 +3981,29 @@ int op_matmul_id(struct htp_ops_context * octx) {
if (kparams->n_hmx) {
s = hmx_mm_op_matmul_id(octx, mmctx);
} else {
uint32_t src0_row_start = 0;
uint32_t src0_row_end = src0_nrows;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_can_row_partition(dst, sizeof(float));
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, can_split ? 32 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
src0_row_start = range.start;
src0_row_end = range.start + range.count;
}
if (src0_row_start >= src0_row_end) {
if (mapping_buf != octx->ctx->ddr_spad_base) {
free(mapping_buf);
}
return HTP_STATUS_OK;
}
const uint32_t nrows = src0_row_end - src0_row_start;
mmctx->src0_row_start = src0_row_start;
mmctx->src0_row_end = src0_row_end;
mmctx->src0_nrows_per_thread = fastdiv(nrows + octx->n_threads - 1, &octx->n_threads_div);
mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32);
if (hvx_mm_init_vec_dot(mmctx, src0->type) == 0) {
s = hvx_mm_matmul_id(octx, mmctx, src1_nrows > 1 ? hvx_mm_id : hvx_mv_id);
} else {
@@ -3858,29 +4019,31 @@ int op_matmul_id(struct htp_ops_context * octx) {
}
int op_matmul_id_nx(struct htp_ops_context * octx) {
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
struct htp_mm_context mmctx_struct = {0};
struct htp_mm_context * mmctx = &mmctx_struct;
const int status = htp_mm_init_context(octx, kparams);
if (status != HTP_STATUS_OK) {
return status;
}
struct htp_thread_trace * tr = &octx->ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
mmctx->octx = octx;
const uint32_t n_weights = kparams->n_weights;
const struct htp_tensor * restrict src0 = octx->src[0];
const struct htp_tensor * restrict act = octx->src[n_weights];
const struct htp_tensor * restrict ids = octx->src[n_weights + 1];
struct htp_mm_context mmctx_struct = {0};
struct htp_mm_context * mmctx = &mmctx_struct;
mmctx->octx = octx;
mmctx->act = act;
const size_t src0_row_size = src0->nb[1];
const size_t src0_row_size_padded = hex_round_up(src0_row_size, 128);
const uint32_t src0_nrows = src0->ne[1];
const uint32_t src1_nrows = act->ne[1] * act->ne[2] * act->ne[3];
mmctx->src0_nrows_per_thread = fastdiv(src0_nrows + octx->n_threads - 1, &octx->ctx->n_threads_div);
mmctx->src0_nrows_per_thread = hex_round_up(mmctx->src0_nrows_per_thread, 32);
const int n_ids = ids->ne[0];
const int n_as = src0->ne[2];
@@ -3946,6 +4109,12 @@ int op_matmul_id_nx(struct htp_ops_context * octx) {
}
int op_matmul_nx(struct htp_ops_context * octx) {
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
const int status = htp_mm_init_context(octx, kparams);
if (status != HTP_STATUS_OK) {
return status;
}
if (kparams->n_hmx) {
return hmx_mm_nx_2d_f32(octx, kparams);
}
@@ -4012,7 +4181,7 @@ int op_matmul_nx(struct htp_ops_context * octx) {
htp_mm_hvx_vtcm_layout_build(&L, kparams->kernel_type, src0->type, act->ne[0], src1_nrows, octx->n_threads,
0, src0_row_size, src1_row_size, 0, kparams->n_prefetch, false, true);
size_t vtcm_size = kparams->vtcm_size > 0 ? (size_t)kparams->vtcm_size : L.total_bytes;
const size_t vtcm_size = L.total_bytes;
if (octx->ctx->vtcm_size < vtcm_size) {
FARF(ERROR, "matmul-nx: current VTCM reservation %zu is too small, needed %zu\n",
@@ -4034,9 +4203,9 @@ int op_matmul_nx(struct htp_ops_context * octx) {
mmctx->vtcm_src0_stride = is_repacked ? 0 : src0_row_size_padded;
mmctx->vtcm_src1_stride = src1_row_size;
mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->ctx->n_threads_div);
mmctx->vtcm_src0_size_per_thread = fastdiv(L.src0_bytes, &octx->n_threads_div);
mmctx->vtcm_src1_size_per_thread = L.src1_bytes;
mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->ctx->n_threads_div);
mmctx->vtcm_dst_size_per_thread = fastdiv(L.dst_bytes, &octx->n_threads_div);
mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks;
mmctx->quant_task_func = quant_task_func;
ggml/src/ggml-hexagon/htp/pad-ops.c
+85 -74
View File
@@ -12,8 +12,11 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
#include "hex-profile.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
/* Circular wrap: maps any integer x into [0, n) */
static inline uint32_t wrap_around(int32_t x, uint32_t n) {
@@ -68,6 +71,7 @@ struct htp_pad_context {
uint32_t nrows_per_thread;
uint32_t total_dst_rows;
uint32_t row_start;
size_t type_size;
@@ -78,39 +82,39 @@ struct htp_pad_context {
size_t dst_row_size_aligned;
};
#define htp_pad_preamble \
const struct htp_tensor * src = octx->src[0]; \
const struct htp_tensor * dst = octx->dst; \
\
const uint32_t ne00 = src->ne[0]; \
const uint32_t nb00 = src->nb[0]; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
const uint32_t ne2 = dst->ne[2]; \
const uint32_t ne3 = dst->ne[3]; \
\
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3]; \
\
const int32_t lp0 = pctx->lp0, rp0 = pctx->rp0; \
const int32_t lp1 = pctx->lp1, rp1 = pctx->rp1; \
const int32_t lp2 = pctx->lp2, rp2 = pctx->rp2; \
const int32_t lp3 = pctx->lp3, rp3 = pctx->rp3; \
\
const size_t type_size = pctx->type_size; \
\
const uint32_t row_start = pctx->nrows_per_thread * ith; \
const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->total_dst_rows);
#define htp_pad_preamble \
const struct htp_tensor * src = octx->src[0]; \
const struct htp_tensor * dst = octx->dst; \
\
const uint32_t ne00 = src->ne[0]; \
const uint32_t nb00 = src->nb[0]; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
const uint32_t ne2 = dst->ne[2]; \
const uint32_t ne3 = dst->ne[3]; \
\
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3]; \
\
const int32_t lp0 = pctx->lp0, rp0 = pctx->rp0; \
const int32_t lp1 = pctx->lp1, rp1 = pctx->rp1; \
const int32_t lp2 = pctx->lp2, rp2 = pctx->rp2; \
const int32_t lp3 = pctx->lp3, rp3 = pctx->rp3; \
\
const size_t type_size = pctx->type_size; \
\
const uint32_t row_start = pctx->row_start + pctx->nrows_per_thread * ith; \
const uint32_t row_end = MIN(row_start + pctx->nrows_per_thread, pctx->row_start + pctx->total_dst_rows);
#define htp_pad_dma_preamble \
const size_t src_row_size = pctx->src_row_size; \
const size_t src_row_size_aligned = pctx->src_row_size_aligned; \
const size_t dst_row_size = pctx->dst_row_size; \
const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \
\
#define htp_pad_dma_preamble \
const size_t src_row_size = pctx->src_row_size; \
const size_t src_row_size_aligned = pctx->src_row_size_aligned; \
const size_t dst_row_size = pctx->dst_row_size; \
const size_t dst_row_size_aligned = pctx->dst_row_size_aligned; \
\
uint8_t * src_spad_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread; \
uint8_t * dst_spad_base = octx->dst_spad.data + ith * octx->dst_spad.size_per_thread; \
\
@@ -125,8 +129,8 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * da
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
uint32_t i1, i2, i3;
@@ -165,18 +169,17 @@ static void pad_job_per_thread_hvx(unsigned int nth, unsigned int ith, void * da
}
}
t2 = HAP_perf_get_qtimer_count();
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
FARF(HIGH, "pad-hvx %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n",
FARF(HIGH, "pad-hvx %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n",
ith, nth,
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end,
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
row_start, row_end);
}
// ---------------------------------------------------------------------------
// HVX + DMA PAD kernel — aligned, double-buffered
// HVX + DMA PAD kernel - aligned, double-buffered
// ---------------------------------------------------------------------------
static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void * data) {
@@ -185,9 +188,6 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void
htp_pad_preamble;
htp_pad_dma_preamble;
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
// -----------------------------------------------------------------------
// Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the
// double-buffer pipeline before the main loop begins.
@@ -222,6 +222,8 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void
// Main loop: pop completed DMAs, compute in VTCM with aligned HVX ops,
// push dst DMA and prefetch src for the next+1 row.
// -----------------------------------------------------------------------
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ir = row_start; ir < row_end; ir++) {
uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma).src;
uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma).dst;
@@ -236,6 +238,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void
lp2, rp2, ne2,
lp3, rp3, ne3);
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
if (!interior) {
hvx_splat_f32_a(dst_spad_cur, 0.0f, ne0);
} else {
@@ -249,6 +252,7 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void
hvx_copy_f32_ua(dst_interior, src_spad_cur, ne00);
}
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
dma_queue_push_vtcm_to_ddr(dma,
dma_make_ptr(dst_ptr, dst_spad_cur),
@@ -274,14 +278,11 @@ static void pad_job_per_thread_hvx_dma(unsigned int nth, unsigned int ith, void
dma_queue_flush(dma);
t2 = HAP_perf_get_qtimer_count();
FARF(HIGH, "pad-hvx-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n",
FARF(HIGH, "pad-hvx-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n",
ith, nth,
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end,
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
row_start, row_end);
}
// ---------------------------------------------------------------------------
@@ -293,8 +294,8 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith,
struct htp_ops_context * octx = pctx->octx;
htp_pad_preamble;
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
uint32_t i1, i2, i3;
@@ -344,18 +345,17 @@ static void pad_job_per_thread_hvx_circular(unsigned int nth, unsigned int ith,
}
}
t2 = HAP_perf_get_qtimer_count();
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
FARF(HIGH, "pad-hvx-circ %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n",
FARF(HIGH, "pad-hvx-circ %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n",
ith, nth,
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end,
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
row_start, row_end);
}
// ---------------------------------------------------------------------------
// HVX + DMA circular PAD kernel — aligned, double-buffered
// HVX + DMA circular PAD kernel - aligned, double-buffered
// ---------------------------------------------------------------------------
static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int ith, void * data) {
@@ -364,9 +364,6 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i
htp_pad_preamble;
htp_pad_dma_preamble;
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
// -----------------------------------------------------------------------
// Priming phase: push 2 pairs of (dummy_dst_DMA, src_DMA) to seed the
// double-buffer pipeline. Every row is a real src DMA (no null DMAs).
@@ -390,6 +387,8 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i
// Main loop: pop completed DMAs, assemble circular row in VTCM with
// aligned HVX ops, push dst DMA and prefetch src for the next+1 row.
// -----------------------------------------------------------------------
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
for (uint32_t ir = row_start; ir < row_end; ir++) {
uint8_t * dst_spad_cur = (uint8_t *) dma_queue_pop(dma).src;
uint8_t * src_spad_cur = (uint8_t *) dma_queue_pop(dma).dst;
@@ -398,7 +397,7 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i
pad_decompose_row(ir, ne1, ne2, &i1, &i2, &i3);
uint8_t * dst_ptr = (uint8_t *) dst->data + i1 * nb1 + i2 * nb2 + i3 * nb3;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
if (lp0 > 0) {
uint8_t * dst_left = dst_spad_cur;
const uint8_t * src_left = src_spad_cur + (size_t)(ne00 - (uint32_t)lp0) * type_size;
@@ -430,6 +429,7 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i
}
}
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir);
dma_queue_push_vtcm_to_ddr(dma,
dma_make_ptr(dst_ptr, dst_spad_cur),
@@ -448,14 +448,11 @@ static void pad_job_per_thread_hvx_circular_dma(unsigned int nth, unsigned int i
dma_queue_flush(dma);
t2 = HAP_perf_get_qtimer_count();
FARF(HIGH, "pad-hvx-circ-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n",
FARF(HIGH, "pad-hvx-circ-dma %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n",
ith, nth,
src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end,
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
row_start, row_end);
}
int op_pad(struct htp_ops_context * octx) {
@@ -489,19 +486,33 @@ int op_pad(struct htp_ops_context * octx) {
const uint32_t ne00 = src0->ne[0];
const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3];
const uint32_t n_threads = MIN(octx->n_threads, total_dst_rows > 0 ? total_dst_rows : 1);
const size_t dst_row_size = (size_t)ne0 * type_size;
uint32_t row_start = 0;
uint32_t nrows = total_dst_rows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_dst_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
const size_t src_row_size = (size_t)ne00 * type_size;
const size_t dst_row_size = (size_t)ne0 * type_size;
const size_t src_row_size_aligned = hex_round_up(src_row_size, VLEN);
const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);
// Total VTCM needed: 2 buffers (ping+pong) for src and dst, per thread
const size_t vtcm_needed = (size_t)n_threads * 2 * (src_row_size_aligned + dst_row_size_aligned);
const int use_dma = (src0->nb[0] == (uint32_t)type_size) &&
(ne00 >= 512) &&
(octx->ctx->vtcm_base != NULL) &&
const int use_dma = (src0->nb[0] == (uint32_t)type_size) && (ne00 >= 512) &&
(octx->ctx->vtcm_size >= vtcm_needed);
if (use_dma) {
@@ -521,8 +532,9 @@ int op_pad(struct htp_ops_context * octx) {
.lp1 = lp1, .rp1 = rp1,
.lp2 = lp2, .rp2 = rp2,
.lp3 = lp3, .rp3 = rp3,
.nrows_per_thread = (total_dst_rows + n_threads - 1) / n_threads,
.total_dst_rows = total_dst_rows,
.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
.total_dst_rows = nrows,
.row_start = row_start,
.type_size = type_size,
.src_row_size = src_row_size,
.src_row_size_aligned = src_row_size_aligned,
@@ -537,11 +549,10 @@ int op_pad(struct htp_ops_context * octx) {
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
lp0, rp0, lp1, rp1, lp2, rp2, lp3, rp3);
if (circular && use_dma) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); }
else if (circular) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_circular, &pctx, n_threads); }
else if (use_dma) { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx_dma, &pctx, n_threads); }
else { worker_pool_run_func(octx->ctx->worker_pool, pad_job_per_thread_hvx, &pctx, n_threads); }
if (circular && use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular_dma, &pctx, n_threads); }
else if (circular) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_circular, &pctx, n_threads); }
else if (use_dma) { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx_dma, &pctx, n_threads); }
else { work_queue_run(octx->ctx->work_queue, pad_job_per_thread_hvx, &pctx, n_threads); }
return HTP_STATUS_OK;
}
ggml/src/ggml-hexagon/htp/repeat-ops.c
+35 -14
View File
@@ -12,8 +12,10 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "htp-ctx.h"
#include "hex-common.h"
#include "hex-profile.h"
#include "htp-ops.h"
#include "htp-ops.h"
#include "htp-tensor.h"
struct htp_repeat_context {
struct htp_ops_context * octx;
@@ -25,6 +27,7 @@ struct htp_repeat_context {
uint32_t nrows_per_thread;
uint32_t total_dst_rows; // ne1 * ne2 * ne3
uint32_t row_start;
size_t type_size;
};
@@ -62,11 +65,11 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat
const size_t row_bytes = ne00 * rctx->type_size;
const uint32_t row_start = rctx->nrows_per_thread * ith;
const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->total_dst_rows);
const uint32_t row_start = rctx->row_start + rctx->nrows_per_thread * ith;
const uint32_t row_end = MIN(row_start + rctx->nrows_per_thread, rctx->row_start + rctx->total_dst_rows);
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
for (uint32_t dst_row = row_start; dst_row < row_end; dst_row++) {
// Decompose flat dst row index into (i1, i2, i3)
@@ -89,12 +92,12 @@ static void repeat_job_per_thread(unsigned int nth, unsigned int ith, void * dat
}
}
t2 = HAP_perf_get_qtimer_count();
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, row_start);
FARF(HIGH, "repeat %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u usec %u\n",
FARF(HIGH, "repeat %d/%d: (%ux%ux%ux%u) -> (%ux%ux%ux%u) rows %u:%u\n",
ith, nth, src->ne[0], src->ne[1], src->ne[2], src->ne[3],
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
row_start, row_end, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
row_start, row_end);
}
int op_repeat(struct htp_ops_context * octx) {
@@ -119,21 +122,39 @@ int op_repeat(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3];
const uint32_t n_threads = MIN(octx->n_threads, total_dst_rows);
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
return HTP_STATUS_OK;
}
const uint32_t total_dst_rows = dst->ne[1] * dst->ne[2] * dst->ne[3];
const size_t dst_row_size = dst->ne[0] * type_size;
uint32_t row_start = 0;
uint32_t nrows = total_dst_rows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_dst_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
struct htp_repeat_context rctx = {
.octx = octx,
.nr0 = dst->ne[0] / src0->ne[0],
.nr1 = dst->ne[1] / src0->ne[1],
.nr2 = dst->ne[2] / src0->ne[2],
.nr3 = dst->ne[3] / src0->ne[3],
.nrows_per_thread = (total_dst_rows + n_threads - 1) / n_threads,
.total_dst_rows = total_dst_rows,
.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
.total_dst_rows = nrows,
.row_start = row_start,
.type_size = type_size,
};
@@ -142,7 +163,7 @@ int op_repeat(struct htp_ops_context * octx) {
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
rctx.nr0, rctx.nr1, rctx.nr2, rctx.nr3);
worker_pool_run_func(octx->ctx->worker_pool, repeat_job_per_thread, &rctx, n_threads);
work_queue_run(octx->ctx->work_queue, repeat_job_per_thread, &rctx, n_threads);
return HTP_STATUS_OK;
}
ggml/src/ggml-hexagon/htp/rope-ops.c
+34 -7
View File
@@ -80,6 +80,8 @@ struct htp_rope_context {
size_t dst_row_stride;
size_t src0_row_size_aligned;
uint32_t src0_nrows;
uint32_t row_start;
uint32_t nrows;
struct fastdiv_values div_ne2_ne1;
struct fastdiv_values div_ne1;
@@ -539,11 +541,11 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) {
htp_rope_preamble;
const uint32_t src0_nrows = rctx->src0_nrows;
const uint32_t src0_nrows = rctx->nrows;
const uint32_t src0_nrows_per_thread = rctx->src0_nrows_per_thread;
const uint32_t src0_start_row = src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
const uint32_t src0_start_row = rctx->row_start + src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, rctx->row_start + src0_nrows);
// no work for this thread
if (src0_start_row >= src0_end_row) {
@@ -706,9 +708,32 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
}
const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params;
assert(kparams->n_threads > 0);
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
assert(octx->ctx->vtcm_size >= kparams->vtcm_size);
const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const size_t dst_data_row_size = dst->ne[0] * sizeof(float);
uint32_t row_start = 0;
uint32_t nrows = total_rows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
const uint32_t ne0 = dst->ne[0];
const size_t src0_row_size = src0->ne[0] * sizeof(float);
const size_t src0_row_stride = src0->nb[1];
@@ -752,15 +777,17 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
rctx.dst_row_stride = dst_row_stride;
rctx.src0_row_size_aligned = kparams->src0_row_size_aligned;
rctx.src0_nrows = kparams->src0_nrows;
rctx.src0_nrows_per_thread = kparams->src0_nrows_per_thread;
rctx.src0_nrows = nrows;
rctx.nrows = nrows;
rctx.row_start = row_start;
rctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
rctx.div_ne2_ne1 = kparams->div_ne2_ne1;
rctx.div_ne1 = kparams->div_ne1;
FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0,
rctx.ext_factor, rctx.theta_scale, rctx.attn_factor);
work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, kparams->n_threads);
work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, n_threads);
return err;
}
ggml/src/ggml-hexagon/htp/set-rows-ops.c
+42 -12
View File
@@ -18,6 +18,7 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
@@ -58,6 +59,9 @@ struct set_rows_context {
const struct htp_set_rows_kernel_params * kparams;
struct htp_set_rows_vtcm_layout vtcm_layout;
uint8_t * vtcm_base;
uint32_t task_start;
uint32_t tasks;
uint32_t tasks_per_thread;
};
#define SET_ROWS_THREAD_DMA_FN(TYPE_NAME, IDX_TYPE, COMPUTE_EXPR) \
@@ -67,12 +71,12 @@ static void set_rows_thread_dma_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsig
const struct htp_set_rows_kernel_params * kparams = srctx->kparams; \
set_rows_preamble; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
const uint32_t dr = kparams->tasks_per_thread; \
const uint32_t ir0 = dr * ith; \
if (ir0 >= kparams->total_tasks) { \
const uint32_t dr = srctx->tasks_per_thread; \
const uint32_t ir0 = srctx->task_start + dr * ith; \
if (ir0 >= srctx->task_start + srctx->tasks) { \
return; \
} \
const uint32_t ir1 = MIN(ir0 + dr, kparams->total_tasks); \
const uint32_t ir1 = MIN(ir0 + dr, srctx->task_start + srctx->tasks); \
dma_queue * dma_queue = octx->ctx->dma[ith]; \
const struct htp_set_rows_vtcm_layout * vtcm_layout = &srctx->vtcm_layout; \
uint8_t * vtcm_src0 = srctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; \
@@ -192,18 +196,44 @@ int op_set_rows(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
if (octx->src[1]->type != HTP_TYPE_I32 && octx->src[1]->type != HTP_TYPE_I64) {
return HTP_STATUS_NO_SUPPORT;
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
return HTP_STATUS_OK;
}
const struct htp_tensor * dst = octx->dst;
const uint32_t total_tasks = kparams->total_tasks;
uint32_t task_start = 0;
uint32_t tasks = total_tasks;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_mdev_data_aligned(dst) && (dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0 && !htp_tensor_is_permuted(dst);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_tasks, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
task_start = range.start;
tasks = range.count;
}
if (tasks == 0) {
return HTP_STATUS_OK;
}
if (!htp_ops_context_set_n_threads(octx, (uint32_t) kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
const uint32_t n_threads = octx->n_threads;
// l2fetch the src1 (indices) tensor in the main thread
hex_l2fetch_block((const void *)octx->src[1]->data, octx->src[1]->ne[3] * octx->src[1]->nb[3]);
struct set_rows_context srctx;
srctx.octx = octx;
srctx.kparams = kparams;
srctx.task_start = task_start;
srctx.tasks = tasks;
srctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, kparams->n_threads);
htp_set_rows_vtcm_layout_build(&srctx.vtcm_layout, octx->dst->type, ne00, n_threads);
srctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;
work_queue_func_t q_func = NULL;
@@ -216,15 +246,15 @@ int op_set_rows(struct htp_ops_context * octx) {
default: return HTP_STATUS_NO_SUPPORT;
}
FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n_threads %d\n",
FARF(HIGH, "set-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu n-threads %d\n",
octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3],
octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3],
octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3],
srctx.vtcm_layout.src0_bytes_per_thread * kparams->n_threads,
srctx.vtcm_layout.dst_bytes_per_thread * kparams->n_threads,
kparams->n_threads);
srctx.vtcm_layout.src0_bytes_per_thread * n_threads,
srctx.vtcm_layout.dst_bytes_per_thread * n_threads,
n_threads);
work_queue_run(octx->ctx->work_queue, q_func, &srctx, kparams->n_threads);
work_queue_run(octx->ctx->work_queue, q_func, &srctx, n_threads);
return HTP_STATUS_OK;
}
ggml/src/ggml-hexagon/htp/softmax-ops.c
+38 -15
View File
@@ -14,9 +14,11 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
#include "hex-profile.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#define htp_softmax_preamble3 \
const uint32_t ne00 = src0->ne[0]; \
@@ -69,6 +71,8 @@ struct htp_softmax_context {
struct fastdiv_values fastdiv_ne13; // For mask broadcasting
uint32_t src0_nrows_per_thread;
uint32_t row_start;
uint32_t nrows;
};
static void apply_mask(float * restrict wp0,
@@ -223,19 +227,17 @@ static void softmax_job_f32(unsigned int nth, unsigned int ith, void * data) {
htp_softmax_preamble3;
const uint32_t src0_nrows = ne01 * ne02 * ne03; // src0 rows
const uint32_t src0_nrows = smctx->nrows;
const uint32_t src0_nrows_per_thread = smctx->src0_nrows_per_thread;
const uint32_t src0_start_row = src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
const uint32_t src0_start_row = smctx->row_start + src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, smctx->row_start + src0_nrows);
// no work for this thread
if (src0_start_row >= src0_end_row) {
return;
}
uint64_t qt = HAP_perf_get_qtimer_count();
int is_aligned = 1;
int opt_path = 0;
@@ -262,6 +264,9 @@ static void softmax_job_f32(unsigned int nth, unsigned int ith, void * data) {
uint32_t prev_i2 = (uint32_t)-1;
float slope = 1.0f;
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, src0_start_row);
for (uint32_t r = src0_start_row; r < src0_end_row; ++r) {
uint32_t i1 = fastmodulo(r, ne01, &smctx->fastdiv_ne01);
uint32_t r_div_ne01 = fastdiv(r, &smctx->fastdiv_ne01);
@@ -323,10 +328,11 @@ static void softmax_job_f32(unsigned int nth, unsigned int ith, void * data) {
}
}
qt = HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - qt);
FARF(HIGH, "softmax-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u : opt %u f16 %u usec %u\n", ith, nth,
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, src0_start_row);
FARF(HIGH, "softmax-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u : opt %u f16 %u\n", ith, nth,
ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13,
ne0, ne1, ne2, ne3, opt_path, smctx->use_f16, (unsigned) qt);
ne0, ne1, ne2, ne3, opt_path, smctx->use_f16);
}
static int execute_op_softmax_f32(struct htp_ops_context * octx) {
@@ -342,13 +348,32 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) {
init_softmax_ctx(&smctx, octx);
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);
const size_t elem_size = sizeof(float);
const size_t dst_row_size = dst->nb[1];
smctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
uint32_t row_start = 0;
uint32_t nrows = src0_nrows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
smctx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
smctx.row_start = row_start;
smctx.nrows = nrows;
const size_t src0_row_size = src0->nb[1];
const size_t src1_row_size = src0_row_size;
const size_t dst_row_size = dst->nb[1];
// VTCM scratchpads for all tensors
// 4 rows per thread, padded to HVX vector size
@@ -383,9 +408,7 @@ static int execute_op_softmax_f32(struct htp_ops_context * octx) {
octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->src1_spad.src = NULL;
octx->dst_spad.data = octx->src1_spad.data + octx->src1_spad.size; octx->dst_spad.src = NULL;
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) return err;
worker_pool_run_func(octx->ctx->worker_pool, softmax_job_f32, &smctx, n_threads);
work_queue_run(octx->ctx->work_queue, softmax_job_f32, &smctx, n_threads);
return err;
}
ggml/src/ggml-hexagon/htp/solve-tri-ops.c
+62 -30
View File
@@ -1,13 +1,16 @@
#pragma clang diagnostic ignored "-Wunused-but-set-variable"
#include <HAP_farf.h>
#include <HAP_perf.h>
#include <string.h>
#include "hex-common.h"
#include "hex-profile.h"
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hvx-types.h"
#include "hvx-utils.h"
@@ -15,6 +18,7 @@ struct htp_solve_tri_context {
struct htp_ops_context * octx;
uint32_t jobs_per_thread;
uint32_t total_jobs;
uint32_t job_start;
uint32_t k_chunks;
uint32_t col_block;
};
@@ -89,11 +93,11 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void
const uint32_t col_block = VLEN_FP32;
const uint32_t k_full = (k / col_block) * col_block;
const uint32_t start_batch = sctx->jobs_per_thread * ith;
const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->total_jobs);
const uint32_t start_batch = sctx->job_start + sctx->jobs_per_thread * ith;
const uint32_t end_batch = MIN(start_batch + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs);
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_batch);
for (uint32_t batch = start_batch; batch < end_batch; ++batch) {
const uint32_t i03 = batch / ne02;
@@ -127,11 +131,10 @@ static void solve_tri_batch_thread_f32(unsigned int nth, unsigned int ith, void
}
}
t2 = HAP_perf_get_qtimer_count();
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) end_batch);
FARF(HIGH, "solve-tri-batch %d/%d: A=(%ux%u) B=(%ux%u) batch %u:%u usec %u\n",
ith, nth, n, n, k, n, start_batch, end_batch,
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
FARF(HIGH, "solve-tri-batch %d/%d: A=(%ux%u) B=(%ux%u) batch %u:%u\n",
ith, nth, n, n, k, n, start_batch, end_batch);
}
// Chunk-level thread: each job is one (batch, col_chunk) pair.
@@ -148,11 +151,11 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void
const uint32_t ne02 = src0->ne[2];
const uint32_t start_job = sctx->jobs_per_thread * ith;
const uint32_t end_job = MIN(start_job + sctx->jobs_per_thread, sctx->total_jobs);
const uint32_t start_job = sctx->job_start + sctx->jobs_per_thread * ith;
const uint32_t end_job = MIN(start_job + sctx->jobs_per_thread, sctx->job_start + sctx->total_jobs);
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_job);
for (uint32_t job = start_job; job < end_job; ++job) {
const uint32_t batch = job / sctx->k_chunks;
@@ -161,16 +164,14 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void
const uint32_t i03 = batch / ne02;
const uint32_t i02 = batch - i03 * ne02;
const uint32_t col0 = chunk * sctx->col_block;
const uint32_t coln = MIN(sctx->col_block, k - col0);
const float * A_batch =
(const float *) ((const uint8_t *) (uintptr_t) src0->data + i02 * src0->nb[2] + i03 * src0->nb[3]);
const float * B_batch =
(const float *) ((const uint8_t *) (uintptr_t) src1->data + i02 * src1->nb[2] + i03 * src1->nb[3]);
float * X_batch = (float *) ((uint8_t *) (uintptr_t) dst->data + i02 * dst->nb[2] + i03 * dst->nb[3]);
const bool use_hvx = (coln >= 8);
const uint32_t col0 = chunk * sctx->col_block;
const uint32_t coln = MIN(sctx->col_block, k - col0);
for (uint32_t row = 0; row < n; ++row) {
const float diag = A_batch[row * n + row];
@@ -179,7 +180,7 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void
const float * A_row = A_batch + row * n;
const float * B_row = B_batch + row * k;
if (use_hvx) {
if (coln >= 8) {
solve_tri_row_hvx(A_row, B_row, X_batch, row, k, col0, coln, inv_diag);
} else {
solve_tri_row_scalar(A_row, B_row, X_batch, row, k, col0, coln, inv_diag);
@@ -187,11 +188,10 @@ static void solve_tri_chunk_thread_f32(unsigned int nth, unsigned int ith, void
}
}
t2 = HAP_perf_get_qtimer_count();
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) end_job);
FARF(HIGH, "solve-tri-chunk %d/%d: A=(%ux%u) B=(%ux%u) job %u:%u usec %u\n",
ith, nth, n, n, k, n, start_job, end_job,
(unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
FARF(HIGH, "solve-tri-chunk %d/%d: A=(%ux%u) B=(%ux%u) jobs %u:%u\n",
ith, nth, n, n, k, n, start_job, end_job);
}
int op_solve_tri(struct htp_ops_context * octx) {
@@ -235,32 +235,64 @@ int op_solve_tri(struct htp_ops_context * octx) {
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], batched);
if (batched) {
uint32_t job_start = 0;
uint32_t njobs = total_batches;
if (octx->ctx->mdev.count > 1) {
const uint32_t batch_size = dst->nb[2];
const uint32_t batches_per_chunk = (batch_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(batch_size, HEX_L2_LINE_SIZE)) : 1;
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_batches, htp_tensor_mdev_data_aligned(dst) ? batches_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
job_start = range.start;
njobs = range.count;
}
if (njobs == 0) {
return HTP_STATUS_OK;
}
// Batch-level parallelism
const uint32_t n_threads = MIN((uint32_t) octx->n_threads, total_batches);
const uint32_t n_threads = octx->n_threads;
struct htp_solve_tri_context sctx = {
.octx = octx,
.jobs_per_thread = (total_batches + n_threads - 1) / n_threads,
.total_jobs = total_batches,
.jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div),
.total_jobs = njobs,
.job_start = job_start,
.k_chunks = k_chunks,
.col_block = col_block,
};
worker_pool_run_func(octx->ctx->worker_pool, solve_tri_batch_thread_f32, &sctx, n_threads);
work_queue_run(octx->ctx->work_queue, solve_tri_batch_thread_f32, &sctx, n_threads);
} else {
// Chunk-level parallelism
const uint32_t total_jobs = total_batches * k_chunks;
const uint32_t n_threads = MIN((uint32_t) octx->n_threads, MAX(total_jobs, 1));
uint32_t job_start = 0;
uint32_t njobs = total_jobs;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_jobs, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
job_start = range.start;
njobs = range.count;
}
if (njobs == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
struct htp_solve_tri_context sctx = {
.octx = octx,
.jobs_per_thread = (total_jobs + n_threads - 1) / n_threads,
.total_jobs = total_jobs,
.jobs_per_thread = fastdiv(njobs + n_threads - 1, &octx->n_threads_div),
.total_jobs = njobs,
.job_start = job_start,
.k_chunks = k_chunks,
.col_block = col_block,
};
worker_pool_run_func(octx->ctx->worker_pool, solve_tri_chunk_thread_f32, &sctx, n_threads);
work_queue_run(octx->ctx->work_queue, solve_tri_chunk_thread_f32, &sctx, n_threads);
}
return HTP_STATUS_OK;
ggml/src/ggml-hexagon/htp/ssm-conv.c
+96 -73
View File
@@ -4,7 +4,6 @@
#include <HAP_farf.h>
#include <HAP_mem.h>
#include <HAP_perf.h>
#include <HAP_ps.h>
#include <hexagon_protos.h>
#include <hexagon_types.h>
@@ -16,8 +15,9 @@
#include "ggml-common.h"
#include "htp-ctx.h"
#include "hex-dma.h"
#include "hex-profile.h"
#include "htp-ops.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hvx-utils.h"
#define htp_ssm_conv_tensors_preamble \
@@ -63,6 +63,8 @@ struct htp_ssm_conv_context {
uint32_t nrows_per_thread;
uint32_t d_inner_tile;
uint64_t t_start;
uint32_t row_start;
uint32_t nrows;
};
#define htp_ssm_conv_preamble \
@@ -75,9 +77,6 @@ struct htp_ssm_conv_context {
static void ssm_conv_thread_f32_f32(unsigned int nth, unsigned int ith, void *data) {
htp_ssm_conv_preamble;
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
const uint32_t d_conv = src1->ne[0];
const uint32_t d_inner = src0->ne[1];
const uint32_t n_t = dst->ne[1];
@@ -95,14 +94,17 @@ static void ssm_conv_thread_f32_f32(unsigned int nth, unsigned int ith, void *da
// Calculate row range for this thread
const uint32_t d_inner_per_thread = scctx->nrows_per_thread;
const uint32_t d_inner_start = d_inner_per_thread * ith;
const uint32_t d_inner_end = MIN(d_inner_start + d_inner_per_thread, d_inner);
const uint32_t d_inner_start = scctx->row_start + d_inner_per_thread * ith;
const uint32_t d_inner_end = MIN(d_inner_start + d_inner_per_thread, scctx->row_start + scctx->nrows);
// No work for this thread
if (d_inner_start >= d_inner_end) {
return;
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) d_inner_start);
for (uint32_t i3 = 0; i3 < n_s; ++i3) {
for (uint32_t i2 = 0; i2 < n_t; ++i2) {
for (uint32_t i1 = d_inner_start; i1 < d_inner_end; ++i1) {
@@ -121,12 +123,12 @@ static void ssm_conv_thread_f32_f32(unsigned int nth, unsigned int ith, void *da
}
}
t2 = HAP_perf_get_qtimer_count();
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) d_inner_end);
FARF(HIGH, "ssm-conv-f32 %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n",
FARF(HIGH, "ssm-conv-f32 %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u\n",
ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], d_inner_start, d_inner_end,
src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1],
dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
dst->ne[2], dst->ne[3]);
}
@@ -257,9 +259,6 @@ static inline void transpose_src0_block(const float * src0_block,
static void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void *data) {
htp_ssm_conv_preamble;
uint64_t t1, t2;
t1 = HAP_perf_get_qtimer_count();
const uint32_t d_conv = src1->ne[0];
const uint32_t d_inner = src0->ne[1];
const uint32_t n_t = dst->ne[1];
@@ -273,13 +272,16 @@ static void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void
const uint32_t dst_stride_seq = dst->nb[2] / sizeof(float);
const uint32_t dr = scctx->nrows_per_thread;
const uint32_t ir0 = dr * ith;
const uint32_t ir1 = MIN(ir0 + dr, d_inner);
const uint32_t ir0 = scctx->row_start + dr * ith;
const uint32_t ir1 = MIN(ir0 + dr, scctx->row_start + scctx->nrows);
if (ir0 >= ir1) {
return;
}
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir0);
const uint32_t d_inner_per_thread = ir1 - ir0;
const uint32_t d_inner_stride = scctx->nrows_per_thread;
const uint32_t d_inner_tile = scctx->d_inner_tile;
@@ -319,97 +321,118 @@ static void ssm_conv_thread_f32_f32_hvx(unsigned int nth, unsigned int ith, void
HVX_Vector w = *(const HVX_Vector *) (src1_T + j * d_inner_stride + tile_off + cb);
acc = Q6_Vqf32_vadd_Vqf32Vqf32(acc, Q6_Vqf32_vmpy_VsfVsf(x, w));
}
HVX_Vector res = Q6_Vsf_equals_Vqf32(acc);
float * dst_ptr = dst_data + i3 * dst_stride_seq + t * dst_stride_token + (ir0 + tile_off + cb);
HVX_Vector y = Q6_Vsf_equals_Vqf32(acc);
float * dst_ptr = dst_data + (ir0 + tile_off + cb) + t * dst_stride_token + i3 * dst_stride_seq;
if (cb_n == C_TILE) {
*(HVX_UVector *) dst_ptr = res;
*(HVX_UVector *) dst_ptr = y;
} else {
hvx_vec_store_u(dst_ptr, cb_n * sizeof(float), res);
hvx_vec_store_u(dst_ptr, cb_n * sizeof(float), y);
}
}
}
}
}
t2 = HAP_perf_get_qtimer_count();
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ir1);
FARF(HIGH, "ssm-conv-f32-hvx %d/%d: %ux%ux%ux%u (%u:%u) tile=%u * %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n",
ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1, d_inner_tile,
FARF(HIGH, "ssm-conv-f32-hvx %d/%d: %ux%ux%ux%u (%u:%u) * %ux%ux%ux%u -> %ux%ux%ux%u\n",
ith, nth, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], ir0, ir1,
src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0], dst->ne[1],
dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
dst->ne[2], dst->ne[3]);
}
int op_ssm_conv_f32(struct htp_ops_context * octx) {
htp_ssm_conv_tensors_preamble;
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
if (src0->type != HTP_TYPE_F32 || src1->type != HTP_TYPE_F32 || dst->type != HTP_TYPE_F32) {
FARF(ERROR, "ssm_conv: only (F32 x F32 -> F32) OPs supported");
return HTP_STATUS_NO_SUPPORT;
}
struct htp_ssm_conv_context scctx = { 0 };
scctx.octx = octx;
const uint32_t d_conv = src1->ne[0];
const uint32_t d_inner = src0->ne[1];
const uint32_t n_t = dst->ne[1]; // tokens per sequence
const uint32_t n_s = dst->ne[2]; // number of sequences in the batch
const uint32_t n_threads = MIN(octx->n_threads, d_inner);
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
return HTP_STATUS_OK;
}
if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {
uint32_t use_hvx = 0;
if (d_inner >= VLEN_FP32 && n_t >= VLEN_FP32) {
use_hvx = 1;
}
uint32_t row_start = 0;
uint32_t nrows = d_inner;
scctx.nrows_per_thread = hex_round_up((d_inner + n_threads - 1) / n_threads, VLEN_FP32);
if (octx->ctx->mdev.count > 1) {
const uint32_t elems_per_chunk = VLEN_FP32;
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(d_inner, htp_tensor_mdev_data_aligned(dst) ? elems_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
const uint32_t d_inner_per_thread = scctx.nrows_per_thread;
const uint32_t ncs = src0->ne[0];
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t src1_T_size = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 256);
const uint32_t src0_T_max = HTP_SSM_CONV_VTCM_BUDGET > src1_T_size ? HTP_SSM_CONV_VTCM_BUDGET - src1_T_size : 0;
const uint32_t n_threads = octx->n_threads;
uint32_t d_inner_tile = (src0_T_max / sizeof(float)) / ncs;
d_inner_tile -= (d_inner_tile % VLEN_FP32);
if (d_inner_tile == 0) {
FARF(HIGH, "ssm_conv-f32: inner tile rounds to 0 (ncs=%u), falling back to scalar\n", ncs);
struct htp_ssm_conv_context scctx = { 0 };
scctx.octx = octx;
scctx.row_start = row_start;
scctx.nrows = nrows;
uint32_t use_hvx = 0;
if (nrows >= VLEN_FP32 && n_t >= VLEN_FP32) {
use_hvx = 1;
}
const uint32_t raw_rpt = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
scctx.nrows_per_thread = hex_round_up(raw_rpt, VLEN_FP32);
const uint32_t d_inner_per_thread = scctx.nrows_per_thread;
const uint32_t ncs = src0->ne[0];
const uint32_t src1_T_size = hex_round_up(d_conv * d_inner_per_thread * sizeof(float), 256);
const uint32_t src0_T_max = HTP_SSM_CONV_VTCM_BUDGET > src1_T_size ? HTP_SSM_CONV_VTCM_BUDGET - src1_T_size : 0;
uint32_t d_inner_tile = (src0_T_max / sizeof(float)) / ncs;
d_inner_tile -= (d_inner_tile % VLEN_FP32);
if (d_inner_tile == 0) {
FARF(HIGH, "ssm_conv-f32: inner tile rounds to 0 (ncs=%u), falling back to scalar\n", ncs);
use_hvx = 0;
} else {
scctx.d_inner_tile = d_inner_tile;
octx->src0_spad.size_per_thread = hex_round_up(d_inner_tile * ncs * sizeof(float), 256);
octx->src1_spad.size_per_thread = src1_T_size;
octx->dst_spad.size_per_thread = 0;
octx->src0_spad.size = octx->src0_spad.size_per_thread * n_threads;
octx->src1_spad.size = octx->src1_spad.size_per_thread * n_threads;
octx->dst_spad.size = 0;
octx->src0_spad.data = octx->ctx->vtcm_base;
octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;
octx->src0_spad.src = NULL;
octx->src1_spad.src = NULL;
const size_t total_spad = octx->src0_spad.size + octx->src1_spad.size;
if (total_spad > octx->ctx->vtcm_size) {
FARF(HIGH, "ssm_conv-f32: scratchpad %zu exceeds VTCM %zu, falling back to scalar\n",
total_spad, octx->ctx->vtcm_size);
use_hvx = 0;
} else {
scctx.d_inner_tile = d_inner_tile;
octx->src0_spad.size_per_thread = hex_round_up(d_inner_tile * ncs * sizeof(float), 256);
octx->src1_spad.size_per_thread = src1_T_size;
octx->dst_spad.size_per_thread = 0;
octx->src0_spad.size = octx->src0_spad.size_per_thread * n_threads;
octx->src1_spad.size = octx->src1_spad.size_per_thread * n_threads;
octx->dst_spad.size = 0;
octx->src0_spad.data = octx->ctx->vtcm_base;
octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;
octx->src0_spad.src = NULL;
octx->src1_spad.src = NULL;
const size_t total_spad = octx->src0_spad.size + octx->src1_spad.size;
if (total_spad > octx->ctx->vtcm_size) {
FARF(HIGH, "ssm_conv-f32: scratchpad %zu exceeds VTCM %zu, falling back to scalar\n",
total_spad, octx->ctx->vtcm_size);
use_hvx = 0;
}
}
}
FARF(HIGH, "ssm-conv-f32: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : use_hvx %d\n", src0->ne[0],
src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0],
dst->ne[1], dst->ne[2], dst->ne[3], use_hvx);
FARF(HIGH, "ssm-conv-f32: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : use_hvx %d\n", src0->ne[0],
src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3], dst->ne[0],
dst->ne[1], dst->ne[2], dst->ne[3], use_hvx);
if (use_hvx) {
worker_pool_run_func(octx->ctx->worker_pool, ssm_conv_thread_f32_f32_hvx, &scctx, n_threads);
} else {
worker_pool_run_func(octx->ctx->worker_pool, ssm_conv_thread_f32_f32, &scctx, n_threads);
}
if (use_hvx) {
work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_f32_hvx, &scctx, n_threads);
} else {
work_queue_run(octx->ctx->work_queue, ssm_conv_thread_f32_f32, &scctx, n_threads);
}
return HTP_STATUS_OK;
ggml/src/ggml-hexagon/htp/sum-rows-ops.c
+53 -27
View File
@@ -13,35 +13,38 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-common.h"
#include "hex-profile.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#define sum_rows_preamble \
const struct htp_tensor *src0 = octx->src[0]; \
const struct htp_tensor *dst = octx->dst; \
\
const uint32_t ne00 = src0->ne[0]; \
const uint32_t ne01 = src0->ne[1]; \
const uint32_t ne02 = src0->ne[2]; \
const uint32_t ne03 = src0->ne[3]; \
\
const uint32_t nb00 = src0->nb[0]; \
const uint32_t nb01 = src0->nb[1]; \
const uint32_t nb02 = src0->nb[2]; \
const uint32_t nb03 = src0->nb[3]; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
const uint32_t ne2 = dst->ne[2]; \
const uint32_t ne3 = dst->ne[3]; \
\
const uint32_t nb0 = dst->nb[0]; \
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3]; \
const uint32_t ne00 = src0->ne[0]; \
const uint32_t ne01 = src0->ne[1]; \
const uint32_t ne02 = src0->ne[2]; \
const uint32_t ne03 = src0->ne[3]; \
\
const uint32_t nb00 = src0->nb[0]; \
const uint32_t nb01 = src0->nb[1]; \
const uint32_t nb02 = src0->nb[2]; \
const uint32_t nb03 = src0->nb[3]; \
\
const uint32_t ne0 = dst->ne[0]; \
const uint32_t ne1 = dst->ne[1]; \
const uint32_t ne2 = dst->ne[2]; \
const uint32_t ne3 = dst->ne[3]; \
\
const uint32_t nb0 = dst->nb[0]; \
const uint32_t nb1 = dst->nb[1]; \
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3]; \
struct sum_rows_context {
struct htp_ops_context * octx;
const uint8_t * src_data;
uint8_t * dst_data;
uint32_t ne00;
@@ -76,6 +79,9 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data)
// Calculate actual number of rows for this thread
const uint32_t n_rows = end_row - start_row;
struct htp_thread_trace * tr = &smctx->octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row);
for (uint32_t ir = 0; ir < n_rows; ir++) {
const float * restrict src_local = src_th + (ir * (src_stride / sizeof(float)));
@@ -89,6 +95,8 @@ static void sum_rows_thread_f32(unsigned int nth, unsigned int ith, void *data)
dst_th[ir] = hvx_reduce_sum_f32((const uint8_t *) src_local, ne00);
}
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) start_row);
}
int op_sum_rows(struct htp_ops_context * octx) {
@@ -102,9 +110,26 @@ int op_sum_rows(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
const uint32_t src0_nrows = ne01 * ne02 * ne03;
const uint32_t n_threads = MIN(octx->n_threads, src0_nrows);
const uint32_t rows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
const uint32_t src0_nrows = ne01 * ne02 * ne03;
const size_t dst_data_row_size = dst->ne[0] * sizeof(float);
uint32_t row_start = 0;
uint32_t nrows = src0_nrows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, sizeof(float), (uint32_t) dst_data_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
const uint32_t rows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
bool opt_path = false;
if ((0 == hex_is_aligned((void *) src0->data, VLEN)) && !(nb01 & (VLEN - 1))) {
@@ -112,17 +137,18 @@ int op_sum_rows(struct htp_ops_context * octx) {
}
struct sum_rows_context smctx = {
.src_data = (const uint8_t *) src0->data,
.dst_data = (uint8_t *) dst->data,
.octx = octx,
.src_data = (const uint8_t *) src0->data + row_start * nb01,
.dst_data = (uint8_t *) dst->data + row_start * nb1,
.ne00 = ne00,
.src_stride = nb01,
.dst_stride = nb1,
.rows_per_thread = rows_per_thread,
.total_rows = src0_nrows,
.total_rows = nrows,
.opt_path = opt_path,
};
worker_pool_run_func(octx->ctx->worker_pool, sum_rows_thread_f32, &smctx, n_threads);
work_queue_run(octx->ctx->work_queue, sum_rows_thread_f32, &smctx, n_threads);
return HTP_STATUS_OK;
}
ggml/src/ggml-hexagon/htp/unary-ops.c
+150 -128
View File
@@ -46,6 +46,7 @@ struct htp_unary_context {
uint32_t block;
uint32_t src0_nrows;
uint32_t src0_nrows_per_thread;
uint32_t row_start;
uint32_t nc;
uint32_t col_tile; // tiled mode
bool broadcast_weight;
@@ -496,7 +497,7 @@ static void tri_f32(const float * restrict src,
}
if (boundary > ne0) boundary = ne0;
// Full HVX vectors — each starts at a 128-byte aligned offset
// Full HVX vectors - each starts at a 128-byte aligned offset
for (uint32_t i = 0; i < nvec; i++) {
const uint32_t vec_start = i * VLEN_FP32;
const uint32_t vec_end = vec_start + VLEN_FP32;
@@ -563,7 +564,7 @@ static void softplus_f32(const float * restrict src,
for (uint32_t i = 0; i < ne0; i++) {
float x = src_f[i];
// For x > 20: softplus(x) ≈ x (avoids exp overflow)
// For x > 20: softplus(x) ~ x (avoids exp overflow)
dst_f[i] = (x > 20.0f) ? x : logf(1.0f + expf(x));
}
}
@@ -661,8 +662,8 @@ static void unary_task_##SUFFIX##_##NAME(unsigned int nth, unsigned int ith, voi
const size_t dst_row_size_aligned = uctx->dst_row_size_aligned; \
\
const uint32_t src0_nrows = uctx->src0_nrows; \
const uint32_t src0_start_row = src0_nrows_per_thread * ith; \
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \
const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; \
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); \
\
if (src0_start_row >= src0_end_row) { \
return; \
@@ -833,124 +834,126 @@ DEFINE_UNARY_TASK_IMPL(unary_abs, _Float16, f16, false, false, abs_f16(src0_vtcm
DEFINE_UNARY_TASK_IMPL(unary_log, _Float16, f16, false, false, log_f16(src0_vtcm, dst_vtcm, block_size, uctx))
// Apply a pointwise unary op to one column tile that is already in VTCM.
#define DEFINE_UNARY_TILED_TASK(NAME, IS_TRI, CORE_TILE_EXPR) \
static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void * data) { \
const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \
struct htp_ops_context * octx = uctx->octx; \
const struct htp_tensor * src = octx->src[0]; \
const struct htp_tensor * dst = octx->dst; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
\
htp_unary_preamble; \
\
int32_t * op_params = octx->op_params; \
const uint32_t col_tile = uctx->col_tile; \
\
const uint32_t src0_nrows = uctx->src0_nrows; \
const uint32_t src0_start_row = uctx->src0_nrows_per_thread * ith; \
const uint32_t src0_end_row = MIN(src0_start_row + uctx->src0_nrows_per_thread, src0_nrows); \
\
if (src0_start_row >= src0_end_row) { \
return; \
} \
\
const uint8_t * restrict data_src = uctx->data_src0; \
uint8_t * restrict data_dst = uctx->data_dst; \
\
uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); \
uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); \
\
const size_t src0_half = uctx->src0_vtcm_half_size; \
const size_t dst_half = uctx->dst_vtcm_half_size; \
\
dma_queue * dmaq = octx->ctx->dma[ith]; \
\
const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; \
const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; \
const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; \
const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; \
\
const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile; \
const int32_t tri_ttype = (IS_TRI) ? op_params[0] : 0; \
\
const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && \
(nb03 == (size_t)ne02 * nb02); \
const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && \
(nb3 == (size_t)ne2 * nb2); \
\
const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; \
\
for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { \
const uint32_t row = src0_start_row + t / tiles_per_row; \
const uint32_t col = (t % tiles_per_row) * col_tile; \
const uint32_t tw = MIN(col_tile, ne0 - col); \
const size_t tb = (size_t) tw * sizeof(float); \
const size_t soff = (src0_contig ? (row * nb01) : \
unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) +\
(size_t) col * sizeof(float); \
\
dma_queue_push(dmaq, dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0); \
dma_queue_push(dmaq, dma_make_ptr(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1);\
} \
\
uint32_t row = src0_start_row; \
uint32_t col = 0; \
uint32_t tile_in_row = 0; \
uint32_t i01 = fastmodulo(row, ne01, div_ne01); \
\
uint32_t prow = src0_start_row + fastdiv(2, div_tpr); \
uint32_t pcol = fastmodulo(2, tiles_per_row, div_tpr) * col_tile; \
uint32_t ptile_in_row = fastmodulo(2, tiles_per_row, div_tpr); \
\
for (uint32_t t = 0; t < total_tiles; t++) { \
uint8_t * dst_vtcm = (uint8_t *) dma_queue_pop(dmaq).src; \
uint8_t * src_vtcm = (uint8_t *) dma_queue_pop(dmaq).dst; \
\
const uint32_t tw = MIN(col_tile, ne0 - col); \
\
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, t); \
CORE_TILE_EXPR; \
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); \
\
const size_t doff = (dst_contig ? (row * nb1) : \
unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) + \
(size_t) col * sizeof(float); \
const size_t tb = (size_t) tw * sizeof(float); \
dma_queue_push(dmaq, dma_make_ptr(data_dst + doff, dst_vtcm), tb, tb, tb, 1); \
\
const uint32_t pt = t + 2; \
if (pt < total_tiles) { \
const uint32_t ptw = MIN(col_tile, ne0 - pcol); \
const size_t ptb = (size_t) ptw * sizeof(float); \
const size_t psoff = (src0_contig ? (prow * nb01) : \
unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, \
nb03)) + \
(size_t) pcol * sizeof(float); \
dma_queue_push(dmaq, dma_make_ptr(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1); \
} \
\
tile_in_row++; \
col += col_tile; \
if (tile_in_row == tiles_per_row) { \
tile_in_row = 0; \
col = 0; \
row++; \
i01++; \
if (i01 == ne01) { \
i01 = 0; \
} \
} \
\
ptile_in_row++; \
pcol += col_tile; \
if (ptile_in_row == tiles_per_row) { \
ptile_in_row = 0; \
pcol = 0; \
prow++; \
} \
} \
\
dma_queue_flush(dmaq); \
#define DEFINE_UNARY_TILED_TASK(NAME, IS_TRI, CORE_TILE_EXPR) \
static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void * data) { \
const struct htp_unary_context * uctx = (const struct htp_unary_context *) data; \
struct htp_ops_context * octx = uctx->octx; \
const struct htp_tensor * src = octx->src[0]; \
const struct htp_tensor * dst = octx->dst; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
\
htp_unary_preamble; \
\
uint32_t src0_nrows_per_thread = uctx->src0_nrows_per_thread; \
\
int32_t * op_params = octx->op_params; \
const uint32_t col_tile = uctx->col_tile; \
\
const uint32_t src0_nrows = uctx->src0_nrows; \
const uint32_t src0_start_row = uctx->row_start + src0_nrows_per_thread * ith; \
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, uctx->row_start + src0_nrows); \
\
if (src0_start_row >= src0_end_row) { \
return; \
} \
\
const uint8_t * restrict data_src = uctx->data_src0; \
uint8_t * restrict data_dst = uctx->data_dst; \
\
uint8_t * src0_vtcm_data = uctx->vtcm_src0 + (ith * uctx->vtcm_src0_size_per_thread); \
uint8_t * dst_vtcm_data = uctx->vtcm_dst + (ith * uctx->vtcm_dst_size_per_thread); \
\
const size_t src0_half = uctx->src0_vtcm_half_size; \
const size_t dst_half = uctx->dst_vtcm_half_size; \
\
dma_queue * dmaq = octx->ctx->dma[ith]; \
\
const struct fastdiv_values * div_ne01 = &uctx->kparams->div_ne01; \
const struct fastdiv_values * div_ne02 = &uctx->kparams->div_ne02; \
const struct fastdiv_values * div_ne012 = &uctx->kparams->div_ne012; \
const struct fastdiv_values * div_tpr = &uctx->kparams->div_tpr; \
\
const uint32_t tiles_per_row = (ne0 + col_tile - 1) / col_tile; \
const int32_t tri_ttype = (IS_TRI) ? op_params[0] : 0; \
\
const bool src0_contig = (nb02 == (size_t)ne01 * nb01) && \
(nb03 == (size_t)ne02 * nb02); \
const bool dst_contig = (nb2 == (size_t)ne1 * nb1) && \
(nb3 == (size_t)ne2 * nb2); \
\
const uint32_t total_tiles = (src0_end_row - src0_start_row) * tiles_per_row; \
\
for (uint32_t t = 0, vtcm_idx = 0; t < total_tiles && vtcm_idx < 2; t++, vtcm_idx++) { \
const uint32_t row = src0_start_row + t / tiles_per_row; \
const uint32_t col = (t % tiles_per_row) * col_tile; \
const uint32_t tw = MIN(col_tile, ne0 - col); \
const size_t tb = (size_t) tw * sizeof(float); \
const size_t soff = (src0_contig ? (row * nb01) : \
unary_row_offset(row, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, nb03)) + \
(size_t) col * sizeof(float); \
\
dma_queue_push(dmaq, dma_make_ptr(data_dst, dst_vtcm_data + (vtcm_idx * dst_half)), 0, 0, 0, 0); \
dma_queue_push(dmaq, dma_make_ptr(src0_vtcm_data + (vtcm_idx * src0_half), data_src + soff), tb, tb, tb, 1); \
} \
\
uint32_t row = src0_start_row; \
uint32_t col = 0; \
uint32_t tile_in_row = 0; \
uint32_t i01 = fastmodulo(row, ne01, div_ne01); \
\
uint32_t prow = src0_start_row + fastdiv(2, div_tpr); \
uint32_t pcol = fastmodulo(2, tiles_per_row, div_tpr) * col_tile; \
uint32_t ptile_in_row = fastmodulo(2, tiles_per_row, div_tpr); \
\
for (uint32_t t = 0; t < total_tiles; t++) { \
uint8_t * dst_vtcm = (uint8_t *) dma_queue_pop(dmaq).src; \
uint8_t * src_vtcm = (uint8_t *) dma_queue_pop(dmaq).dst; \
\
const uint32_t tw = MIN(col_tile, ne0 - col); \
\
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, t); \
CORE_TILE_EXPR; \
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, t); \
\
const size_t doff = (dst_contig ? (row * nb1) : \
unary_row_offset(row, ne1, ne2, div_ne01, div_ne02, div_ne012, nb1, nb2, nb3)) + \
(size_t) col * sizeof(float); \
const size_t tb = (size_t) tw * sizeof(float); \
dma_queue_push(dmaq, dma_make_ptr(data_dst + doff, dst_vtcm), tb, tb, tb, 1); \
\
const uint32_t pt = t + 2; \
if (pt < total_tiles) { \
const uint32_t ptw = MIN(col_tile, ne0 - pcol); \
const size_t ptb = (size_t) ptw * sizeof(float); \
const size_t psoff = (src0_contig ? (prow * nb01) : \
unary_row_offset(prow, ne01, ne02, div_ne01, div_ne02, div_ne012, nb01, nb02, \
nb03)) + \
(size_t) pcol * sizeof(float); \
dma_queue_push(dmaq, dma_make_ptr(src_vtcm, data_src + psoff), ptb, ptb, ptb, 1); \
} \
\
tile_in_row++; \
col += col_tile; \
if (tile_in_row == tiles_per_row) { \
tile_in_row = 0; \
col = 0; \
row++; \
i01++; \
if (i01 == ne01) { \
i01 = 0; \
} \
} \
\
ptile_in_row++; \
pcol += col_tile; \
if (ptile_in_row == tiles_per_row) { \
ptile_in_row = 0; \
pcol = 0; \
prow++; \
} \
} \
\
dma_queue_flush(dmaq); \
}
static inline void tile_scale_f32(uint8_t * dst_vtcm, const uint8_t * src_vtcm, uint32_t tw, const int32_t * op_params) {
@@ -1146,14 +1149,32 @@ static int execute_op_unary(struct htp_ops_context * octx) {
const struct htp_unary_kernel_params * kparams = (const struct htp_unary_kernel_params *) octx->kernel_params;
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const uint32_t n_threads = kparams->n_threads;
const size_t elem_size = is_f16 ? sizeof(_Float16) : sizeof(float);
const size_t src0_data_row_size = src0->ne[0] * elem_size;
const size_t dst_data_row_size = dst->ne[0] * elem_size;
uint32_t row_start = 0;
uint32_t nrows = src0_nrows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, (uint32_t) elem_size, (uint32_t) dst_data_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
const size_t src0_row_size_aligned = kparams->src0_row_size_aligned;
const size_t dst_row_size_aligned = kparams->dst_row_size_aligned;
@@ -1191,8 +1212,9 @@ static int execute_op_unary(struct htp_ops_context * octx) {
struct htp_unary_context uctx = {
.octx = octx,
.kparams = kparams,
.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads,
.src0_nrows = src0_nrows,
.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div),
.src0_nrows = nrows,
.row_start = row_start,
.data_src0 = (const uint8_t *)src0->data,
.data_src1 = (octx->op == HTP_OP_RMS_NORM_MUL) ? (const uint8_t *)src1->data : NULL,
@@ -1287,7 +1309,7 @@ static int execute_op_unary(struct htp_ops_context * octx) {
}
if (task_func) {
worker_pool_run_func(octx->ctx->worker_pool, task_func, &uctx, n_threads);
work_queue_run(octx->ctx->work_queue, task_func, &uctx, n_threads);
} else {
FARF(ERROR, "execute_op_unary: task function is NULL for op %d\n", octx->op);
err = HTP_STATUS_NO_SUPPORT;