upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-23 11:10:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

hexagon: L2 cache handling rework (dirty bit tracking with lazy flushing) and more MUL_MAT updates (#25762)

Browse Source 
* hex-mm: fix artificial limit in the solver that restricted number of act-prep threads

* hex-mm: fix warning

* hex-prof: do not apply --top to the timeline report

* hmx-mm: add suport for tiled act-processing to better distribute hvx work

* hex-l2: add tracing for l2flush events

* workqueue: redo the legacy workpool api to match hmx-queue and dma-queue

* hmx-mm: fix f32 activation buffer alignmnet for nhvx=5,6,7

* hex-work: minor cleanup for work-queue apis

* hex-work: further cleanup of the work-queue api

* hex-l2: optimize l2flushes at the opbatch level

* hex-work: remove unused mask

* hex-work: no need to drop hvx ctx in the work-queue

* hex-work: add explicit wakeup/suspend and make threads spin

* hex-bufs: mark any non-weight tensor as compute

* hex-dma: dma-queue support for alias queues and cached dma

* hex-l2: track tensor aliases and delay or skip flushes as much as possible

* hex-l2: simplify tensor alias handling

* hex-l2: handle overlapping views as a circular list of aliases

* hex-tens: add flags helper

* hex-l2: add helper for marking tensors clearn/dirty

* hex-l2: mark binary and rope outputs as l2-clean and keep the rest as is for now

* hex-l2: proper support for handling all tensor overlap scenarios

* hex-trace: instrument matmul init code and cleanup trace checks

* hex-thread: introduce dedicated main thread with explicit stack and priority

* hex-l2: track dirty state as bitmap and introduce threaded flush

* hex-trace: remove redundant checks for ctx != null

* hex-l2: allocate entire context as one buffer and l2fetch it after big flushes

* hex-l2: disable tensor clearing in binary and rope for now seems to cause issues with fusion

* hmx-mm: update act proc to use fastdivs and fix DMA overflow

* hmx-mm: make MUL_MAT_ID kernels robust to multi-chunk cases (start_row>0)

* hex-queue: remove obsolete queue interfaces and flush hmx-queue at the end of the op-batch

* hex-queue: dont use early wakeup for small op-batches

* hex-tensors: properly cap max_tensors in op-batches and dirty_map

* hex-l2: make sure threaded l2flush does proper rounding

* hex-l2: factor out htp_tensor_flush for reuse (if needed)

* hex-l2: optimize tensor flushes by coalescing flush-all

* hex-l2: optimize multi-threaded flush

* hex-drv: futureproof version checks

* hexagon: fix errors and warnings on windows

* hex-main: update main thread to only use dspqueue_read, dspqueue_peek is not available on some platforms

* hex-main: add fallback mode for dspqueue with callbacks

* hex-main: introduce fallback mode for using dspqueue callbacks for full op processing

* hex-main: remove early wakeup, not helping and seems to cause some errors with certain batch sizes

* hex-l2: make sure to use invalidate version of flushall

* hex-l2: dont try to trace early l2flush at the start of op-batch

* hex-main: remove offset_ctx that must be zero anyway

* hex-hmx: fix hmx_queue_depth to use idx_write - idx_read

* hex-hmx: use atomic_load for idx_read/write

* hex-main: add static assert to make sure n_threads are aligned
This commit is contained in:
Max Krasnyansky
2026-07-16 09:28:04 -07:00
committed by GitHub
parent f15bd60901
commit b2dd28a3b6
33 changed files with 2476 additions and 1532 deletions
Download Patch File Download Diff File Expand all files Collapse all files

325
ggml/src/ggml-hexagon/ggml-hexagon.cpp
View File

@@ -21,6 +21,11 @@
#include <algorithm>
#ifdef _WIN32
# define WIN32_LEAN_AND_MEAN
# ifndef NOMINMAX
# define NOMINMAX
# endif
# include <windows.h>
# include <sal.h>
#else
# include <semaphore.h>
@@ -28,7 +33,9 @@
#endif
#pragma clang diagnostic ignored "-Wnested-anon-types"
#pragma clang diagnostic ignored "-Wlanguage-extension-token"
#pragma clang diagnostic ignored "-Wgnu-anonymous-struct"
#pragma clang diagnostic ignored "-Wmicrosoft-enum-value"
#include <AEEStdErr.h>
#include <dspqueue.h>
@@ -134,6 +141,8 @@ static const char * htp_event_name(uint16_t id) {
case HTP_TRACE_EVT_HVX_FA_K_PREP: return "HVX_K_PREP";
case HTP_TRACE_EVT_HVX_FA_V_PREP: return "HVX_V_PREP";
case HTP_TRACE_EVT_HMX_COMP: return "HMX_COMP";
case HTP_TRACE_EVT_L2FLUSH: return "L2FLUSH";
case HTP_TRACE_EVT_INIT: return "INIT";
default: return "UNKNOWN";
}
}
@@ -501,6 +510,8 @@ static void repack_q4_0_tiled(ggml_tensor * t, const void * data, size_t size) {
}
}
}
GGML_UNUSED(size);
}
// repack q4_0_tiled tensor into q4_0 data
@@ -554,6 +565,8 @@ static void repack_tiled_q4_0(void * data, const ggml_tensor * t, size_t size) {
}
}
}
GGML_UNUSED(size);
}
// repack q4_1 data into q4_1_tiled tensor
@@ -611,6 +624,8 @@ static void repack_q4_1_tiled(ggml_tensor * t, const void * data, size_t size) {
}
}
}
GGML_UNUSED(size);
}
// repack q4_1_tiled tensor into q4_1 data
@@ -665,6 +680,8 @@ static void repack_tiled_q4_1(void * data, const ggml_tensor * t, size_t size) {
}
}
}
GGML_UNUSED(size);
}
// repack q8_0 data into q8_0_tiled tensor
@@ -711,6 +728,8 @@ static void repack_q8_0_tiled(ggml_tensor * t, const void * data, size_t size) {
}
}
}
GGML_UNUSED(size);
}
// repack q8_0_tiled tensor into q8_0 data
@@ -761,6 +780,8 @@ static void repack_tiled_q8_0(void * data, const ggml_tensor * t, size_t size) {
}
}
}
GGML_UNUSED(size);
}
// repack mxfp4 data into mxfp4_tiled tensor
@@ -812,6 +833,8 @@ static void repack_mxfp4_tiled(ggml_tensor * t, const void * data, size_t size)
}
}
}
GGML_UNUSED(size);
}
// repack mxfp4_tiled tensor into mxfp4 data
@@ -865,6 +888,8 @@ static void repack_tiled_mxfp4(void * data, const ggml_tensor * t, size_t size)
}
}
}
GGML_UNUSED(size);
}
static void ggml_backend_hexagon_buffer_set_tensor(ggml_backend_buffer_t buffer,
@@ -965,11 +990,12 @@ static void ggml_backend_hexagon_buffer_get_tensor(ggml_backend_buffer_t buffer,
static bool ggml_backend_hexagon_buffer_cpy_tensor(ggml_backend_buffer_t buffer,
const struct ggml_tensor * src,
struct ggml_tensor * dst) {
// we might optimize this later, for now take the slow path (ie get/set_tensor)
return false;
GGML_UNUSED(buffer);
GGML_UNUSED(src);
GGML_UNUSED(dst);
// we might optimize this later, for now take the slow path (ie get/set_tensor)
return false;
}
static void ggml_backend_hexagon_buffer_clear(ggml_backend_buffer_t buffer, uint8_t value) {
@@ -1025,9 +1051,9 @@ static ggml_backend_buffer_t ggml_backend_hexagon_repack_buffer_type_alloc_buffe
}
}
static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer_type_t buffer_type) {
static size_t ggml_backend_hexagon_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {
return 128; // HVX alignment
GGML_UNUSED(buffer_type);
GGML_UNUSED(buft);
}
static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * t) {
@@ -1039,20 +1065,24 @@ static size_t ggml_backend_hexagon_buffer_type_get_alloc_size(ggml_backend_buffe
return ggml_row_size(t->type, ne0) * ne1 * ne2 * ne3;
}
return ggml_nbytes(t);
GGML_UNUSED(buft);
}
static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buffer_type) {
auto * context = static_cast<ggml_backend_hexagon_buffer_type_context *>(buffer_type->context);
static size_t ggml_backend_hexagon_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {
auto * context = static_cast<ggml_backend_hexagon_buffer_type_context *>(buft->context);
return context->sess->max_bufsize;
}
static bool ggml_backend_hexagon_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
return opt_hostbuf;
GGML_UNUSED(buft);
}
static bool ggml_backend_hexagon_repack_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
return false;
GGML_UNUSED(buft);
}
@@ -1098,6 +1128,14 @@ struct ggml_hexagon_opbatch {
std::unordered_map<const ggml_tensor*, int> t_map; // tensor ptr to index
std::unordered_multimap<void*, int> d_map; // tensor data to index
struct tensor_range {
uint64_t start;
uint64_t end;
int bi;
std::vector<int> tensors;
};
std::vector<tensor_range> ranges;
unsigned int n_bufs; // num buffers in the batch
unsigned int n_tens; // num tensors ...
unsigned int n_ops; // num ops ...
@@ -1117,6 +1155,7 @@ struct ggml_hexagon_opbatch {
b_map.clear();
t_map.clear();
d_map.clear();
ranges.clear();
}
ggml_hexagon_opbatch(ggml_hexagon_session *sess, size_t batch_size, size_t max_vmem) {
@@ -1124,7 +1163,7 @@ struct ggml_hexagon_opbatch {
n_bufs_max = HTP_OP_MAX_BUFS;
n_ops_max = batch_size;
n_tens_max = n_ops_max + n_ops_max * HTP_OP_MAX_INPUTS;
n_tens_max = std::min<size_t>(n_ops_max + n_ops_max * HTP_OP_MAX_INPUTS, HTP_OP_MAX_TENSORS);
b_vmem_max = max_vmem;
@@ -1170,6 +1209,71 @@ struct ggml_hexagon_opbatch {
return bi;
}
void add_range(const htp_tensor * h, int ti) {
uint64_t t_start = h->data;
uint64_t t_end = t_start + h->size;
int bi = h->bi;
int first_match = -1;
int unused_idx = -1;
for (size_t i = 0; i < ranges.size(); i++) {
if (ranges[i].bi == -1) {
unused_idx = i;
continue;
}
if (ranges[i].bi != bi) {
continue;
}
if (ranges[i].start >= t_end || ranges[i].end <= t_start) {
continue;
}
if (first_match == -1) {
first_match = i;
HEX_VERBOSE("ggml-hex: %s range-grow #%d : bi %d [%p, %p) + #%d [%p, %p) -> [%p, %p)\n",
sess->c_name(), (int) i, ranges[i].bi,
(void *) (h_bufs[ranges[i].bi].base + ranges[i].start),
(void *) (h_bufs[ranges[i].bi].base + ranges[i].end),
ti,
(void *) (h_bufs[bi].base + t_start),
(void *) (h_bufs[bi].base + t_end),
(void *) (h_bufs[ranges[i].bi].base + std::min(ranges[i].start, t_start)),
(void *) (h_bufs[ranges[i].bi].base + std::max(ranges[i].end, t_end)));
ranges[i].start = std::min(ranges[i].start, t_start);
ranges[i].end = std::max(ranges[i].end, t_end);
ranges[i].tensors.push_back(ti);
} else {
HEX_VERBOSE("ggml-hex: %s range-merge #%d [%p, %p) + #%d [%p, %p) -> [%p, %p)\n",
sess->c_name(), first_match,
(void *) (h_bufs[bi].base + ranges[first_match].start),
(void *) (h_bufs[bi].base + ranges[first_match].end),
(int) i,
(void *) (h_bufs[bi].base + ranges[i].start),
(void *) (h_bufs[bi].base + ranges[i].end),
(void *) (h_bufs[bi].base + std::min(ranges[first_match].start, ranges[i].start)),
(void *) (h_bufs[bi].base + std::max(ranges[first_match].end, ranges[i].end)));
ranges[first_match].start = std::min(ranges[first_match].start, ranges[i].start);
ranges[first_match].end = std::max(ranges[first_match].end, ranges[i].end);
ranges[first_match].tensors.insert(
ranges[first_match].tensors.end(),
ranges[i].tensors.begin(),
ranges[i].tensors.end()
);
ranges[i].bi = -1;
}
}
if (first_match == -1) {
if (unused_idx != -1) {
ranges[unused_idx] = {t_start, t_end, bi, {ti}};
} else {
ranges.push_back({t_start, t_end, bi, {ti}});
}
}
}
bool same_shape(const htp_tensor * h, const ggml_tensor * t) const {
int64_t ne0 = t->ne[0];
int64_t ne1 = t->ne[1];
@@ -1213,6 +1317,7 @@ struct ggml_hexagon_opbatch {
htp_tensor &h = h_tens[ti];
h.bi = add_buffer(sbuf);
h.ti = ti;
h.data = t_offset;
h.type = t->type;
@@ -1235,8 +1340,11 @@ struct ggml_hexagon_opbatch {
h.nb[0] = t->nb[0]; h.nb[1] = t->nb[1]; h.nb[2] = t->nb[2]; h.nb[3] = t->nb[3];
}
h.alias = ti;
add_range(&h, ti);
h.flags = 0;
if (ggml_backend_buffer_get_usage(t->buffer) == GGML_BACKEND_BUFFER_USAGE_COMPUTE) {
if (ggml_backend_buffer_get_usage(t->buffer) != GGML_BACKEND_BUFFER_USAGE_WEIGHTS) {
h.flags |= HTP_TENSOR_COMPUTE;
}
@@ -1313,6 +1421,17 @@ struct ggml_hexagon_opbatch {
o.dst[i] = (i < outputs.size() && outputs[i]) ? add_tensor(outputs[i]) : 0xffff;
}
}
void finalize_ranges() {
for (const auto & r : ranges) {
if (r.bi == -1) {
continue;
}
for (size_t i = 0; i < r.tensors.size(); i++) {
h_tens[r.tensors[i]].alias = r.tensors[(i + 1) % r.tensors.size()];
}
}
}
};
struct ggml_hexagon_opqueue {
@@ -1571,6 +1690,8 @@ void ggml_hexagon_session::flush_pending(bool all) {
void ggml_hexagon_session::flush_batch() {
if (op_batch->empty()) { return; }
op_batch->finalize_ranges();
htp_opbatch_req req {};
dspqueue_buffer dbuf{};
@@ -1647,7 +1768,7 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
GGML_LOG_DEBUG("ggml-hex: %s allocating new session\n", this->name.c_str());
domain * my_domain = get_domain(this->domain_id);
domain * my_domain = htpdrv_get_domain(this->domain_id);
if (my_domain == NULL) {
GGML_LOG_ERROR("ggml-hex: unable to get domain struct for CDSP\n");
throw std::runtime_error("ggml-hex: failed to get CDSP domain (see log for details)");
@@ -1793,16 +1914,6 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
}
}
if (opt_profile) {
htp_iface_pmu_conf pmu_conf{};
std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events);
err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf);
if (err != 0) {
GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err);
}
}
// Allocate buffers and state for op batching
this->op_queue = new ggml_hexagon_opqueue(this, opt_opbatch, opt_opqueue);
@@ -1821,6 +1932,16 @@ void ggml_hexagon_session::allocate(int dev_id) noexcept(false) {
throw std::runtime_error("ggml-hex: iface start failed (see log for details)");
}
this->valid_iface = true;
if (opt_profile) {
htp_iface_pmu_conf pmu_conf{};
std::copy(opt_pmu_evt.begin(), opt_pmu_evt.end(), pmu_conf.events);
err = htp_iface_profiler(this->handle, opt_profile, &pmu_conf);
if (err != 0) {
GGML_LOG_ERROR("ggml-hex: failed to enable profiling: 0x%08x\n", (unsigned) err);
}
}
}
void ggml_hexagon_session::release() noexcept(true) {
@@ -1929,6 +2050,8 @@ static bool ggml_hexagon_flash_attn_is_hmx_eligible(
}
return true;
GGML_UNUSED(sinks);
}
static bool ggml_hexagon_precompute_flash_attn_params(
@@ -2149,8 +2272,9 @@ static bool ggml_hexagon_supported_gated_delta_net(const struct ggml_hexagon_ses
return false;
}
GGML_UNUSED(sess);
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_matmul_is_hmx_eligible(
@@ -2198,6 +2322,8 @@ static bool ggml_hexagon_matmul_is_hmx_eligible(
}
return true;
GGML_UNUSED(dst);
}
static bool ggml_hexagon_precompute_hmx_mm_params(
@@ -2234,109 +2360,15 @@ static bool ggml_hexagon_precompute_hmx_mm_params(
if (is_batched_val && wtype == GGML_TYPE_F16 && group_size > 1) {
// Try grouped path first
const bool use_dma_activation = (src1->nb[1]/sizeof(float) > (size_t)ne00_padded);
size_t best_mblocks = SIZE_MAX;
int best_act_threads = 0;
size_t best_m_chunk = 0;
size_t best_n_chunk = 0;
size_t best_vtcm_size = 0;
int act_threads = n_threads;
while (act_threads >= 1) {
const size_t f32_scratch_size = use_dma_activation ? hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * ne00_padded * sizeof(float), HTP_MM_HMX_TILE_SIZE) : 0;
size_t group_overhead = 256 + f32_scratch_size;
size_t group_size_per_n, group_size_per_m, group_size_per_mn;
htp_mm_hmx_get_batched_chunk_costs(ne00_padded, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn);
size_t m_chunk_candidate = 0;
size_t n_chunk_candidate = 0;
size_t vtcm_size_candidate = 0;
if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded,
(size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT,
&m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) {
size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, ne00_padded, m_chunk_candidate, n_chunk_candidate, group_size, use_dma_activation, pipeline, act_threads);
if (exact_size <= vtcm_budget) {
size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate;
if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) {
best_mblocks = mblocks;
best_act_threads = act_threads;
best_m_chunk = m_chunk_candidate;
best_n_chunk = n_chunk_candidate;
best_vtcm_size = exact_size;
}
}
}
if (act_threads == 1) {
act_threads = 0;
} else {
act_threads /= 2;
}
}
if (best_act_threads > 0) {
m_chunk = best_m_chunk;
n_chunk = best_n_chunk;
vtcm_size = best_vtcm_size;
act_threads_selected = best_act_threads;
if (htp_mm_hmx_solve_batched_params(wtype, ne00_padded, ne01_padded, ne11, group_size, use_dma_activation, n_threads, pipeline, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {
use_grouped = true;
}
}
if (!use_grouped) {
// Fallback to simple 2D path (group_size = 1)
size_t best_mblocks = SIZE_MAX;
int best_act_threads = 0;
size_t best_m_chunk = 0;
size_t best_n_chunk = 0;
size_t best_vtcm_size = 0;
// For MUL_MAT_ID the kernel runs one 2D matmul per expert, with M equal to the number of rows routed to that expert.
// A single expert can receive up to all routed rows (dst->ne[1]*dst->ne[2] = n_expert_used*n_tokens), so size the chunk
// search for that upper bound rather than ne12 (token positions only).
// We recompute m_chunk per expert against the actual count in the NPU kernel.
const int m_id_rows = (int) ((size_t) dst->ne[1] * dst->ne[2]);
const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded;
const int m_for_cost = is_matmul_id ? m_id_rows : ne11;
int act_threads = n_threads;
while (act_threads >= 1) {
const size_t act_f32_size = is_matmul_id ? 0 : hex_align_up(act_threads * HTP_MM_DMA_ACT_MULTIPLIER * ne00_padded * sizeof(float), HTP_MM_HMX_TILE_SIZE);
size_t simple_2d_overhead = 256 + act_f32_size;
size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn;
htp_mm_hmx_get_2d_chunk_costs(wtype, ne00_padded, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn);
size_t m_chunk_candidate = 0;
size_t n_chunk_candidate = 0;
size_t vtcm_size_candidate = 0;
if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded,
(size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT,
&m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) {
size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, ne00_padded, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size);
if (exact_size <= vtcm_budget) {
size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate;
if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) {
best_mblocks = mblocks;
best_act_threads = act_threads;
best_m_chunk = m_chunk_candidate;
best_n_chunk = n_chunk_candidate;
best_vtcm_size = exact_size;
}
}
}
if (act_threads == 1) {
act_threads = 0;
} else {
act_threads /= 2;
}
}
if (best_act_threads > 0) {
m_chunk = best_m_chunk;
n_chunk = best_n_chunk;
vtcm_size = best_vtcm_size;
act_threads_selected = best_act_threads;
} else {
const int m_id_rows = (int) ((size_t) dst->ne[1] * dst->ne[2]);
if (!htp_mm_hmx_solve_2d_params(wtype, ne00_padded, m_id_rows, ne01_padded, ne11_padded, ne11, n_threads, pipeline, is_matmul_id, aligned_tile_size, vtcm_budget, &m_chunk, &n_chunk, &act_threads_selected, &vtcm_size)) {
return false;
}
}
@@ -2352,6 +2384,8 @@ static bool ggml_hexagon_precompute_hmx_mm_params(
kparams->src1_row_size = (wtype == GGML_TYPE_Q4_1) ? htp_mm_q8_1_tiled_row_size(ne10) : htp_mm_q8_0_tiled_row_size(ne10);
kparams->vtcm_size = vtcm_size;
kparams->vtcm_src0_size = 0;
kparams->div_n_act_threads = init_fastdiv_values(act_threads_selected);
kparams->div_ne00_padded = init_fastdiv_values(ne00_padded);
kparams->vtcm_src1_size = 0;
kparams->vtcm_dst_size = 0;
@@ -2361,6 +2395,8 @@ static bool ggml_hexagon_precompute_hmx_mm_params(
kparams->kernel_type = HTP_MM_KERNEL_HMX_2D;
}
return true;
GGML_UNUSED(src0);
}
static void ggml_hexagon_precompute_hvx_mm_params(
@@ -2955,6 +2991,8 @@ static bool ggml_hexagon_supported_binary(const struct ggml_hexagon_session * se
}
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -2981,6 +3019,8 @@ static bool ggml_hexagon_supported_add_id(const struct ggml_hexagon_session * se
}
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -3006,6 +3046,8 @@ static bool ggml_hexagon_supported_unary(const struct ggml_hexagon_session * ses
}
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_sum_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -3025,10 +3067,11 @@ static bool ggml_hexagon_supported_sum_rows(const struct ggml_hexagon_session *
}
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op) {
static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * src1 = op->src[1];
const struct ggml_tensor * dst = op;
@@ -3057,6 +3100,8 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session
}
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -3122,6 +3167,8 @@ static bool ggml_hexagon_supported_softmax(const struct ggml_hexagon_session * s
}
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -3142,6 +3189,8 @@ static bool ggml_hexagon_supported_set_rows(const struct ggml_hexagon_session *
}
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -3162,6 +3211,8 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *
}
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -3182,6 +3233,8 @@ static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * s
}
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -3243,6 +3296,8 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess
return false;
}
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -3282,6 +3337,8 @@ static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session *
}
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -3292,8 +3349,9 @@ static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess,
return false;
}
GGML_UNUSED(sess);
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_cumsum(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -3308,8 +3366,9 @@ static bool ggml_hexagon_supported_cumsum(const struct ggml_hexagon_session * se
return false;
}
GGML_UNUSED(sess);
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_diag(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -3331,8 +3390,9 @@ static bool ggml_hexagon_supported_diag(const struct ggml_hexagon_session * sess
return false;
}
GGML_UNUSED(sess);
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_solve_tri(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -3364,8 +3424,9 @@ static bool ggml_hexagon_supported_solve_tri(const struct ggml_hexagon_session *
return false;
}
GGML_UNUSED(sess);
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_tri(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -3812,6 +3873,8 @@ static void ggml_backend_hexagon_graph_optimize(ggml_backend_t backend, ggml_cgr
}
}
}
GGML_UNUSED(backend);
}
static struct ggml_backend_i hexagon_backend_i = {
@@ -3930,6 +3993,8 @@ static bool ggml_hexagon_supported_buffers(ggml_hexagon_session *sess, const str
}
static bool ggml_hexagon_supported_cpy(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
GGML_UNUSED(sess);
const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * dst = op;
@@ -4000,6 +4065,7 @@ static bool ggml_hexagon_supported_concat(const struct ggml_hexagon_session * se
}
return true;
GGML_UNUSED(sess);
}
static bool ggml_hexagon_supported_fill(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -4009,8 +4075,8 @@ static bool ggml_hexagon_supported_fill(const struct ggml_hexagon_session * sess
return false;
}
GGML_UNUSED(sess);
return true;
GGML_UNUSED(sess);
}
static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {
@@ -4293,6 +4359,7 @@ static void * ggml_backend_hexagon_get_proc_address(ggml_backend_reg_t reg, cons
}
return NULL;
GGML_UNUSED(reg);
}
template<typename T> std::vector<T> str_to_vec(const char* str) {
@@ -4351,10 +4418,18 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) {
// Init Arch first since it affects other defaults
if (!str_arch) {
int err = get_hex_arch_ver(CDSP_DOMAIN_ID, &opt_arch);
int err = htpdrv_get_arch(CDSP_DOMAIN_ID, &opt_arch);
if (err != 0) {
GGML_LOG_ERROR("ggml-hex: failed to query HTP version (err %d) defaulting to v73\n", err);
opt_arch = 73;
} else {
if (opt_arch < 73) {
GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is under supported range, capping at v73\n", opt_arch);
opt_arch = 73;
} else if (opt_arch > 81) {
GGML_LOG_WARN("ggml-hex: Hexagon arch v%d is over supported range, capping at v81\n", opt_arch);
opt_arch = 81;
}
}
} else {
if (str_arch[0] == 'v' || str_arch[0] == 'V') {

47
ggml/src/ggml-hexagon/htp-drv.cpp
View File

@@ -1,13 +1,8 @@
// sample drv interface
#pragma clang diagnostic ignored "-Wgnu-anonymous-struct"
#pragma clang diagnostic ignored "-Wmissing-prototypes"
#pragma clang diagnostic ignored "-Wsign-compare"
#include <filesystem>
#include <set>
#include <sstream>
#include <string>
#ifdef _WIN32
# define WIN32_LEAN_AND_MEAN
# ifndef NOMINMAX
@@ -16,9 +11,17 @@
# include <windows.h>
# include <winevt.h>
#else
# include <dlfcn.h>
# include <unistd.h>
# include <dlfcn.h>
# include <unistd.h>
#endif
#pragma clang diagnostic ignored "-Wgnu-anonymous-struct"
#pragma clang diagnostic ignored "-Wmissing-prototypes"
#pragma clang diagnostic ignored "-Wsign-compare"
#pragma clang diagnostic ignored "-Wlanguage-extension-token"
#pragma clang diagnostic ignored "-Wmicrosoft-enum-value"
#pragma clang diagnostic ignored "-Wnested-anon-types"
#include "ggml-impl.h"
#include "htp-drv.h"
#include "libdl.h"
@@ -359,7 +362,7 @@ int htpdrv_init() {
return AEE_SUCCESS;
}
domain * get_domain(int domain_id) {
domain * htpdrv_get_domain(int domain_id) {
int i = 0;
int size = sizeof(supported_domains) / sizeof(domain);
@@ -372,7 +375,7 @@ domain * get_domain(int domain_id) {
return NULL;
}
int get_hex_arch_ver(int domain, int * arch) {
int htpdrv_get_arch(int domain, int * arch) {
if (!remote_handle_control_pfn) {
GGML_LOG_ERROR("ggml-hex: remote_handle_control is not supported on this device\n");
return AEE_EUNSUPPORTEDAPI;
@@ -394,25 +397,7 @@ int get_hex_arch_ver(int domain, int * arch) {
return err;
}
switch (arch_ver.capability & 0xff) {
case 0x68:
*arch = 68;
return 0;
case 0x69:
*arch = 69;
return 0;
case 0x73:
*arch = 73;
return 0;
case 0x75:
*arch = 75;
return 0;
case 0x79:
*arch = 79;
return 0;
case 0x81:
*arch = 81;
return 0;
}
return -1;
uint32_t val = arch_ver.capability & 0xff;
*arch = (int) ((val >> 4) * 10 + (val & 0x0f));
return 0;
}

8
ggml/src/ggml-hexagon/htp-drv.h
View File

@@ -96,17 +96,17 @@ extern "C" {
HTPDRV_API int htpdrv_init(void);
/**
* get_domain API: get domain struct from domain value.
* htpdrv_get_domain API: get domain struct from domain value.
*
* @param[in] domain value of a domain
* @return Returns domain struct of the domain if it is supported or else
* returns NULL.
*
*/
HTPDRV_API domain * get_domain(int domain_id);
HTPDRV_API domain * htpdrv_get_domain(int domain_id);
/**
* get_hex_arch_ver API: query the Hexagon processor architecture version information
* htpdrv_get_arch API: query the Hexagon processor architecture version information
*
* @param[in] domain_id value of a domain
* @param[out] Arch version (73, 75, ...)
@@ -114,7 +114,7 @@ HTPDRV_API domain * get_domain(int domain_id);
* non-zero if error, return value points to the error.
*
*/
HTPDRV_API int get_hex_arch_ver(int domain, int * arch);
HTPDRV_API int htpdrv_get_arch(int domain, int * arch);
#ifdef __cplusplus
}

9
ggml/src/ggml-hexagon/htp/CMakeLists.txt
View File

@@ -17,9 +17,12 @@ set(HTP_LIB ggml-htp-${DSP_VERSION})
add_library(${HTP_LIB} SHARED
main.c
htp_iface_skel.c
worker-pool.c
hex-dma.c
work-queue.c
dma-queue.c
hmx-queue.c
htp-tensor.c
matmul-ops.c
flash-attn-ops.c
gated-delta-net-ops.c
binary-ops.c
unary-ops.c
@@ -38,8 +41,6 @@ add_library(${HTP_LIB} SHARED
diag-ops.c
solve-tri-ops.c
pad-ops.c
flash-attn-ops.c
matmul-ops.c
argsort-ops.c
)

11
ggml/src/ggml-hexagon/htp/act-ops.c
View File

@@ -16,6 +16,7 @@
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#define htp_act_preamble \
const struct htp_tensor * src0 = actx->octx->src[0]; \
@@ -766,17 +767,11 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
}
int op_activations(struct htp_ops_context * octx) {
int err = HTP_STATUS_OK;
switch (octx->src[0]->type) {
case HTP_TYPE_F32:
err = execute_op_activations_f32(octx);
break;
return execute_op_activations_f32(octx);
default:
err = HTP_STATUS_NO_SUPPORT;
break;
return HTP_STATUS_NO_SUPPORT;
}
return err;
}

4
ggml/src/ggml-hexagon/htp/argsort-ops.c
View File

@@ -345,7 +345,7 @@ static void htp_argsort_f32_##ne00##_##order_name(unsigned int n, unsigned int i
int32_t * indices_buf = (int32_t *) (spad + values_size); \
uint32_t nb01 = src0->nb[1]; \
uint32_t nb1 = dst->nb[1]; \
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[i] : NULL; \
struct htp_thread_trace * tr = &octx->ctx->trace[i]; \
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start_row); \
for (uint32_t r = start_row; r < end_row; r++) { \
uint32_t src_offset = r * nb01; \
@@ -411,7 +411,7 @@ static void htp_argsort_f32_fallback(unsigned int n, unsigned int i, void * data
const HVX_Vector ind_init_vec = *(HVX_Vector *)argosrt_ramp_lut;
const HVX_Vector ind_diff_vec = Q6_V_vsplat_R(32);
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[i] : NULL;
struct htp_thread_trace * tr = &octx->ctx->trace[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, start_row);
for (uint32_t r = start_row; r < end_row; r++) {

1
ggml/src/ggml-hexagon/htp/binary-ops.c
View File

@@ -16,6 +16,7 @@
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#ifndef MIN
#define MIN(a, b) ((a) < (b) ? (a) : (b))

11
ggml/src/ggml-hexagon/htp/cumsum-ops.c
View File

@@ -9,6 +9,7 @@
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hvx-types.h"
#include "hvx-utils.h"
#include "hex-dma.h"
@@ -255,16 +256,10 @@ int op_cumsum_f32(struct htp_ops_context * octx) {
int op_cumsum(struct htp_ops_context * octx) {
const struct htp_tensor * dst = octx->dst;
int err = HTP_STATUS_OK;
switch (dst->type) {
case HTP_TYPE_F32:
err = op_cumsum_f32(octx);
break;
return op_cumsum_f32(octx);
default:
err = HTP_STATUS_NO_SUPPORT;
break;
return HTP_STATUS_NO_SUPPORT;
}
return err;
}

106
ggml/src/ggml-hexagon/htp/dma-queue.c Normal file
View File

@@ -0,0 +1,106 @@
#include "dma-queue.h"
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#pragma clang diagnostic ignored "-Wunused-function"
static inline uint32_t pow2_ceil(uint32_t x) {
if (x <= 1) {
return 1;
}
int p = 2;
x--;
while (x >>= 1) {
p <<= 1;
}
return p;
}
static inline uintptr_t align_up(uintptr_t addr, size_t align) {
return (addr + align - 1) & ~(align - 1);
}
size_t dma_queue_sizeof(size_t capacity) {
capacity = pow2_ceil(capacity);
size_t size_q = sizeof(dma_queue);
size_t offset_r = align_up(size_q, HEX_L2_LINE_SIZE);
size_t size_r = sizeof(dma_ring);
size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE);
size_t size_desc = capacity * sizeof(dma_descriptor_2d);
size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE);
size_t size_dptr = capacity * sizeof(dma_ptr);
return offset_dptr + size_dptr;
}
size_t dma_queue_alignof(void) {
return HEX_L2_LINE_SIZE;
}
dma_queue_t dma_queue_init(void * ptr, size_t capacity, uintptr_t vtcm_base, size_t vtcm_size, struct htp_thread_trace * trace) {
capacity = pow2_ceil(capacity);
size_t size_q = sizeof(dma_queue);
size_t offset_r = align_up(size_q, HEX_L2_LINE_SIZE);
size_t size_r = sizeof(dma_ring);
size_t offset_desc = align_up(offset_r + size_r, HEX_L2_LINE_SIZE);
size_t size_desc = capacity * sizeof(dma_descriptor_2d);
size_t offset_dptr = align_up(offset_desc + size_desc, HEX_L2_LINE_SIZE);
size_t size_dptr = capacity * sizeof(dma_ptr);
size_t total_size = offset_dptr + size_dptr;
memset(ptr, 0, total_size);
dma_queue * q = (dma_queue *) ptr;
dma_ring * r = (dma_ring *) ((uintptr_t) ptr + offset_r);
q->ring = r;
q->nocache = 0;
q->alias = false;
r->trace = trace;
r->vtcm_base = vtcm_base;
r->vtcm_end = vtcm_base + vtcm_size;
r->capacity = capacity;
r->idx_mask = capacity - 1;
r->push_idx = 0;
r->pop_idx = 0;
r->desc = (dma_descriptor_2d *) ((uintptr_t) ptr + offset_desc);
r->dptr = (dma_ptr *) ((uintptr_t) ptr + offset_dptr);
r->tail = &r->desc[capacity - 1];
FARF(HIGH, "dma-queue: capacity %u, unified memory size %zu\n", capacity, total_size);
return q;
}
void dma_queue_free(dma_queue_t q) {
(void) q;
}
size_t dma_queue_alias_sizeof(void) {
return sizeof(dma_queue);
}
dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q, uint8_t nocache) {
dma_queue * q = (dma_queue *) ptr;
memset(q, 0, sizeof(dma_queue));
q->ring = main_q->ring;
q->nocache = nocache;
q->alias = true;
return q;
}
void dma_queue_alias_free(dma_queue_t q) {
(void) q;
}
void dma_queue_flush(dma_queue_t q) {
while (dma_queue_pop(q).dst != NULL) ;
}

396
ggml/src/ggml-hexagon/htp/dma-queue.h Normal file
View File

@@ -0,0 +1,396 @@
#ifndef HTP_DMA_H
#define HTP_DMA_H
#include <HAP_farf.h>
#include <hexagon_types.h>
#include <stdbool.h>
#include <stdint.h>
#include "hex-utils.h"
#include "hex-profile.h"
#ifdef __cplusplus
extern "C" {
#endif
// Define the HW descriptor structs here since the ones in HexSDK are a bit out of date
typedef struct dma_descriptor_1d_s {
void * next;
uint32_t size:24;
uint32_t desc_size:2;
uint32_t dst_comp:1;
uint32_t src_comp:1;
uint32_t dst_bypass:1;
uint32_t src_bypass:1;
uint32_t order:1;
uint32_t done:1;
void * src;
void * dst;
} dma_descriptor_1d;
#if __HVX_ARCH__ < 75
typedef struct dma_descriptor_2d_s {
void * next;
uint32_t reserved0:24;
uint32_t desc_size:2;
uint32_t dst_comp:1;
uint32_t src_comp:1;
uint32_t dst_bypass:1;
uint32_t src_bypass:1;
uint32_t order:1;
uint32_t done:1;
void * src;
void * dst;
uint32_t desc_type:8;
uint32_t reserved1:24;
uint32_t row_size:16;
uint32_t nrows:16;
uint32_t src_stride:16;
uint32_t dst_stride:16;
uint32_t src_offset:16;
uint32_t dst_offset:16;
} dma_descriptor_2d;
#else
typedef struct dma_descriptor_2d_s {
void * next;
uint32_t dst_stride:24;
uint32_t desc_size:2;
uint32_t dst_comp:1;
uint32_t src_comp:1;
uint32_t dst_bypass:1;
uint32_t src_bypass:1;
uint32_t order:1;
uint32_t done:1;
void * src;
void * dst;
uint32_t desc_type:8;
uint32_t reserved0:24;
uint32_t row_size:24;
uint32_t nrows_lo:8;
uint32_t nrows_hi:8;
uint32_t src_stride:24;
uint32_t offset:24;
uint32_t reserved1:8;
} dma_descriptor_2d;
#endif
typedef struct {
void *dst;
const void *src;
} dma_ptr;
typedef struct dma_ring_s dma_ring;
struct dma_ring_s {
dma_descriptor_2d * desc; // descriptor pointers
dma_descriptor_2d * tail; // tail pointer
dma_ptr * dptr; // dst/src pointers
uint32_t push_idx;
uint32_t pop_idx;
uint32_t capacity;
uint32_t idx_mask;
struct htp_thread_trace * trace;
uintptr_t vtcm_base;
uintptr_t vtcm_end;
};
typedef struct dma_queue_s dma_queue;
typedef dma_queue * dma_queue_t;
struct dma_queue_s {
dma_ring * ring; // Points to the descriptor ring state
uint8_t nocache; // Queue-specific bypass flag
bool alias; // When set, dma_queue_delete will not free the ring
};
void dma_queue_flush(dma_queue_t q);
size_t dma_queue_sizeof(size_t capacity);
size_t dma_queue_alignof(void);
dma_queue_t dma_queue_init(void * ptr, size_t capacity, uintptr_t vtcm_base, size_t vtcm_size, struct htp_thread_trace * trace);
void dma_queue_free(dma_queue_t q);
size_t dma_queue_alias_sizeof(void);
dma_queue_t dma_queue_alias_init(void * ptr, dma_queue_t main_q, uint8_t nocache);
void dma_queue_alias_free(dma_queue_t q);
// TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead
// but those do not seem to always compiler properly.
static inline void dmstart(void * next) {
asm volatile(" release(%0):at" : : "r"(next));
asm volatile(" dmstart(%0)" : : "r"(next));
}
static inline void dmlink(void * cur, void * next) {
asm volatile(" release(%0):at" : : "r"(next));
asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next));
}
static inline unsigned int dmpoll(void) {
unsigned int ret = 0;
asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory");
return ret;
}
static inline unsigned int dmwait(void) {
unsigned int ret = 0;
asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory");
return ret;
}
static inline dma_ptr dma_make_ptr(void *dst, const void *src)
{
dma_ptr p = { dst, src };
return p;
}
static inline bool dma_is_vtcm(const dma_queue * q, const void * ptr) {
return (uintptr_t) ptr >= q->ring->vtcm_base && (uintptr_t) ptr < q->ring->vtcm_end;
}
static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t size) {
dma_ring * r = q->ring;
if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) {
FARF(HIGH, "dma-push: queue full\n");
return false;
}
dma_descriptor_1d * desc = (dma_descriptor_1d *) &r->desc[r->push_idx];
desc->src = (void *) dptr.src;
desc->dst = (void *) dptr.dst;
desc->size = size;
r->dptr[r->push_idx] = dptr;
if (size) {
desc->next = NULL;
desc->desc_size = 0; // 1D mode
desc->src_bypass = dma_is_vtcm(q, dptr.src) ? 1 : q->nocache;
desc->dst_bypass = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache;
desc->order = 0;
desc->done = 0;
htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx);
dmlink(r->tail, desc);
r->tail = (dma_descriptor_2d *) desc;
} else {
desc->desc_size = 0;
desc->done = 1;
}
r->push_idx = (r->push_idx + 1) & r->idx_mask;
return true;
}
static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
dma_ring * r = q->ring;
if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) {
FARF(HIGH, "dma-push: queue full\n");
return false;
}
dma_descriptor_2d * desc = &r->desc[r->push_idx];
desc->next = NULL;
desc->reserved0 = 0;
desc->reserved1 = 0;
desc->desc_size = 1; // 2d mode
desc->src_bypass = dma_is_vtcm(q, dptr.src) ? 1 : q->nocache;
desc->dst_bypass = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache;
desc->src_comp = 0;
desc->dst_comp = 0;
desc->order = 0;
desc->done = 0;
desc->src_stride = src_stride;
desc->dst_stride = dst_stride;
desc->src = (void *) dptr.src;
desc->dst = (void *) dptr.dst;
desc->row_size = row_size;
#if __HVX_ARCH__ < 75
desc->desc_type = 0; // 2d (16-bit) mode
desc->nrows = nrows;
desc->src_offset = 0;
desc->dst_offset = 0;
#else
desc->desc_type = 9; // 2d (24-bit) mode
desc->nrows_lo = (nrows & 0xff);
desc->nrows_hi = (nrows >> 8);
desc->offset = 0;
#endif
r->dptr[r->push_idx] = dptr;
if (nrows) {
htp_trace_event_start(r->trace, HTP_TRACE_EVT_DMA, r->push_idx);
dmlink(r->tail, desc);
r->tail = desc;
} else {
desc->done = 1;
}
r->push_idx = (r->push_idx + 1) & r->idx_mask;
return true;
}
static inline dma_ptr dma_queue_pop(dma_queue * q) {
dma_ring * r = q->ring;
dma_ptr dptr = { NULL };
if (r->push_idx == r->pop_idx) {
return dptr;
}
dma_descriptor_2d * desc = &r->desc[r->pop_idx];
// Wait for desc to complete
if (!desc->done) {
while (!desc->done) {
dmpoll();
}
}
htp_trace_event_stop(r->trace, HTP_TRACE_EVT_DMA, r->pop_idx);
dptr = r->dptr[r->pop_idx];
r->pop_idx = (r->pop_idx + 1) & r->idx_mask;
return dptr;
}
static inline dma_ptr dma_queue_pop_nowait(dma_queue * q) {
dma_ring * r = q->ring;
dma_ptr dptr = { NULL };
if (r->push_idx == r->pop_idx) {
return dptr;
}
dptr = r->dptr[r->pop_idx];
r->pop_idx = (r->pop_idx + 1) & r->idx_mask;
return dptr;
}
static inline bool dma_queue_empty(dma_queue * q) {
return q->ring->push_idx == q->ring->pop_idx;
}
static inline uint32_t dma_queue_depth(dma_queue * q) {
return (q->ring->push_idx - q->ring->pop_idx) & q->ring->idx_mask;
}
static inline uint32_t dma_queue_capacity(dma_queue * q) {
return q->ring->capacity;
}
#if __HVX_ARCH__ < 75
// Overflow-safe DMA push: all 2d descriptor fields (row_size, nrows, src_stride, dst_stride) are 16-bit, max 65535.
// This version transparently handles values that exceed the 16-bit limit and submits chained DMA transtions.
#define DMA_MAX_FIELD_VAL 65535u
static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
// Fast path: everything fits in 16 bits
if (nrows == 0 || __builtin_expect(
row_size <= DMA_MAX_FIELD_VAL &&
nrows <= DMA_MAX_FIELD_VAL &&
src_stride <= DMA_MAX_FIELD_VAL &&
dst_stride <= DMA_MAX_FIELD_VAL, 1)) {
return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows);
}
// Contiguous block
// Use 1d DMA mode which supports sizes up to 24-bits (16MB)
if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) {
size_t total = row_size * nrows;
return dma_queue_push_single_1d(q, dptr, total);
}
// Stride overflow - fall back to row-by-row.
{
const uint8_t *src = (const uint8_t *) dptr.src;
uint8_t *dst = (uint8_t *) dptr.dst;
for (size_t r = 0; r < nrows; ++r) {
dma_ptr p = dma_make_ptr(dst + r * dst_stride, src + r * src_stride);
if (!dma_queue_push_single_1d(q, p, row_size))
return false;
if (r + 1 < nrows)
dma_queue_pop(q);
}
return true;
}
}
#else // HVX_ARCH >= 75
static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
// On v75 and up we always use 2d 24-bit mode
return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows);
}
#endif
static inline bool dma_queue_push_ddr_to_vtcm(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) {
return dma_queue_push(q, dptr, dst_row_size, src_row_size, src_row_size, nrows);
}
static inline bool dma_queue_push_vtcm_to_ddr(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) {
return dma_queue_push(q, dptr, dst_row_size, src_row_size, dst_row_size, nrows);
}
#define DMA_CACHE_MAX_SIZE 256U
typedef struct {
uint8_t *base;
uint32_t line_size;
uint32_t capacity;
uint32_t src[DMA_CACHE_MAX_SIZE];
uint16_t age[DMA_CACHE_MAX_SIZE];
} dma_cache;
static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_size, uint32_t capacity)
{
c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity;
c->base = base;
c->line_size = line_size;
for (unsigned i=0; i < c->capacity; i++) {
c->src[i] = 0;
c->age[i] = 0;
}
}
static inline bool dma_cache_push(dma_queue *q, dma_cache *c, const uint8_t * src, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows)
{
uint32_t o_idx = 0;
uint16_t o_age = 0;
uint8_t * dst = 0;
for (unsigned i=0; i < c->capacity; i++) {
if (c->src[i] == (uint32_t) src) {
c->age[i] = 0;
dst = c->base + (i * c->line_size); nrows = 0; // dummy dma
} else {
c->age[i]++;
if (c->age[i] > o_age) { o_age = c->age[i]; o_idx = i; }
}
}
if (!dst) {
c->age[o_idx] = 0;
c->src[o_idx] = (uint32_t) src;
dst = c->base + o_idx * c->line_size; // normal nrows dma
return dma_queue_push(q, dma_make_ptr(dst, src), dst_stride, src_stride, row_size, nrows);
}
return dma_queue_push_single_1d(q, dma_make_ptr(dst, src), 0);
}
#ifdef __cplusplus
} // extern "C"
#endif
#endif /* HTP_DMA_H */

44
ggml/src/ggml-hexagon/htp/flash-attn-ops.c
View File

@@ -24,7 +24,7 @@
#include "hvx-reduce.h"
#include "hvx-flash-attn.h"
#include "htp-vtcm.h"
#include "worker-pool.h"
#include "work-queue.h"
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
@@ -204,7 +204,7 @@ static void flash_attn_ext_f16_thread(unsigned int nth, unsigned int ith, void *
if (ir0 >= ir1) return;
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
dma_queue * dma = octx->ctx->dma[ith];
@@ -486,7 +486,7 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data)
return;
}
struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL;
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_K_PREP, (uint16_t) (args->kv_start + start));
hmx_interleave_rows_to_tiles(factx->vtcm_k_tiles, (const __fp16 *) args->curr_k, total_rows, factx->DK,
args->src_stride, start, end);
@@ -494,7 +494,7 @@ static void fa_k_interleave_thread(unsigned int n, unsigned int i, void * data)
}
static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_rows, size_t src_stride, void * curr_k, uint32_t kv_start) {
worker_pool_context_t wp = factx->octx->ctx->worker_pool;
work_queue_t wp = factx->octx->ctx->work_queue;
uint32_t n = 1;
if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) {
n = factx->n_threads;
@@ -502,7 +502,7 @@ static void fa_phase_k_interleave(struct hmx_fa_context * factx, uint32_t kv_row
uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2);
fa_k_int_args_t args = { factx, kv_rows, src_stride, curr_k, kv_start, rows_per_t };
if (n > 1) {
worker_pool_run_func(wp, fa_k_interleave_thread, &args, n);
work_queue_run(wp, fa_k_interleave_thread, &args, n);
} else {
fa_k_interleave_thread(1, 0, &args);
}
@@ -534,7 +534,7 @@ static void fa_v_interleave_thread(unsigned int n, unsigned int i, void * data)
__fp16 * v_tiles_dst = (__fp16 *) args->v_tiles_dst;
struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL;
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_V_PREP, (uint16_t) (args->kv_start + start));
hmx_interleave_cols_to_tiles(v_tiles_dst, (const __fp16 *) args->v_src, total_rows, factx->DV,
args->src_stride, (uint32_t) args->n_col_tiles, start, end);
@@ -548,7 +548,7 @@ static void fa_phase_v_interleave(struct hmx_fa_context * factx,
void * v_tiles_dst,
size_t n_col_tiles,
uint32_t kv_start) {
worker_pool_context_t wp = factx->octx->ctx->worker_pool;
work_queue_t wp = factx->octx->ctx->work_queue;
uint32_t n = 1;
if (factx->n_threads > 1 && kv_rows >= factx->n_threads * 2) {
n = factx->n_threads;
@@ -556,7 +556,7 @@ static void fa_phase_v_interleave(struct hmx_fa_context * factx,
uint32_t rows_per_t = hex_align_up(hmx_ceil_div(kv_rows, n), 2);
fa_v_int_args_t args = { factx, kv_rows, src_stride, v_src, v_tiles_dst, n_col_tiles, kv_start, rows_per_t };
if (n > 1) {
worker_pool_run_func(wp, fa_v_interleave_thread, &args, n);
work_queue_run(wp, fa_v_interleave_thread, &args, n);
} else {
fa_v_interleave_thread(1, 0, &args);
}
@@ -589,7 +589,7 @@ static void fa_q_load_thread(unsigned int n, unsigned int i, void * data) {
const size_t start = (size_t) i * rows_per_t;
const size_t end = hex_smin(start + rows_per_t, factx->g_br);
struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL;
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_Q_PREP, (uint16_t) (args->q_start * G + start));
// Parallel initialization of per-block state
@@ -720,7 +720,7 @@ static void fa_phase_q_load(struct hmx_fa_context * factx,
uint32_t kv_head,
uint32_t ib3,
size_t n_rows_g) {
worker_pool_context_t wp = factx->octx->ctx->worker_pool;
work_queue_t wp = factx->octx->ctx->work_queue;
uint32_t n = 1;
if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) {
n = factx->n_threads;
@@ -739,7 +739,7 @@ static void fa_phase_q_load(struct hmx_fa_context * factx,
args.q_transposed = q->nb[1] < q->nb[2];
atomic_init(&args.barrier, n);
if (n > 1) {
worker_pool_run_func(wp, fa_q_load_thread, &args, n);
work_queue_run(wp, fa_q_load_thread, &args, n);
} else {
fa_q_load_thread(1, 0, &args);
}
@@ -772,7 +772,7 @@ static void fa_o_store_thread_f32(unsigned int n, unsigned int i, void * data) {
return;
}
struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL;
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));
const struct htp_tensor * dst = args->dst;
@@ -820,7 +820,7 @@ static void fa_o_store_thread_f16(unsigned int n, unsigned int i, void * data) {
return;
}
struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL;
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) (args->q_start * G + start));
const struct htp_tensor * dst = args->dst;
@@ -862,7 +862,7 @@ static void fa_phase_o_store(struct hmx_fa_context * factx,
uint32_t kv_head,
uint32_t ib3,
size_t n_rows_g) {
worker_pool_context_t wp = factx->octx->ctx->worker_pool;
work_queue_t wp = factx->octx->ctx->work_queue;
uint32_t n = 1;
if (factx->n_threads > 1 && n_rows_g >= (size_t) (factx->n_threads * 2)) {
n = factx->n_threads;
@@ -871,7 +871,7 @@ static void fa_phase_o_store(struct hmx_fa_context * factx,
fa_o_store_args_t args = { factx, dst, o_tile_src, q_start, kv_head, ib3, n_rows_g, rows_per_t };
worker_callback_t store_fn = factx->is_dst_fp32 ? fa_o_store_thread_f32 : fa_o_store_thread_f16;
if (n > 1) {
worker_pool_run_func(wp, store_fn, &args, n);
work_queue_run(wp, store_fn, &args, n);
} else {
store_fn(1, 0, &args);
}
@@ -930,7 +930,7 @@ static inline void fa_softmax_impl(
return;
}
struct htp_thread_trace * tr = factx->octx->ctx ? &factx->octx->ctx->trace[i] : NULL;
struct htp_thread_trace * tr = &factx->octx->ctx->trace[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_FA_SFM, (uint16_t) (args->q_start * G + vec_start * 64));
// Per-thread row scratch: thread i uses bufs at offset i * 2 * stride
@@ -1290,7 +1290,7 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx,
fa_softmax_args_t * sargs,
size_t n_row_tiles,
size_t n_row_tiles_g_br) {
worker_pool_context_t wp = factx->octx->ctx->worker_pool;
work_queue_t wp = factx->octx->ctx->work_queue;
const size_t n_row_vec_cnt = hmx_ceil_div(sargs->n_rows_g, 64);
worker_callback_t softmax_fn = fa_softmax_thread;
@@ -1307,7 +1307,7 @@ static void fa_phase_softmax_and_build_d(struct hmx_fa_context * factx,
if (factx->n_threads > 1 && n_row_vec_cnt >= 2) {
uint32_t n_use = (uint32_t) hex_smin((size_t) factx->n_threads, n_row_vec_cnt);
sargs->thread_div = init_fastdiv_values(n_use);
worker_pool_run_func(wp, softmax_fn, sargs, n_use);
work_queue_run(wp, softmax_fn, sargs, n_use);
} else {
softmax_fn(1, 0, sargs);
}
@@ -1519,8 +1519,8 @@ static void fa_pop_mask_dma_gqa(dma_queue * dma, uint32_t G) {
// ============================================================================
int hmx_flash_attn_ext(struct htp_ops_context * octx) {
struct htp_thread_trace * tr_hvx = octx->ctx ? &octx->ctx->trace[0] : NULL;
struct htp_thread_trace * tr_hmx = octx->ctx ? &octx->ctx->trace[HTP_MAX_NTHREADS] : NULL;
struct htp_thread_trace * tr_hvx = &octx->ctx->trace[0];
struct htp_thread_trace * tr_hmx = &octx->ctx->trace[HTP_MAX_NTHREADS];
const struct htp_tensor * q = octx->src[0];
const struct htp_tensor * k = octx->src[1];
const struct htp_tensor * v = octx->src[2];
@@ -1735,7 +1735,7 @@ int hmx_flash_attn_ext(struct htp_ops_context * octx) {
const size_t k_src_stride = size_k_row_padded / sizeof(__fp16);
const size_t v_src_stride = size_v_row_padded / sizeof(__fp16);
struct hmx_queue * hmx_q = ctx->hmx_queue;
hmx_queue_t hmx_q = ctx->hmx_queue;
if (factx.pipeline) {
// Pipeline path
@@ -2084,7 +2084,7 @@ int op_flash_attn_ext(struct htp_ops_context * octx) {
}
if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {
worker_pool_run_func(octx->ctx->worker_pool, flash_attn_ext_f16_thread, &factx, octx->n_threads);
work_queue_run(octx->ctx->work_queue, flash_attn_ext_f16_thread, &factx, octx->n_threads);
}
return HTP_STATUS_OK;

24
ggml/src/ggml-hexagon/htp/hex-bitmap.h Normal file
View File

@@ -0,0 +1,24 @@
#ifndef HEX_BITMAP_H
#define HEX_BITMAP_H
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
static inline void bitmap_set(uint32_t * bitmap, uint32_t idx) {
bitmap[idx / 32] |= (1U << (idx % 32));
}
static inline void bitmap_clear(uint32_t * bitmap, uint32_t idx) {
bitmap[idx / 32] &= ~(1U << (idx % 32));
}
static inline bool bitmap_test(const uint32_t * bitmap, uint32_t idx) {
return (bitmap[idx / 32] & (1U << (idx % 32))) != 0;
}
static inline void bitmap_reset(uint32_t * bitmap, size_t size_in_bits) {
memset(bitmap, 0, ((size_in_bits + 31) / 32) * sizeof(uint32_t));
}
#endif // HEX_BITMAP_H

63
ggml/src/ggml-hexagon/htp/hex-dma.c
View File

@@ -1,63 +0,0 @@
#include "hex-dma.h"
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#pragma clang diagnostic ignored "-Wunused-function"
static inline uint32_t pow2_ceil(uint32_t x) {
if (x <= 1) {
return 1;
}
int p = 2;
x--;
while (x >>= 1) {
p <<= 1;
}
return p;
}
dma_queue * dma_queue_create(size_t capacity) {
dma_queue * q = (dma_queue *) memalign(32, sizeof(dma_queue));
if (q == NULL) {
FARF(ERROR, "%s: failed to allocate DMA queue\n", __FUNCTION__);
return NULL;
}
capacity = pow2_ceil(capacity);
memset(q, 0, sizeof(dma_queue));
q->capacity = capacity;
q->idx_mask = capacity - 1;
q->desc = (dma_descriptor_2d *) memalign(64, capacity * sizeof(dma_descriptor_2d));
memset(q->desc, 0, capacity * sizeof(dma_descriptor_2d));
q->dptr = (dma_ptr *) memalign(4, capacity * sizeof(dma_ptr));
memset(q->dptr, 0, capacity * sizeof(dma_ptr));
q->tail = &q->desc[capacity - 1];
if (!q->desc && !q->dptr) {
FARF(ERROR, "%s: failed to allocate DMA queue items\n", __FUNCTION__);
return NULL;
}
FARF(HIGH, "dma-queue: capacity %u\n", capacity);
return q;
}
void dma_queue_delete(dma_queue * q) {
if (!q) {
return;
}
free(q->desc);
free(q->dptr);
free(q);
}
void dma_queue_flush(dma_queue * q) {
while (dma_queue_pop(q).dst != NULL) ;
}

377
ggml/src/ggml-hexagon/htp/hex-dma.h
View File

@@ -1,375 +1,2 @@
#ifndef HTP_DMA_H
#define HTP_DMA_H
#include <HAP_farf.h>
#include <hexagon_types.h>
#include <stdbool.h>
#include <stdint.h>
#include "hex-utils.h"
#include "hex-profile.h"
#ifdef __cplusplus
extern "C" {
#endif
// Define the HW descriptor structs here since the ones in HexSDK are a bit out of date
typedef struct dma_descriptor_1d_s {
void * next;
uint32_t size:24;
uint32_t desc_size:2;
uint32_t dst_comp:1;
uint32_t src_comp:1;
uint32_t dst_bypass:1;
uint32_t src_bypass:1;
uint32_t order:1;
uint32_t done:1;
void * src;
void * dst;
} dma_descriptor_1d;
#if __HVX_ARCH__ < 75
typedef struct dma_descriptor_2d_s {
void * next;
uint32_t reserved0:24;
uint32_t desc_size:2;
uint32_t dst_comp:1;
uint32_t src_comp:1;
uint32_t dst_bypass:1;
uint32_t src_bypass:1;
uint32_t order:1;
uint32_t done:1;
void * src;
void * dst;
uint32_t desc_type:8;
uint32_t reserved1:24;
uint32_t row_size:16;
uint32_t nrows:16;
uint32_t src_stride:16;
uint32_t dst_stride:16;
uint32_t src_offset:16;
uint32_t dst_offset:16;
} dma_descriptor_2d;
#else
typedef struct dma_descriptor_2d_s {
void * next;
uint32_t dst_stride:24;
uint32_t desc_size:2;
uint32_t dst_comp:1;
uint32_t src_comp:1;
uint32_t dst_bypass:1;
uint32_t src_bypass:1;
uint32_t order:1;
uint32_t done:1;
void * src;
void * dst;
uint32_t desc_type:8;
uint32_t reserved0:24;
uint32_t row_size:24;
uint32_t nrows_lo:8;
uint32_t nrows_hi:8;
uint32_t src_stride:24;
uint32_t offset:24;
uint32_t reserved1:8;
} dma_descriptor_2d;
#endif
typedef struct {
void *dst;
const void *src;
} dma_ptr;
typedef struct {
dma_descriptor_2d * desc; // descriptor pointers
dma_descriptor_2d * tail; // tail pointer
dma_ptr * dptr; // dst/src pointers
uint32_t push_idx;
uint32_t pop_idx;
uint32_t capacity;
uint32_t idx_mask;
struct htp_thread_trace * trace;
} dma_queue;
dma_queue * dma_queue_create(size_t capacity);
void dma_queue_delete(dma_queue * q);
void dma_queue_flush(dma_queue * q);
// TODO: technically we don't need these and could use Q6_dmstart/wait/etc instead
// but those do not seem to always compiler properly.
static inline void dmstart(void * next) {
asm volatile(" release(%0):at" : : "r"(next));
asm volatile(" dmstart(%0)" : : "r"(next));
}
static inline void dmlink(void * cur, void * next) {
asm volatile(" release(%0):at" : : "r"(next));
asm volatile(" dmlink(%0, %1)" : : "r"(cur), "r"(next));
}
static inline unsigned int dmpoll(void) {
unsigned int ret = 0;
asm volatile(" %0 = dmpoll" : "=r"(ret) : : "memory");
return ret;
}
static inline unsigned int dmwait(void) {
unsigned int ret = 0;
asm volatile(" %0 = dmwait" : "=r"(ret) : : "memory");
return ret;
}
static inline dma_ptr dma_make_ptr(void *dst, const void *src)
{
dma_ptr p = { dst, src };
return p;
}
static const uint32_t dma_src_l2_bypass_on = 1;
static const uint32_t dma_dst_l2_bypass_on = 1;
static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t size) {
if (((q->push_idx + 1) & q->idx_mask) == q->pop_idx) {
FARF(HIGH, "dma-push: queue full\n");
return false;
}
dma_descriptor_1d * desc = (dma_descriptor_1d *) &q->desc[q->push_idx];
desc->src = (void *) dptr.src;
desc->dst = (void *) dptr.dst;
desc->size = size;
q->dptr[q->push_idx] = dptr;
if (size) {
desc->next = NULL;
desc->desc_size = 0; // 1D mode
desc->src_bypass = dma_src_l2_bypass_on;
desc->dst_bypass = dma_dst_l2_bypass_on;
desc->order = 0;
desc->done = 0;
htp_trace_event_start(q->trace, HTP_TRACE_EVT_DMA, q->push_idx);
dmlink(q->tail, desc);
q->tail = (dma_descriptor_2d *) desc;
} else {
desc->desc_size = 0;
desc->done = 1;
}
q->push_idx = (q->push_idx + 1) & q->idx_mask;
return true;
}
static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
if (((q->push_idx + 1) & q->idx_mask) == q->pop_idx) {
FARF(HIGH, "dma-push: queue full\n");
return false;
}
dma_descriptor_2d * desc = &q->desc[q->push_idx];
desc->next = NULL;
desc->reserved0 = 0;
desc->reserved1 = 0;
desc->desc_size = 1; // 2d mode
desc->src_bypass = dma_src_l2_bypass_on;
desc->dst_bypass = dma_dst_l2_bypass_on;
desc->src_comp = 0;
desc->dst_comp = 0;
desc->order = 0;
desc->done = 0;
desc->src_stride = src_stride;
desc->dst_stride = dst_stride;
desc->src = (void *) dptr.src;
desc->dst = (void *) dptr.dst;
desc->row_size = row_size;
#if __HVX_ARCH__ < 75
desc->desc_type = 0; // 2d (16-bit) mode
desc->nrows = nrows;
desc->src_offset = 0;
desc->dst_offset = 0;
#else
desc->desc_type = 9; // 2d (24-bit) mode
desc->nrows_lo = (nrows & 0xff);
desc->nrows_hi = (nrows >> 8);
desc->offset = 0;
#endif
q->dptr[q->push_idx] = dptr;
if (nrows) {
htp_trace_event_start(q->trace, HTP_TRACE_EVT_DMA, q->push_idx);
dmlink(q->tail, desc);
q->tail = desc;
} else {
desc->done = 1;
}
// FARF(ERROR, "dma-push: i %u row-size %u nrows %d dst %p src %p\n", q->push_idx, row_size, nrows, dptr.dst, dptr.src);
q->push_idx = (q->push_idx + 1) & q->idx_mask;
return true;
}
static inline dma_ptr dma_queue_pop(dma_queue * q) {
dma_ptr dptr = { NULL };
if (q->push_idx == q->pop_idx) {
return dptr;
}
dma_descriptor_2d * desc = &q->desc[q->pop_idx];
// Wait for desc to complete
if (!desc->done) {
while (!desc->done) {
dmpoll();
}
}
htp_trace_event_stop(q->trace, HTP_TRACE_EVT_DMA, q->pop_idx);
dptr = q->dptr[q->pop_idx];
// FARF(ERROR, "dma-pop: i %u dst %p src %p\n", q->pop_idx, dptr.dst, dptr.src);
q->pop_idx = (q->pop_idx + 1) & q->idx_mask;
return dptr;
}
static inline dma_ptr dma_queue_pop_nowait(dma_queue * q) {
dma_ptr dptr = { NULL };
if (q->push_idx == q->pop_idx) {
return dptr;
}
dptr = q->dptr[q->pop_idx];
// FARF(ERROR, "dma-pop-nowait: i %u dst %p src %p\n", q->pop_idx, dptr.dst, dptr.src);
q->pop_idx = (q->pop_idx + 1) & q->idx_mask;
return dptr;
}
static inline bool dma_queue_empty(dma_queue * q) {
return q->push_idx == q->pop_idx;
}
static inline uint32_t dma_queue_depth(dma_queue * q) {
return (q->push_idx - q->pop_idx) & q->idx_mask;
}
static inline uint32_t dma_queue_capacity(dma_queue * q) {
return q->capacity;
}
#if __HVX_ARCH__ < 75
// Overflow-safe DMA push: all 2d descriptor fields (row_size, nrows, src_stride, dst_stride) are 16-bit, max 65535.
// This version transparently handles values that exceed the 16-bit limit and submits chained DMA transtions.
#define DMA_MAX_FIELD_VAL 65535u
static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
// Fast path: everything fits in 16 bits
if (nrows == 0 || __builtin_expect(
row_size <= DMA_MAX_FIELD_VAL &&
nrows <= DMA_MAX_FIELD_VAL &&
src_stride <= DMA_MAX_FIELD_VAL &&
dst_stride <= DMA_MAX_FIELD_VAL, 1)) {
return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows);
}
// Contiguous block
// Use 1d DMA mode which supports sizes up to 24-bits (16MB)
if (nrows == 1 || (row_size == src_stride && row_size == dst_stride)) {
size_t total = row_size * nrows;
return dma_queue_push_single_1d(q, dptr, total);
}
// Stride overflow — fall back to row-by-row.
{
const uint8_t *src = (const uint8_t *) dptr.src;
uint8_t *dst = (uint8_t *) dptr.dst;
for (size_t r = 0; r < nrows; ++r) {
dma_ptr p = dma_make_ptr(dst + r * dst_stride, src + r * src_stride);
if (!dma_queue_push_single_1d(q, p, row_size))
return false;
if (r + 1 < nrows)
dma_queue_pop(q);
}
return true;
}
}
#else // HVX_ARCH >= 75
static inline bool dma_queue_push(dma_queue *q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) {
// On v75 and up we always use 2d 24-bit mode
return dma_queue_push_single_2d(q, dptr, dst_stride, src_stride, row_size, nrows);
}
#endif
static inline bool dma_queue_push_ddr_to_vtcm(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) {
return dma_queue_push(q, dptr, dst_row_size, src_row_size, src_row_size, nrows);
}
static inline bool dma_queue_push_vtcm_to_ddr(dma_queue * q, dma_ptr dptr, size_t dst_row_size, size_t src_row_size, size_t nrows) {
return dma_queue_push(q, dptr, dst_row_size, src_row_size, dst_row_size, nrows);
}
#define DMA_CACHE_MAX_SIZE 256U
typedef struct {
uint8_t *base;
uint32_t line_size;
uint32_t capacity;
uint32_t src[DMA_CACHE_MAX_SIZE];
uint16_t age[DMA_CACHE_MAX_SIZE];
} dma_cache;
static inline void dma_cache_init(dma_cache *c, uint8_t *base, uint32_t line_size, uint32_t capacity)
{
c->capacity = (capacity > DMA_CACHE_MAX_SIZE) ? DMA_CACHE_MAX_SIZE : capacity;
c->base = base;
c->line_size = line_size;
for (unsigned i=0; i < c->capacity; i++) {
c->src[i] = 0;
c->age[i] = 0;
}
}
static inline bool dma_cache_push(dma_queue *q, dma_cache *c, const uint8_t * src, uint32_t dst_stride, uint32_t src_stride, uint32_t row_size, uint32_t nrows)
{
uint32_t o_idx = 0;
uint16_t o_age = 0;
uint8_t * dst = 0;
for (unsigned i=0; i < c->capacity; i++) {
if (c->src[i] == (uint32_t) src) {
c->age[i] = 0;
dst = c->base + (i * c->line_size); nrows = 0; // dummy dma
} else {
c->age[i]++;
if (c->age[i] > o_age) { o_age = c->age[i]; o_idx = i; }
}
}
if (!dst) {
c->age[o_idx] = 0;
c->src[o_idx] = (uint32_t) src;
dst = c->base + o_idx * c->line_size; // normal nrows dma
return dma_queue_push(q, dma_make_ptr(dst, src), dst_stride, src_stride, row_size, nrows);
}
return dma_queue_push_single_1d(q, dma_make_ptr(dst, src), 0);
}
#ifdef __cplusplus
} // extern "C"
#endif
#endif /* HTP_DMA_H */
#pragma once
#include "dma-queue.h"

10
ggml/src/ggml-hexagon/htp/hex-profile.h
View File

@@ -44,11 +44,11 @@ struct htp_thread_trace {
};
static inline void htp_trace_event(struct htp_thread_trace * tr, uint16_t id, uint16_t info, uint32_t type) {
if (tr && tr->events && tr->count < tr->max_events) {
uint32_t idx = tr->count;
tr->events[idx].id = id;
tr->events[idx].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0);
tr->events[idx].cycles = (uint32_t) hex_get_cycles();
if (tr->count < tr->max_events) {
uint32_t i = tr->count;
tr->events[i].id = id;
tr->events[i].info = info | (type == HTP_TRACE_EVT_STOP ? 0x8000 : 0);
tr->events[i].cycles = (uint32_t) hex_get_cycles();
tr->count++;
}
}

31
ggml/src/ggml-hexagon/htp/hex-utils.h
View File

@@ -30,21 +30,26 @@ static inline void hex_l2fetch(const void * p, uint32_t width, uint32_t stride,
Q6_l2fetch_AP((void *) p, control);
}
#define HEX_L2_LINE_SIZE 64
#define HEX_L2_FLUSH_SIZE (128 * 1024)
static inline void hex_l2fetch_block(const void * addr, size_t size) {
if (size == 0) return;
const uint32_t width = 16384; // 16KB rows
const uint32_t height = (size + width - 1) / width;
hex_l2fetch(addr, width, width, height);
}
#define HEX_L2_LINE_SIZE 128
#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (lines per loop iteration)
#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024)
#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024)
static inline void hex_l2flush(void * addr, size_t size) {
if (size > HEX_L2_FLUSH_SIZE) {
qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);
} else {
const uint32_t s = (uint32_t) addr;
const uint32_t e = s + size;
for (uint32_t i = s; i < e; i += HEX_L2_LINE_SIZE * 4) {
Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 0);
Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 1);
Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 2);
Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 3);
}
const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1);
const uint32_t e = (((uint32_t) addr) + size + HEX_L2_LINE_SIZE - 1) & ~(HEX_L2_LINE_SIZE - 1);
for (uint32_t i = s; i < e; i += HEX_L2_BLOCK_SIZE) {
Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 0);
Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 1);
Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 2);
Q6_dccleaninva_A((void *) i + HEX_L2_LINE_SIZE * 3);
}
}

81
ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h
View File

@@ -1005,10 +1005,62 @@ static void transfer_activation_row_pair_fp32_to_fp16(
}
}
static void transfer_activation_row_pair_fp32_to_fp16_col_chunk(
__fp16 *restrict vtcm_dst,
const float *restrict row0, // offset by c_first
const float *restrict row1, // offset by c_first
uint32_t r,
uint32_t k_block,
uint32_t c_first,
uint32_t c_len,
uint32_t k_chunk_valid,
bool row0_valid,
bool row1_valid) {
uint32_t r0 = r / HTP_MM_HMX_TILE_N_ROWS; // tile row index
uint32_t r1 = r % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx
uint32_t c = 0;
for (; c + 32 <= k_chunk_valid; c += 32) {
HVX_Vector v0 = Q6_V_vzero();
HVX_Vector v1 = Q6_V_vzero();
if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c);
if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c);
HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
uint32_t c0 = (c_first + c) / HTP_MM_HMX_TILE_N_COLS; // tile column index
uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
tile[r1 / 2] = v_out;
}
if (c < c_len) {
HVX_Vector v0 = Q6_V_vzero();
HVX_Vector v1 = Q6_V_vzero();
if (row0_valid) v0 = *(const HVX_Vector *)(row0 + c);
if (row1_valid) v1 = *(const HVX_Vector *)(row1 + c);
uint32_t rem = (k_chunk_valid > c) ? (k_chunk_valid - c) : 0;
HVX_VectorPred mask = Q6_Q_vsetq2_R(rem > 0 ? rem * sizeof(float) : 0);
v0 = Q6_V_vmux_QVV(mask, v0, Q6_V_vzero());
v1 = Q6_V_vmux_QVV(mask, v1, Q6_V_vzero());
HVX_Vector v_out = hvx_vec_f32_to_f16_shuff(v0, v1);
uint32_t c0 = (c_first + c) / HTP_MM_HMX_TILE_N_COLS; // tile column index
uint32_t tile_idx = r0 * (k_block / HTP_MM_HMX_TILE_N_COLS) + c0;
HVX_Vector *tile = (HVX_Vector *) (vtcm_dst + tile_idx * HTP_MM_HMX_TILE_N_ELMS);
tile[r1 / 2] = v_out;
}
}
static void transfer_activation_chunk_fp32_to_fp16_gathered(
__fp16 *restrict vtcm_dst,
const float *restrict src,
uint32_t start_row,
uint32_t vtcm_start_row,
uint32_t n_rows,
uint32_t k_block,
const struct mmid_row_mapping *matrix_rows,
@@ -1029,8 +1081,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered(
for (r = 0; r < n_rows_tiled; r += 2) {
uint32_t r_idx0 = start_row + r + 0;
uint32_t r_idx1 = start_row + r + 1;
uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index
uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx
uint32_t lr = vtcm_start_row + r; // vtcm-local row
uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index
uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx
struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0];
struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1];
@@ -1073,9 +1126,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered(
}
for (; r < n_rows_padded; r += 2) {
uint32_t r_idx0 = start_row + r;
uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index
uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx
uint32_t lr = vtcm_start_row + r; // vtcm-local row
uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index
uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx
const bool row0_valid = (start_row + r + 0) < cne1;
const bool row1_valid = (start_row + r + 1) < cne1;
@@ -1135,6 +1188,7 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered_flat(
__fp16 *restrict vtcm_dst,
const float *restrict src,
uint32_t start_row,
uint32_t vtcm_start_row,
uint32_t n_rows,
uint32_t k_block,
const struct mmid_row_mapping *matrix_rows,
@@ -1152,8 +1206,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered_flat(
for (r = 0; r < n_rows_tiled; r += 2) {
uint32_t r_idx0 = start_row + r + 0;
uint32_t r_idx1 = start_row + r + 1;
uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index
uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx
uint32_t lr = vtcm_start_row + r; // vtcm-local row
uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index
uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx
struct mmid_row_mapping mapping0 = matrix_rows[cur_a * mapping_stride + r_idx0];
struct mmid_row_mapping mapping1 = matrix_rows[cur_a * mapping_stride + r_idx1];
@@ -1193,9 +1248,9 @@ static void transfer_activation_chunk_fp32_to_fp16_gathered_flat(
}
for (; r < n_rows_padded; r += 2) {
uint32_t r_idx0 = start_row + r;
uint32_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS; // tile row index
uint32_t r1 = r_idx0 % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx
uint32_t lr = vtcm_start_row + r; // vtcm-local row
uint32_t r0 = lr / HTP_MM_HMX_TILE_N_ROWS; // tile row index
uint32_t r1 = lr % HTP_MM_HMX_TILE_N_ROWS; // intra-tile row idx
const bool row0_valid = (start_row + r + 0) < cne1;
const bool row1_valid = (start_row + r + 1) < cne1;
@@ -1253,6 +1308,7 @@ static void transfer_output_chunk_fp16_to_fp32_scattered(
float *restrict dst,
const __fp16 *restrict vtcm_src,
uint32_t start_row,
uint32_t vtcm_start_row,
uint32_t n_rows,
uint32_t n_cols,
const struct mmid_row_mapping *matrix_rows,
@@ -1269,8 +1325,9 @@ static void transfer_output_chunk_fp16_to_fp32_scattered(
for (size_t r = 0; r < n_rows; r += 2) {
uint32_t r_idx0 = start_row + r + 0;
uint32_t r_idx1 = start_row + r + 1;
const size_t r0 = r_idx0 / HTP_MM_HMX_TILE_N_ROWS;
const size_t r1 = (r_idx0 % HTP_MM_HMX_TILE_N_ROWS) / 2; // index of the row pair within the tile
uint32_t lr = vtcm_start_row + r; // vtcm-local row
const size_t r0 = (lr / HTP_MM_HMX_TILE_N_ROWS);
const size_t r1 = (lr % HTP_MM_HMX_TILE_N_ROWS) / 2; // index of the row pair within the tile
const __fp16 *row_base = vtcm_src + r0 * tile_row_stride;
if (r_idx0 >= cne1) break;

57
ggml/src/ggml-hexagon/htp/hmx-queue.c
View File

@@ -14,7 +14,7 @@
#define QURT_LOWEST_PRIO (254)
static inline void hmx_lock(struct hmx_queue *q)
static inline void hmx_lock(hmx_queue_t q)
{
if (!q->hmx_locked) {
HAP_compute_res_hmx_lock(q->hap_rctx);
@@ -22,7 +22,7 @@ static inline void hmx_lock(struct hmx_queue *q)
}
}
static inline void hmx_unlock(struct hmx_queue *q)
static inline void hmx_unlock(hmx_queue_t q)
{
if (q->hmx_locked) {
HAP_compute_res_hmx_unlock(q->hap_rctx);
@@ -30,7 +30,7 @@ static inline void hmx_unlock(struct hmx_queue *q)
}
}
static inline void hmx_queue_process(struct hmx_queue *q, bool* killed) {
static inline void hmx_queue_process(hmx_queue_t q, bool* killed) {
unsigned int ir = atomic_load(&q->idx_read);
while (ir != atomic_load(&q->idx_write)) {
@@ -61,7 +61,7 @@ static inline void hmx_queue_process(struct hmx_queue *q, bool* killed) {
}
static void hmx_queue_thread(void * arg) {
struct hmx_queue * q = (struct hmx_queue *) arg;
hmx_queue_t q = (hmx_queue_t) arg;
FARF(HIGH, "hmx-queue-thread: started");
@@ -93,34 +93,41 @@ static void hmx_queue_thread(void * arg) {
FARF(HIGH, "hmx-queue-thread: stopped");
}
struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx) {
size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size) {
capacity = hex_ceil_pow2(capacity);
size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE);
size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE);
size_t size_stack = stack_size;
return size_q + size_desc + size_stack;
}
size_t hmx_queue_alignof(void) {
return HEX_L2_LINE_SIZE;
}
hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace) {
capacity = hex_ceil_pow2(capacity);
size_t size_q = hex_align_up(sizeof(struct hmx_queue_s), HEX_L2_LINE_SIZE);
size_t size_desc = hex_align_up(capacity * sizeof(struct hmx_queue_desc), HEX_L2_LINE_SIZE);
uint8_t * block = (uint8_t *) ptr;
hmx_queue_t q = (hmx_queue_t) block; block += size_q;
memset(q, 0, sizeof(struct hmx_queue_s));
struct hmx_queue * q = (struct hmx_queue *) memalign(32, sizeof(struct hmx_queue));
if (q == NULL) {
FARF(ERROR, "%s: failed to allocate DMA queue\n", __FUNCTION__);
return NULL;
}
memset(q, 0, sizeof(struct hmx_queue));
q->capacity = capacity;
q->idx_mask = capacity - 1;
q->hap_rctx = hap_rctx;
q->external_mem = true;
q->desc = (struct hmx_queue_desc *) memalign(64, capacity * sizeof(struct hmx_queue_desc));
if (!q->desc) {
FARF(ERROR, "hmx-queue: failed to allocate HMX queue descriptors\n");
return NULL;
}
q->desc = (struct hmx_queue_desc *) block; block += size_desc;
memset(q->desc, 0, capacity * sizeof(struct hmx_queue_desc));
const size_t stack_size = HMX_QUEUE_THREAD_STACK_SIZE;
q->stack = (unsigned char *) memalign(64, stack_size);
if (!q->stack) {
FARF(ERROR, "hmx-queue: thread stack allocation failed (%zu bytes)", stack_size);
return NULL;
}
q->stack = block;
memset(q->stack, 0, stack_size);
q->trace = trace;
// Match caller thread priority (same pattern as worker-pool.c).
int prio = qurt_thread_get_priority(qurt_thread_get_id());
if (prio < 1) {
@@ -148,7 +155,7 @@ struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx) {
return q;
}
void hmx_queue_delete(struct hmx_queue * q) {
void hmx_queue_free(hmx_queue_t q) {
if (!q) {
return;
}
@@ -160,8 +167,4 @@ void hmx_queue_delete(struct hmx_queue * q) {
int status;
qurt_thread_join(q->thread, &status);
free(q->desc);
free(q->stack);
free(q);
}

41
ggml/src/ggml-hexagon/htp/hmx-queue.h
View File

@@ -17,8 +17,6 @@
extern "C" {
#endif
#define HMX_QUEUE_THREAD_STACK_SIZE (16 * 1024)
#if __HVX_ARCH__ > 79
#define HMX_QUEUE_POLL_COUNT 2000
#else
@@ -41,7 +39,7 @@ struct hmx_queue_desc {
atomic_uint done;
};
struct hmx_queue {
struct hmx_queue_s {
struct hmx_queue_desc * desc;
atomic_uint idx_write; // updated by producer (push)
atomic_uint idx_read; // updated by consumer (process)
@@ -55,19 +53,24 @@ struct hmx_queue {
uint32_t hap_rctx;
bool hmx_locked;
struct htp_thread_trace * trace;
bool external_mem; // memory owned externally
};
struct hmx_queue * hmx_queue_create(size_t capacity, uint32_t hap_rctx);
void hmx_queue_delete(struct hmx_queue * q);
typedef struct hmx_queue_s * hmx_queue_t;
size_t hmx_queue_sizeof(size_t capacity, uint32_t stack_size);
size_t hmx_queue_alignof(void);
hmx_queue_t hmx_queue_init(void * ptr, size_t capacity, uint32_t stack_size, uint32_t hap_rctx, struct htp_thread_trace * trace);
void hmx_queue_free(hmx_queue_t q);
static inline struct hmx_queue_desc hmx_queue_make_desc(hmx_queue_func func, void * data) {
struct hmx_queue_desc d = { func, data };
return d;
}
static inline bool hmx_queue_push(struct hmx_queue * q, struct hmx_queue_desc d) {
static inline bool hmx_queue_push(hmx_queue_t q, struct hmx_queue_desc d) {
unsigned int ir = atomic_load(&q->idx_read);
unsigned int iw = q->idx_write;
unsigned int iw = atomic_load(&q->idx_write);
if (((iw + 1) & q->idx_mask) == ir) {
FARF(HIGH, "hmx-queue-push: queue is full\n");
@@ -87,25 +90,25 @@ static inline bool hmx_queue_push(struct hmx_queue * q, struct hmx_queue_desc d)
return true;
}
static inline bool hmx_queue_signal(struct hmx_queue *q, enum hmx_queue_signal sig) {
static inline bool hmx_queue_signal(hmx_queue_t q, enum hmx_queue_signal sig) {
return hmx_queue_push(q, hmx_queue_make_desc((hmx_queue_func) sig, NULL));
}
static inline bool hmx_queue_empty(struct hmx_queue * q) {
return q->idx_pop == q->idx_write;
static inline bool hmx_queue_empty(hmx_queue_t q) {
return q->idx_pop == atomic_load(&q->idx_write);
}
static inline uint32_t hmx_queue_depth(struct hmx_queue * q) {
return (q->idx_read - q->idx_read) & q->idx_mask;
static inline uint32_t hmx_queue_depth(hmx_queue_t q) {
return (atomic_load(&q->idx_write) - atomic_load(&q->idx_read)) & q->idx_mask;
}
static inline uint32_t hmx_queue_capacity(struct hmx_queue * q) {
static inline uint32_t hmx_queue_capacity(hmx_queue_t q) {
return q->capacity;
}
static inline struct hmx_queue_desc hmx_queue_pop_one(struct hmx_queue * q) {
static inline struct hmx_queue_desc hmx_queue_pop_one(hmx_queue_t q) {
unsigned int ip = q->idx_pop;
unsigned int iw = q->idx_write;
unsigned int iw = atomic_load(&q->idx_write);
struct hmx_queue_desc rd = { NULL, NULL };
if (ip == iw) {
@@ -126,7 +129,7 @@ static inline struct hmx_queue_desc hmx_queue_pop_one(struct hmx_queue * q) {
return rd;
}
static inline struct hmx_queue_desc hmx_queue_pop(struct hmx_queue * q) {
static inline struct hmx_queue_desc hmx_queue_pop(hmx_queue_t q) {
while (1) {
struct hmx_queue_desc d = hmx_queue_pop_one(q);
@@ -138,15 +141,15 @@ static inline struct hmx_queue_desc hmx_queue_pop(struct hmx_queue * q) {
}
}
static inline void hmx_queue_flush(struct hmx_queue * q) {
static inline void hmx_queue_flush(hmx_queue_t q) {
while (hmx_queue_pop_one(q).func != NULL) ;
}
static inline void hmx_queue_wakeup(struct hmx_queue * q) {
static inline void hmx_queue_wakeup(hmx_queue_t q) {
hmx_queue_signal(q, HMX_QUEUE_WAKEUP);
}
static inline void hmx_queue_suspend(struct hmx_queue *q) {
static inline void hmx_queue_suspend(hmx_queue_t q) {
hmx_queue_signal(q, HMX_QUEUE_SUSPEND);
}

23
ggml/src/ggml-hexagon/htp/htp-ctx.h
View File

@@ -5,7 +5,8 @@
#include "hmx-queue.h"
#include "htp-ops.h"
#include "hex-profile.h"
#include "worker-pool.h"
#include "work-queue.h"
#include "hex-fastdiv.h"
#include <assert.h>
#include <dspqueue.h>
@@ -52,6 +53,9 @@ struct htp_ops_context {
const struct htp_tensor * dsts[HTP_OP_MAX_OUTPUTS];
};
dma_queue ** src_dma[HTP_OP_MAX_INPUTS];
dma_queue ** dst_dma[HTP_OP_MAX_OUTPUTS];
// TODO convert these to an array
struct htp_spad src0_spad;
struct htp_spad src1_spad;
@@ -65,11 +69,16 @@ struct htp_ops_context {
// Main context for htp DSP backend
struct htp_context {
dspqueue_t queue;
dma_queue * dma[HTP_MAX_NTHREADS];
dspqueue_t dsp_queue;
struct htp_mmap mmap[HTP_MAX_MMAPS];
worker_pool_context_t worker_pool;
dma_queue_t dma[HTP_MAX_NTHREADS];
dma_queue_t dma_cached[HTP_MAX_NTHREADS];
work_queue_t work_queue;
hmx_queue_t hmx_queue;
uint32_t n_threads;
struct fastdiv_values n_threads_div;
int thread_id;
int thread_prio;
@@ -86,6 +95,7 @@ struct htp_context {
atomic_bool vtcm_needs_release;
uint64_t max_vmem;
uint32_t dirty_map[HTP_OP_MAX_TENSORS / 32];
// Persistent DDR scratchpad for MUL_MAT_ID mappings
void * ddr_spad_base;
@@ -93,7 +103,10 @@ struct htp_context {
struct htp_ops_context octx;
struct hmx_queue * hmx_queue; // Async HMX queue for pipeline overlap
qurt_thread_t main_thread;
void * main_stack;
atomic_bool killed;
size_t footprint;
};
int op_matmul(struct htp_ops_context * octx);

11
ggml/src/ggml-hexagon/htp/htp-ops.h
View File

@@ -108,8 +108,7 @@ enum htp_op_code {
#define HTP_OP_MAX_KERN_PARAMS 32
#define HTP_OP_MAX_BUFS 16
#define HTP_OP_MAX_REQS 256
#define HTP_OP_MAX_TENSORS (HTP_OP_MAX_REQS * HTP_OP_MAX_INPUTS + HTP_OP_MAX_REQS)
#define HTP_OP_MAX_TENSORS 8192 // must stay under 64K (uint16)
#define HTP_OP_MAX_VMEM_DEFAULT (3355443200u)
@@ -117,16 +116,18 @@ enum htp_op_code {
enum htp_tensor_flags {
HTP_TENSOR_COMPUTE = (1U << 0), // Tensor buffer temporal compute data (not weights)
HTP_TENSOR_FLUSHED = (1U << 1) // Tensor buffer has been flushed (set by the NPU)
HTP_TENSOR_DIRTY = (1U << 1) // Tensor buffer is dirty and needs to be flushed
};
// Tensor descriptor
struct htp_tensor {
uint32_t data; // Buffer offset in the messages, and data pointer on the NPU
uint32_t alias; // Index of the canonical tensor for this memory buffer
uint32_t size; // Data size in bytes
uint32_t flags; // Buffer / tensor flags
uint16_t type; // Data type
uint32_t type; // Data type
uint16_t bi; // Buffer index
uint16_t ti; // Tensor index
uint32_t ne[HTP_OP_MAX_DIMS]; // Number of elements
uint32_t nb[HTP_OP_MAX_DIMS]; // Stride in bytes (see ggml.h ggml_tensor)
};
@@ -169,6 +170,8 @@ enum htp_profiler_mode {
enum htp_trace_event_id {
HTP_TRACE_EVT_DMA = 0,
HTP_TRACE_EVT_L2FLUSH = 1,
HTP_TRACE_EVT_INIT = 2,
HTP_TRACE_EVT_HVX_COMP = 20,
HTP_TRACE_EVT_HVX_A_QUANT = 21,

204
ggml/src/ggml-hexagon/htp/htp-tensor.c Normal file
View File

@@ -0,0 +1,204 @@
#include "htp-tensor.h"
#include <qurt.h>
#include <qurt_memory.h>
#include "hex-common.h"
#include "hex-utils.h"
#include "hex-fastdiv.h"
#include "hex-profile.h"
#include "htp-ctx.h"
#include "work-queue.h"
struct l2flush_task {
struct htp_thread_trace * trace;
uint32_t start;
uint32_t end;
uint32_t chunk_size;
uint32_t ti;
};
static void l2flush_thread_worker(unsigned int n, unsigned int i, void * data) {
struct l2flush_task * task = (struct l2flush_task *) data;
const uint32_t start = task->start;
const uint32_t end = task->end;
const uint32_t ti = task->ti;
const uint32_t chunk_size = task->chunk_size;
const uint32_t thread_s = start + i * chunk_size;
if (thread_s >= end) {
return;
}
uint32_t thread_e = thread_s + chunk_size;
if (thread_e > end) {
thread_e = end;
}
struct htp_thread_trace * tr = &task->trace[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, ti);
hex_l2flush((void *) (uintptr_t) thread_s, thread_e - thread_s);
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, ti);
}
static void flush_all_dcache(struct htp_context * ctx) {
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, 0);
qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);
hex_l2fetch_block(ctx, ctx->footprint);
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, 0);
bitmap_reset(ctx->dirty_map, HTP_OP_MAX_TENSORS);
}
static void flush_tensor_range(struct htp_context * ctx, const struct htp_tensor * t) {
struct htp_thread_trace * tr = &ctx->trace[0];
if (t->size > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) {
struct l2flush_task task;
task.start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE);
task.end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE);
task.ti = t->ti;
task.trace = ctx->trace;
const uint32_t total_size = task.end - task.start;
const uint32_t n_blocks = (total_size + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE;
const uint32_t blocks_per_thread = fastdiv(n_blocks + ctx->n_threads - 1, &ctx->n_threads_div);
task.chunk_size = blocks_per_thread * HEX_L2_BLOCK_SIZE;
work_queue_run(ctx->work_queue, l2flush_thread_worker, &task, ctx->n_threads);
} else {
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);
hex_l2flush((void *) t->data, t->size);
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);
}
htp_tensor_make_clean(t, ctx->dirty_map);
}
void htp_tensor_flush(struct htp_context * ctx, const struct htp_tensor * t) {
if (!bitmap_test(ctx->dirty_map, t->ti)) {
return;
}
if (t->size > HEX_L2_FLUSH_ALL_THRESHOLD) {
flush_all_dcache(ctx);
return;
}
flush_tensor_range(ctx, t);
}
// One dirty tensor's line-aligned range, placed in the flattened global block space.
struct l2flush_range {
uint32_t start; // line-aligned start address
uint32_t end; // line-aligned end address
uint32_t block_first; // global block index of this range's first block
uint32_t n_blocks; // number of HEX_L2_BLOCK_SIZE chunks (last may be partial)
};
struct l2flush_multi_task {
struct htp_thread_trace * trace;
struct l2flush_range ranges[HTP_OP_MAX_INPUTS];
uint32_t n_ranges;
uint32_t total_blocks;
uint32_t blocks_per_thread;
};
static void l2flush_multi_worker(unsigned int n, unsigned int i, void * data) {
(void) n;
struct l2flush_multi_task * task = (struct l2flush_multi_task *) data;
const uint32_t gb_first = i * task->blocks_per_thread;
uint32_t gb_last = gb_first + task->blocks_per_thread;
if (gb_last > task->total_blocks) {
gb_last = task->total_blocks;
}
if (gb_first >= gb_last) {
return;
}
struct htp_thread_trace * tr = &task->trace[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, gb_first);
for (uint32_t r = 0; r < task->n_ranges; r++) {
const struct l2flush_range * rg = &task->ranges[r];
const uint32_t rb_first = rg->block_first;
const uint32_t rb_last = rg->block_first + rg->n_blocks;
const uint32_t lo = gb_first > rb_first ? gb_first : rb_first;
const uint32_t hi = gb_last < rb_last ? gb_last : rb_last;
if (lo >= hi) {
continue;
}
const uint32_t s = rg->start + (lo - rb_first) * HEX_L2_BLOCK_SIZE;
uint32_t e = rg->start + (hi - rb_first) * HEX_L2_BLOCK_SIZE;
if (e > rg->end) {
e = rg->end;
}
hex_l2flush((void *) (uintptr_t) s, e - s);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, gb_first);
}
void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n) {
uint64_t total_dirty = 0;
for (uint32_t i = 0; i < n; i++) {
const struct htp_tensor * t = tensors[i];
if (t && bitmap_test(ctx->dirty_map, t->ti)) {
total_dirty += t->size;
}
}
if (total_dirty == 0) {
return;
}
if (total_dirty > HEX_L2_FLUSH_ALL_THRESHOLD) {
flush_all_dcache(ctx);
return;
}
// Aggregate is small enough to walk. Thread it across all dirty ranges at once
// when it is worth the dispatch, otherwise flush sequentially.
if (total_dirty > HEX_L2_FLUSH_WQ_THRESHOLD && ctx->n_threads > 1) {
struct l2flush_multi_task task;
task.trace = ctx->trace;
task.n_ranges = 0;
uint32_t block_acc = 0;
for (uint32_t i = 0; i < n; i++) {
const struct htp_tensor * t = tensors[i];
if (!t || !bitmap_test(ctx->dirty_map, t->ti)) {
continue;
}
// Clear as we go: dedups a tensor passed as multiple srcs (e.g. mul(x,x)).
htp_tensor_make_clean(t, ctx->dirty_map);
struct l2flush_range * rg = &task.ranges[task.n_ranges++];
rg->start = hex_align_down((size_t) t->data, HEX_L2_LINE_SIZE);
rg->end = hex_align_up((size_t) t->data + t->size, HEX_L2_LINE_SIZE);
rg->block_first = block_acc;
rg->n_blocks = (rg->end - rg->start + HEX_L2_BLOCK_SIZE - 1) / HEX_L2_BLOCK_SIZE;
block_acc += rg->n_blocks;
}
task.total_blocks = block_acc;
task.blocks_per_thread = fastdiv(block_acc + ctx->n_threads - 1, &ctx->n_threads_div);
work_queue_run(ctx->work_queue, l2flush_multi_worker, &task, ctx->n_threads);
return;
}
struct htp_thread_trace * tr = &ctx->trace[0];
for (uint32_t i = 0; i < n; i++) {
const struct htp_tensor * t = tensors[i];
if (!t || !bitmap_test(ctx->dirty_map, t->ti)) {
continue;
}
htp_trace_event_start(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);
hex_l2flush((void *) t->data, t->size);
htp_trace_event_stop(tr, HTP_TRACE_EVT_L2FLUSH, t->ti);
htp_tensor_make_clean(t, ctx->dirty_map);
}
}

36
ggml/src/ggml-hexagon/htp/htp-tensor.h Normal file
View File

@@ -0,0 +1,36 @@
#ifndef HTP_TENSOR_H
#define HTP_TENSOR_H
#include <stdint.h>
#include "htp-ops.h"
#include "hex-bitmap.h"
static inline struct htp_tensor * htp_tensor_alias(const struct htp_tensor * t) {
return (struct htp_tensor *) (uintptr_t) t->alias;
}
static inline void * htp_tensor_data(const struct htp_tensor * t) {
return (void *) (uintptr_t) t->data;
}
static inline uint32_t * htp_tensor_flags(const struct htp_tensor * t) {
return (uint32_t *) &t->flags;
}
static inline void htp_tensor_make_dirty(const struct htp_tensor * t, uint32_t * dirty_map) {
struct htp_tensor * curr = (struct htp_tensor *) t;
do {
bitmap_set(dirty_map, curr->ti);
curr = htp_tensor_alias(curr);
} while (curr != t);
}
static inline void htp_tensor_make_clean(const struct htp_tensor * t, uint32_t * dirty_map) {
bitmap_clear(dirty_map, t->ti);
}
struct htp_context;
void htp_tensor_flush(struct htp_context * ctx, const struct htp_tensor * t);
void htp_tensor_flush_all(struct htp_context * ctx, const struct htp_tensor * const * tensors, uint32_t n);
#endif // HTP_TENSOR_H

807
ggml/src/ggml-hexagon/htp/main.c
View File

@@ -25,112 +25,44 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-bitmap.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "htp_iface.h"
#include "worker-pool.h"
#include "work-queue.h"
#include "hex-profile.h"
#define HMX_QUEUE_CAPACITY 16
#define HMX_QUEUE_STACK_SIZE 16384
#define WORK_QUEUE_CAPACITY 16
#define WORK_QUEUE_STACK_SIZE 16384
#define MAIN_THREAD_STACK_SIZE 32768
_Static_assert(WORK_QUEUE_MAX_N_THREADS >= HTP_MAX_NTHREADS,
"work-queue thread cap must be >= HTP_MAX_NTHREADS");
struct htp_handle {
struct htp_context * ctx;
};
AEEResult htp_iface_open(const char * uri, remote_handle64 * handle) {
struct htp_context * ctx;
int err = 0;
ctx = calloc(1, sizeof(*ctx));
if (ctx == NULL) {
(void) uri;
struct htp_handle * h = calloc(1, sizeof(*h));
if (h == NULL) {
return AEE_ENOMEMORY;
}
// Use the context structure as the handle
*handle = (remote_handle64) ctx;
// Enable FARF logs
HAP_setFARFRuntimeLoggingParams(0xffff, NULL, 0);
// Set client class
{
HAP_power_request_t request;
memset(&request, 0, sizeof(HAP_power_request_t));
request.type = HAP_power_set_apptype;
request.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS;
if ((err = HAP_power_set((void *) ctx, &request)) != 0) {
return err;
}
}
{
HAP_power_request_t request;
memset(&request, 0, sizeof(request));
request.type = HAP_power_set_DCVS_v3;
request.dcvs_v3.set_dcvs_enable = TRUE;
request.dcvs_v3.dcvs_enable = FALSE;
request.dcvs_v3.set_bus_params = TRUE;
request.dcvs_v3.bus_params.min_corner = HAP_DCVS_VCORNER_MAX;
request.dcvs_v3.bus_params.max_corner = HAP_DCVS_VCORNER_MAX;
request.dcvs_v3.bus_params.target_corner = HAP_DCVS_VCORNER_MAX;
request.dcvs_v3.set_core_params = TRUE;
request.dcvs_v3.core_params.min_corner = HAP_DCVS_VCORNER_MAX;
request.dcvs_v3.core_params.max_corner = HAP_DCVS_VCORNER_MAX;
request.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX;
request.dcvs_v3.set_sleep_disable = TRUE;
request.dcvs_v3.sleep_disable = TRUE;
#if (__HEXAGON_ARCH__ >= 79)
HAP_set_dcvs_v3_protected_bus_corners(&request, 1);
#endif
if ((err = HAP_power_set((void *) ctx, &request)) != 0) {
return err;
}
memset(&request, 0, sizeof(request));
request.type = HAP_power_set_HVX;
request.hvx.power_up = TRUE;
if ((err = HAP_power_set((void *) ctx, &request)) != 0) {
return err;
}
}
#if __HVX_ARCH__ >= 75
{
// Power on HMX and set HMX clock
HAP_power_request_t request;
memset(&request, 0, sizeof(HAP_power_request_t));
request.type = HAP_power_set_HMX_v2;
request.hmx_v2.set_power = TRUE;
request.hmx_v2.power_up = TRUE;
request.hmx_v2.set_clock = TRUE;
request.hmx_v2.target_corner = HAP_DCVS_EXP_VCORNER_MAX;
request.hmx_v2.min_corner = HAP_DCVS_EXP_VCORNER_MAX;
request.hmx_v2.max_corner = HAP_DCVS_EXP_VCORNER_MAX;
request.hmx_v2.perf_mode = HAP_CLK_PERF_HIGH;
FARF(ALWAYS, "Setting HMX clock\n");
err = HAP_power_set((void *) ctx, &request);
if (err != AEE_SUCCESS) {
FARF(ERROR, "ggml-hex: error setting HMX clock.");
return err;
}
}
#else
{
// Power on HMX
HAP_power_request_t request;
memset(&request, 0, sizeof(HAP_power_request_t));
request.type = HAP_power_set_HMX;
request.hmx.power_up = TRUE;
FARF(ALWAYS, "Powering HMX on\n");
err = HAP_power_set((void *) ctx, &request);
if (err != AEE_SUCCESS) {
FARF(ERROR, "ggml-hex: error powering on HMX.");
return err;
}
}
#endif
*handle = (remote_handle64) h;
return AEE_SUCCESS;
}
AEEResult htp_iface_etm(remote_handle64 handle, uint32_t enable) {
struct htp_handle * h = (struct htp_handle *) handle;
if (!h) {
return AEE_EBADPARM;
}
int err = enable ? HAP_user_etm_enable() : HAP_user_etm_disable();
if (err) {
if (err == AEE_EVERSIONNOTSUPPORT) {
@@ -143,10 +75,11 @@ AEEResult htp_iface_etm(remote_handle64 handle, uint32_t enable) {
}
AEEResult htp_iface_profiler(remote_handle64 handle, uint32_t mode, const htp_iface_pmu_conf* pmu_conf) {
struct htp_context * ctx = (struct htp_context *) handle;
if (!ctx) {
struct htp_handle * h = (struct htp_handle *) handle;
if (!h || !h->ctx) {
return AEE_EBADPARM;
}
struct htp_context * ctx = h->ctx;
if (mode == HTP_PROF_PMU) {
const uint32_t* events = pmu_conf->events;
@@ -179,48 +112,55 @@ AEEResult htp_iface_profiler(remote_handle64 handle, uint32_t mode, const htp_if
}
AEEResult htp_iface_close(remote_handle64 handle) {
struct htp_context * ctx = (struct htp_context *) handle;
if (!ctx) {
struct htp_handle * h = (struct htp_handle *) handle;
if (!h) {
return AEE_EBADPARM;
}
if (ctx->queue) {
FARF(ERROR, "Closing handle with queue still open");
return AEE_EITEMBUSY;
}
// release the mmaps (if any)
for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
if (ctx->mmap[i].size) {
#if __HVX_ARCH__ > 73
HAP_munmap2((void *) ctx->mmap[i].base, ctx->mmap[i].size);
#else
HAP_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size);
#endif
ctx->mmap[i].size = 0;
ctx->mmap[i].base = NULL;
ctx->mmap[i].fd = -1;
struct htp_context * ctx = h->ctx;
if (ctx) {
if (ctx->dsp_queue) {
FARF(ERROR, "Closing handle with queue still open");
return AEE_EITEMBUSY;
}
// release the mmaps (if any)
for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
if (ctx->mmap[i].size) {
#if __HVX_ARCH__ > 73
HAP_munmap2((void *) ctx->mmap[i].base, ctx->mmap[i].size);
#else
HAP_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size);
#endif
ctx->mmap[i].size = 0;
ctx->mmap[i].base = NULL;
ctx->mmap[i].fd = -1;
}
}
if (ctx->profiler) {
qurt_pmu_enable(1);
}
if (ctx->etm) {
HAP_user_etm_disable();
}
// Free the unified block (ctx is the base address of the block)
free(ctx);
h->ctx = NULL;
}
if (ctx->profiler) {
qurt_pmu_enable(1);
}
if (ctx->etm) {
HAP_user_etm_disable();
}
free(ctx);
free(h);
return AEE_SUCCESS;
}
AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint32_t size) {
struct htp_context * ctx = (struct htp_context *) handle;
if (!ctx) {
struct htp_handle * h = (struct htp_handle *) handle;
if (!h || !h->ctx) {
return AEE_EBADPARM;
}
struct htp_context * ctx = h->ctx;
// See if we already have this mapping
for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
@@ -262,10 +202,11 @@ AEEResult htp_iface_mmap(remote_handle64 handle, uint32_t fd, uint32_t size) {
}
AEEResult htp_iface_munmap(remote_handle64 handle, uint32 fd) {
struct htp_context * ctx = (struct htp_context *) handle;
if (!ctx) {
struct htp_handle * h = (struct htp_handle *) handle;
if (!h || !h->ctx) {
return AEE_EBADPARM;
}
struct htp_context * ctx = h->ctx;
for (uint32_t i=0; i<HTP_MAX_MMAPS; i++) {
struct htp_mmap *m = &ctx->mmap[i];
@@ -358,56 +299,44 @@ static void vtcm_free(struct htp_context * ctx) {
}
}
static void htp_main_thread(void * context);
static void htp_packet_callback(dspqueue_t queue, int error, void * context);
static void htp_error_callback(dspqueue_t queue, int error, void * context);
AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp_queue_id, uint32_t n_hvx, uint32_t n_hmx, uint64_t max_vmem) {
struct htp_context * ctx = (struct htp_context *) handle;
if (!ctx) {
struct htp_handle * h = (struct htp_handle *) handle;
if (!h) {
return AEE_EBADPARM;
}
if (ctx->queue) {
if (h->ctx) {
FARF(ERROR, "Queue already open");
return AEE_EITEMBUSY;
}
// Import queue created on the CPU
int err = dspqueue_import(dsp_queue_id, // Queue ID from dspqueue_export
htp_packet_callback, // Packet callback
htp_error_callback, // Error callback; no errors expected on the DSP
(void *) ctx, // Callback context
&ctx->queue);
// Cache the original FastRPC thread priority, then calculate compute priority
int fastrpc_tid = qurt_thread_get_id();
int fastrpc_prio = qurt_thread_get_priority(fastrpc_tid);
int main_prio = fastrpc_prio - 10;
if (main_prio < 1) main_prio = 1;
dspqueue_t dsp_queue = NULL;
bool use_callbacks = false;
// Import queue with NULL callbacks to avoid starting dspueue internal threads
int err = dspqueue_import(dsp_queue_id, NULL, NULL, (void *) h, &dsp_queue);
if (err == AEE_EBADPARM) {
// Fallback for devices that don't support NULL callbacks
FARF(HIGH, "dspqueue import with NULL callbacks failed, trying with callbacks");
use_callbacks = true;
err = dspqueue_import(dsp_queue_id, htp_packet_callback, htp_error_callback, (void *) h, &dsp_queue);
}
if (err) {
FARF(ERROR, "Queue import failed with 0x%08x", (unsigned) err);
return err;
}
ctx->max_vmem = max_vmem;
ctx->thread_id = qurt_thread_get_id();
ctx->thread_prio = qurt_thread_get_priority(ctx->thread_id);
// allocate VTCM
err = vtcm_alloc(ctx);
if (err != AEE_SUCCESS) {
FARF(ERROR, "Unable to allocate VTCM");
return AEE_ENOMEMORY;
}
ctx->hmx_enabled = n_hmx;
ctx->hmx_queue = NULL;
if (n_hmx) {
ctx->hmx_queue = hmx_queue_create(16, ctx->vtcm_rctx);
if (ctx->hmx_queue) {
ctx->hmx_queue->trace = &ctx->trace[HTP_MAX_NTHREADS];
} else {
FARF(ERROR, "hmx-queue-create failed");
ctx->hmx_enabled = false;
}
}
FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx);
qurt_sysenv_max_hthreads_t hw_threads;
qurt_sysenv_get_max_hw_threads(&hw_threads);
uint32_t hw_nhvx = (qurt_hvx_get_units() >> 8) & 0xFF;
@@ -422,27 +351,216 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp
n_hvx = HTP_MAX_NTHREADS;
}
ctx->n_threads = n_hvx;
for (int i = 0; i < ctx->n_threads; i++) {
ctx->dma[i] = dma_queue_create(256); // queue depth
if (ctx->dma[i]) {
ctx->dma[i]->trace = &ctx->trace[i];
// layout segments of our contiguous block
// 1. htp_context : sits at the base (block is 4K-aligned via memalign below)
size_t offset = sizeof(struct htp_context);
// 2. main_stack
size_t offset_main_stack = 0;
size_t size_main_stack = 0;
if (!use_callbacks) {
offset_main_stack = hex_align_up(offset, 4096);
size_main_stack = MAIN_THREAD_STACK_SIZE;
offset = offset_main_stack + size_main_stack;
}
// 3. work_queue
size_t wq_align = work_queue_alignof();
size_t offset_wq = hex_align_up(offset, wq_align);
size_t size_wq = work_queue_sizeof(n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE);
offset = offset_wq + size_wq;
// 4. dma_queue
size_t dma_align = dma_queue_alignof();
size_t offset_dma = hex_align_up(offset, dma_align);
size_t size_dma = 0;
for (uint32_t i = 0; i < n_hvx; i++) {
size_dma = hex_align_up(size_dma, dma_queue_alignof());
size_dma += dma_queue_sizeof(256);
size_dma = hex_align_up(size_dma, dma_queue_alignof());
size_dma += dma_queue_alias_sizeof();
}
offset = offset_dma + size_dma;
// 5. hmx_queue
size_t offset_hmx = 0;
size_t size_hmx = 0;
if (n_hmx) {
size_t hmx_align = hmx_queue_alignof();
offset_hmx = hex_align_up(offset, hmx_align);
size_hmx = hmx_queue_sizeof(HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE);
offset = offset_hmx + size_hmx;
}
size_t footprint = hex_align_up(offset, 128);
void * block = memalign(4096, footprint);
if (!block) {
FARF(ERROR, "Unable to allocate unified block of size %zu\n", footprint);
dspqueue_close(dsp_queue);
return AEE_ENOMEMORY;
}
memset(block, 0, footprint);
h->ctx = (struct htp_context *) block;
struct htp_context * ctx = h->ctx;
ctx->footprint = footprint;
ctx->thread_id = fastrpc_tid;
ctx->thread_prio = main_prio;
ctx->max_vmem = max_vmem;
ctx->dsp_queue = dsp_queue;
err = vtcm_alloc(ctx);
if (err != AEE_SUCCESS) {
FARF(ERROR, "Unable to allocate VTCM");
htp_iface_stop(handle);
return AEE_ENOMEMORY;
}
HAP_setFARFRuntimeLoggingParams(0xffff, NULL, 0);
// Set client class
{
HAP_power_request_t request;
memset(&request, 0, sizeof(HAP_power_request_t));
request.type = HAP_power_set_apptype;
request.apptype = HAP_POWER_COMPUTE_CLIENT_CLASS;
if ((err = HAP_power_set((void *) ctx, &request)) != 0) {
htp_iface_stop(handle);
return err;
}
}
// DCVS setup
{
HAP_power_request_t request;
memset(&request, 0, sizeof(request));
request.type = HAP_power_set_DCVS_v3;
request.dcvs_v3.set_dcvs_enable = TRUE;
request.dcvs_v3.dcvs_enable = FALSE;
request.dcvs_v3.set_bus_params = TRUE;
request.dcvs_v3.bus_params.min_corner = HAP_DCVS_VCORNER_MAX;
request.dcvs_v3.bus_params.max_corner = HAP_DCVS_VCORNER_MAX;
request.dcvs_v3.bus_params.target_corner = HAP_DCVS_VCORNER_MAX;
request.dcvs_v3.set_core_params = TRUE;
request.dcvs_v3.core_params.min_corner = HAP_DCVS_VCORNER_MAX;
request.dcvs_v3.core_params.max_corner = HAP_DCVS_VCORNER_MAX;
request.dcvs_v3.core_params.target_corner = HAP_DCVS_VCORNER_MAX;
request.dcvs_v3.set_sleep_disable = TRUE;
request.dcvs_v3.sleep_disable = TRUE;
#if (__HEXAGON_ARCH__ >= 79)
HAP_set_dcvs_v3_protected_bus_corners(&request, 1);
#endif
if ((err = HAP_power_set((void *) ctx, &request)) != 0) {
htp_iface_stop(handle);
return err;
}
memset(&request, 0, sizeof(request));
request.type = HAP_power_set_HVX;
request.hvx.power_up = TRUE;
if ((err = HAP_power_set((void *) ctx, &request)) != 0) {
htp_iface_stop(handle);
return err;
}
}
#if __HVX_ARCH__ >= 75
{
// Power on HMX and set HMX clock
HAP_power_request_t request;
memset(&request, 0, sizeof(HAP_power_request_t));
request.type = HAP_power_set_HMX_v2;
request.hmx_v2.set_power = TRUE;
request.hmx_v2.power_up = TRUE;
request.hmx_v2.set_clock = TRUE;
request.hmx_v2.target_corner = HAP_DCVS_EXP_VCORNER_MAX;
request.hmx_v2.min_corner = HAP_DCVS_EXP_VCORNER_MAX;
request.hmx_v2.max_corner = HAP_DCVS_EXP_VCORNER_MAX;
request.hmx_v2.perf_mode = HAP_CLK_PERF_HIGH;
FARF(ALWAYS, "Setting HMX clock\n");
err = HAP_power_set((void *) ctx, &request);
if (err != AEE_SUCCESS) {
FARF(ERROR, "ggml-hex: error setting HMX clock.");
htp_iface_stop(handle);
return err;
}
}
#else
{
// Power on HMX
HAP_power_request_t request;
memset(&request, 0, sizeof(HAP_power_request_t));
request.type = HAP_power_set_HMX;
request.hmx.power_up = TRUE;
FARF(ALWAYS, "Powering HMX on\n");
err = HAP_power_set((void *) ctx, &request);
if (err != AEE_SUCCESS) {
FARF(ERROR, "ggml-hex: error powering on HMX.");
htp_iface_stop(handle);
return err;
}
}
#endif
ctx->hmx_enabled = n_hmx;
ctx->hmx_queue = NULL;
if (n_hmx) {
void * hmx_ptr = (void *) ((uintptr_t) block + offset_hmx);
ctx->hmx_queue = hmx_queue_init(hmx_ptr, HMX_QUEUE_CAPACITY, HMX_QUEUE_STACK_SIZE, ctx->vtcm_rctx, &ctx->trace[HTP_MAX_NTHREADS]);
}
FARF(HIGH, "HMX %s (n_hmx=%d)", ctx->hmx_enabled ? "enabled" : "disabled", n_hmx);
ctx->n_threads = n_hvx;
ctx->n_threads_div = init_fastdiv_values(ctx->n_threads);
// Initialize DMA queues
uint8_t * dma_ptr_curr = (uint8_t *) ((uintptr_t) block + offset_dma);
size_t size_dma_q = dma_queue_sizeof(256);
size_t size_dma_alias = dma_queue_alias_sizeof();
for (int i = 0; i < ctx->n_threads; i++) {
dma_ptr_curr = (uint8_t *) hex_align_up((uintptr_t) dma_ptr_curr, dma_queue_alignof());
ctx->dma_cached[i] = dma_queue_init(dma_ptr_curr, 256, (uintptr_t) ctx->vtcm_base, ctx->vtcm_size, &ctx->trace[i]);
dma_ptr_curr += size_dma_q;
dma_ptr_curr = (uint8_t *) hex_align_up((uintptr_t) dma_ptr_curr, dma_queue_alignof());
ctx->dma[i] = dma_queue_alias_init(dma_ptr_curr, ctx->dma_cached[i], 1);
dma_ptr_curr += size_dma_alias;
}
ctx->ddr_spad_size = 512 * 1024; // 512 KB
ctx->ddr_spad_base = memalign(128, ctx->ddr_spad_size);
// init worker pool
err = worker_pool_init(&ctx->worker_pool, n_hvx);
if (err != AEE_SUCCESS) {
FARF(ERROR, "Unable to create worker pool");
if (ctx->ddr_spad_base) {
free(ctx->ddr_spad_base);
ctx->ddr_spad_base = NULL;
ctx->ddr_spad_size = 0;
void * wq_ptr = (void *) ((uintptr_t) block + offset_wq);
ctx->work_queue = work_queue_init(wq_ptr, n_hvx, WORK_QUEUE_CAPACITY, WORK_QUEUE_STACK_SIZE);
ctx->main_stack = NULL;
ctx->main_thread = 0;
atomic_store(&ctx->killed, false);
if (!use_callbacks) {
// Start main compute thread
ctx->main_stack = (void *) ((uintptr_t) block + offset_main_stack);
qurt_thread_attr_t attr;
qurt_thread_attr_init(&attr);
qurt_thread_attr_set_stack_addr(&attr, ctx->main_stack);
qurt_thread_attr_set_stack_size(&attr, size_main_stack);
qurt_thread_attr_set_priority(&attr, main_prio);
qurt_thread_attr_set_name(&attr, "htp-main");
int err_thread = qurt_thread_create(&ctx->main_thread, &attr, htp_main_thread, ctx);
if (err_thread) {
FARF(ERROR, "Unable to create htp main thread: %d", err_thread);
htp_iface_stop(handle);
return AEE_ENOMEMORY;
}
return err;
}
FARF(HIGH, "session %u started: n-hvx %u vtcm-size %zu vtcm-rctx %u n-threads %u thread-id %d thread-prio %d \n",
@@ -452,35 +570,34 @@ AEEResult htp_iface_start(remote_handle64 handle, uint32_t sess_id, uint64_t dsp
}
AEEResult htp_iface_stop(remote_handle64 handle) {
struct htp_context * ctx = (struct htp_context *) handle;
if (!ctx) {
struct htp_handle * h = (struct htp_handle *) handle;
if (!h || !h->ctx) {
return AEE_EBADPARM;
}
struct htp_context * ctx = h->ctx;
if (!ctx->queue) {
FARF(ERROR, "Queue not open");
return AEE_EBADSTATE;
if (ctx->main_thread) {
atomic_store(&ctx->killed, true);
int status;
(void) qurt_thread_join(ctx->main_thread, &status);
ctx->main_thread = 0;
}
// Close queue. dspqueue_close() will also wait for callbacks to finish.
int err = dspqueue_close(ctx->queue);
ctx->queue = NULL;
int err = dspqueue_close(ctx->dsp_queue); ctx->dsp_queue = NULL;
if (err != 0) {
FARF(ERROR, "Queue close failed with 0x%08x", (unsigned) err);
return err;
}
if (ctx->worker_pool) {
// Release worker pool
worker_pool_release(&ctx->worker_pool);
}
work_queue_free(ctx->work_queue);
for (int i = 0; i < ctx->n_threads; i++) {
dma_queue_delete(ctx->dma[i]);
dma_queue_alias_free(ctx->dma[i]);
dma_queue_free(ctx->dma_cached[i]);
}
if (ctx->hmx_queue) {
hmx_queue_delete(ctx->hmx_queue);
hmx_queue_free(ctx->hmx_queue);
ctx->hmx_queue = NULL;
}
ctx->hmx_enabled = false;
@@ -493,6 +610,9 @@ AEEResult htp_iface_stop(remote_handle64 handle) {
ctx->ddr_spad_size = 0;
}
free(ctx);
h->ctx = NULL;
return AEE_SUCCESS;
}
@@ -671,8 +791,6 @@ static int execute_op(struct htp_ops_context * octx) {
case HTP_OP_INVALID:
break;
// No default to catch missing cases
}
FARF(ERROR, "Unknown Op %u", octx->op);
@@ -778,12 +896,14 @@ static void prep_op_bufs(struct htp_context *ctx, struct htp_buf_desc *bufs, uin
}
}
static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, uint32_t idx, struct htp_tensor *t) {
static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t idx, struct htp_tensor *t) {
uint32_t offset = t->data;
uint32_t size = t->size;
uint32_t bi = t->bi;
uint32_t alias = t->alias;
t->data = bufs[bi].base + offset; // update data to the actual pointer
t->data = (uint32_t) (bufs[bi].base + offset); // update data to the actual pointer
t->alias = (uint32_t) (tens + alias); // update alias to the actual pointer
FARF(HIGH, "prep-tensor #%u: bi %u offset %u size %u data %p : %u:%u:%u:%u", idx, t->bi, offset, t->size, (void*) t->data,
t->ne[0], t->ne[1], t->ne[3], t->ne[3]);
@@ -791,7 +911,7 @@ static void prep_tensor(struct htp_context *ctx, struct htp_buf_desc *bufs, uint
static void prep_tensors(struct htp_context *ctx, struct htp_buf_desc *bufs, struct htp_tensor *tens, uint32_t n_tens) {
for (uint32_t i=0; i < n_tens; i++) {
prep_tensor(ctx, bufs, i, tens + i);
prep_tensor(ctx, bufs, tens, i, tens + i);
}
}
@@ -805,29 +925,36 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u
// Prep input tensors
for (uint32_t i=0; i<HTP_OP_MAX_INPUTS; i++) {
struct htp_tensor *src = op->src[i] == 0xffff ? NULL : tens + op->src[i];
octx->src[i] = src;
if (!src) continue;
if (!(src->flags & HTP_TENSOR_FLUSHED) && (src->flags & HTP_TENSOR_COMPUTE)) {
// flush compute buffers on input
hex_l2flush((void *) src->data, src->size);
uint16_t src_idx = op->src[i];
if (src_idx == 0xffff) {
octx->src[i] = NULL;
octx->src_dma[i] = NULL;
continue;
}
struct htp_tensor *src = tens + src_idx;
octx->src[i] = src;
octx->src_dma[i] = octx->ctx->dma; // FIXME: ? octx->ctx->dma_cached : octx->ctx->dma;
FARF(HIGH, "prep-src #%u: data %p size %u : %u:%u:%u:%u", op->src[i], (void*) src->data, src->size,
src->ne[0], src->ne[1], src->ne[3], src->ne[3]);
}
htp_tensor_flush_all(octx->ctx, octx->src, HTP_OP_MAX_INPUTS);
// Prep output tensors
for (uint32_t i = 0; i < HTP_OP_MAX_OUTPUTS; i++) {
uint16_t dst_idx = op->dst[i];
if (dst_idx == 0xffff) {
octx->dsts[i] = NULL;
octx->dsts[i] = NULL;
octx->dst_dma[i] = NULL;
continue;
}
struct htp_tensor *dst = tens + dst_idx;
octx->dsts[i] = dst;
octx->dsts[i] = dst;
octx->dst_dma[i] = octx->ctx->dma; // FIXME: ? octx->ctx->dma_cached : octx->ctx->dma;
htp_tensor_make_dirty(dst, octx->ctx->dirty_map);
FARF(HIGH, "prep-dst[%u] #%u: data %p size %u : %u:%u:%u:%u", i, dst_idx, (void*) dst->data, dst->size,
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
@@ -841,34 +968,134 @@ static int proc_op_req(struct htp_ops_context * octx, struct htp_tensor *tens, u
octx->src3_spad.src = NULL;
octx->dst_spad.src = NULL;
// flush buffers on output
for (uint32_t i = 0; i < HTP_OP_MAX_OUTPUTS; i++) {
if (octx->dsts[i]) {
struct htp_tensor *dst = (struct htp_tensor *)octx->dsts[i];
hex_l2flush((void *) dst->data, dst->size);
dst->flags |= HTP_TENSOR_FLUSHED;
FARF(HIGH, "post-dst[%u] #%u: data %p size %u : %u:%u:%u:%u", i, op->dst[i], (void*) dst->data, dst->size,
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]);
}
}
return status;
}
static void process_opbatch(struct htp_context * ctx, const struct htp_opbatch_req * req, const struct dspqueue_buffer * dbuf) {
dspqueue_t queue = ctx->dsp_queue;
int err;
const uint32_t n_bufs = req->n_bufs;
const uint32_t n_tens = req->n_tensors;
const uint32_t n_ops = req->n_ops;
const uint32_t b_size = sizeof(struct htp_buf_desc) * n_bufs;
const uint32_t t_size = sizeof(struct htp_tensor) * n_tens;
const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops;
const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops;
const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req->n_traces * sizeof(struct htp_trace_desc);
if (dbuf->size < b_size + t_size + o_size + p_size + tr_size) {
FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf->size, b_size + t_size + o_size + p_size + tr_size);
return;
}
FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", req->id,
n_bufs, n_tens, n_ops, req->n_traces, dbuf->size, b_size, t_size, o_size);
// Clean cache at the start of the batch
// We cant trace this part because the trace buffer is setup later
qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);
hex_l2fetch_block(ctx, ctx->footprint);
bitmap_reset(ctx->dirty_map, HTP_OP_MAX_TENSORS);
// Setup descriptor pointers
uint8_t * m_ptr = dbuf->ptr;
struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size;
struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size;
struct htp_op_desc* ops = (struct htp_op_desc*) m_ptr; m_ptr += o_size;
struct htp_prof_desc* pds = (struct htp_prof_desc*) m_ptr;
prep_op_bufs(ctx, bufs, n_bufs);
prep_tensors(ctx, bufs, tens, n_tens);
struct htp_ops_context *octx = &ctx->octx;
memset(octx, 0, sizeof(*octx));
octx->n_threads = ctx->n_threads;
octx->ctx = ctx;
memset(ctx->trace, 0, sizeof(ctx->trace));
if (ctx->profiler == HTP_PROF_TRACE) {
struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size);
for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
ctx->trace[t].events = &trace_events[t * req->n_traces];
ctx->trace[t].max_events = req->n_traces;
}
}
work_queue_wakeup(ctx->work_queue);
if (ctx->hmx_queue) {
hmx_queue_wakeup(ctx->hmx_queue);
}
int op_status = HTP_STATUS_OK;
for (uint32_t i = 0; i < n_ops && op_status == HTP_STATUS_OK; i++) {
struct profile_data prof;
profile_start(ctx->profiler, &prof);
op_status = proc_op_req(octx, tens, i, &ops[i]);
profile_stop(ctx->profiler, &prof);
if (ctx->profiler) {
pds[i].opcode = ops[i].opcode;
pds[i].usecs = prof.usecs;
pds[i].cycles_start = prof.cycles_start;
pds[i].cycles_stop = prof.cycles_stop;
for (int j = 0; j < HEX_NUM_PMU_COUNTERS; j++) {
pds[i].pmu[j] = prof.pmu_counters[j];
}
}
}
if (ctx->hmx_queue) {
hmx_queue_suspend(ctx->hmx_queue);
hmx_queue_flush(ctx->hmx_queue);
}
work_queue_suspend(ctx->work_queue);
struct htp_opbatch_rsp rsp;
memset(&rsp, 0, sizeof(rsp));
rsp.id = req->id;
rsp.status = op_status;
rsp.n_bufs = n_bufs;
rsp.n_tensors = n_tens;
rsp.n_ops = n_ops;
if (ctx->profiler == HTP_PROF_TRACE) {
for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
rsp.n_traces[t] = ctx->trace[t].count;
}
}
struct dspqueue_buffer write_dbuf = *dbuf;
write_dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;
// Flush remaining dirty tensors at the end of the batch
htp_trace_event_start(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0);
qurt_mem_cache_clean((qurt_addr_t) 0, 0, QURT_MEM_CACHE_FLUSH_INVALIDATE_ALL, QURT_MEM_DCACHE);
htp_trace_event_stop(&ctx->trace[0], HTP_TRACE_EVT_L2FLUSH, 0);
err = dspqueue_write(queue, 0, 1, &write_dbuf, sizeof(rsp), (const uint8_t *) &rsp, DSPQUEUE_TIMEOUT_NONE);
if (err != 0) {
FARF(ERROR, "dspqueue_write failed: 0x%08x", (unsigned) err);
}
}
#define DSPQUEUE_READ_TIMEOUT_USEC 5000
#define DSPQUEUE_POLL_TIMEOUT_USEC 100
#define DSPQUEUE_POLL_COUNT 100
static void htp_packet_callback(dspqueue_t queue, int error, void * context) {
struct htp_context * ctx = (struct htp_context *) context;
static void process_ops(struct htp_context * ctx) {
dspqueue_t queue = ctx->dsp_queue;
int err;
uint32_t poll_count = DSPQUEUE_POLL_COUNT;
vtcm_acquire(ctx);
while (!ctx->vtcm_needs_release) {
while (!ctx->vtcm_needs_release && !atomic_load(&ctx->killed)) {
struct htp_opbatch_req req;
uint32_t r_size = sizeof(req);
@@ -898,111 +1125,41 @@ static void htp_packet_callback(dspqueue_t queue, int error, void * context) {
// Reset poll count for valid requests
poll_count = DSPQUEUE_POLL_COUNT;
const uint32_t n_bufs = req.n_bufs;
const uint32_t n_tens = req.n_tensors;
const uint32_t n_ops = req.n_ops;
const uint32_t b_size = sizeof(struct htp_buf_desc) * n_bufs;
const uint32_t t_size = sizeof(struct htp_tensor) * n_tens;
const uint32_t o_size = sizeof(struct htp_op_desc) * n_ops;
const uint32_t p_size = sizeof(struct htp_prof_desc) * n_ops;
const uint32_t tr_size = (HTP_MAX_NTHREADS + 1) * req.n_traces * sizeof(struct htp_trace_desc);
if (dbuf.size < b_size + t_size + o_size + p_size + tr_size) {
FARF(ERROR, "invalid opbatch memory block size %u (req %u)", dbuf.size, b_size + t_size + o_size + p_size + tr_size);
break;
}
FARF(HIGH, "processing opbatch #%u: n-bufs %u n-tensors %u n-ops %u n-traces %u : m-size %u b-size %u t-size %u o-size %u", req.id,
n_bufs, n_tens, n_ops, req.n_traces, dbuf.size, b_size, t_size, o_size);
// Setup descriptor pointers
uint8_t * m_ptr = dbuf.ptr;
struct htp_buf_desc* bufs = (struct htp_buf_desc*) m_ptr; m_ptr += b_size;
struct htp_tensor* tens = (struct htp_tensor*) m_ptr; m_ptr += t_size;
struct htp_op_desc* ops = (struct htp_op_desc*) m_ptr; m_ptr += o_size;
struct htp_prof_desc* pds = (struct htp_prof_desc*) m_ptr;
prep_op_bufs(ctx, bufs, n_bufs);
prep_tensors(ctx, bufs, tens, n_tens);
struct htp_ops_context *octx = &ctx->octx;
memset(octx, 0, sizeof(*octx));
octx->n_threads = ctx->n_threads;
octx->ctx = ctx;
if (ctx->profiler == HTP_PROF_TRACE) {
memset(ctx->trace, 0, sizeof(ctx->trace));
struct htp_trace_desc * trace_events = (struct htp_trace_desc *) (m_ptr + p_size);
for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
ctx->trace[t].events = &trace_events[t * req.n_traces];
ctx->trace[t].max_events = req.n_traces;
}
} else {
for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
ctx->trace[t].events = NULL;
ctx->trace[t].max_events = 0;
}
}
int op_status = HTP_STATUS_OK;
uint32_t op_wakeup = n_ops / 2; // half-way throgh the batch
hmx_queue_wakeup(ctx->hmx_queue);
for (uint32_t i=0; i < n_ops; i++) {
struct profile_data prof;
if (i == op_wakeup) {
dspqueue_write_early_wakeup_noblock(queue, 0, 0);
}
profile_start(ctx->profiler, &prof);
op_status = proc_op_req(octx, tens, i, &ops[i]);
profile_stop(ctx->profiler, &prof);
if (op_status != HTP_STATUS_OK) {
break;
}
if (ctx->profiler) {
pds[i].opcode = ops[i].opcode;
pds[i].usecs = prof.usecs;
pds[i].cycles_start = prof.cycles_start;
pds[i].cycles_stop = prof.cycles_stop;
for (int j = 0; j < HEX_NUM_PMU_COUNTERS; j++) {
pds[i].pmu[j] = prof.pmu_counters[j];
}
}
}
hmx_queue_suspend(ctx->hmx_queue);
struct htp_opbatch_rsp rsp;
rsp.id = req.id;
rsp.status = op_status;
rsp.n_bufs = n_bufs;
rsp.n_tensors = n_tens;
rsp.n_ops = n_ops;
memset(rsp.pad, 0, sizeof(rsp.pad));
if (ctx->profiler == HTP_PROF_TRACE) {
for (int t = 0; t <= HTP_MAX_NTHREADS; t++) {
rsp.n_traces[t] = ctx->trace[t].count;
}
} else {
memset(rsp.n_traces, 0, sizeof(rsp.n_traces));
}
dbuf.flags = DSPQUEUE_BUFFER_FLAG_FLUSH_SENDER | DSPQUEUE_BUFFER_FLAG_INVALIDATE_RECIPIENT;
err = dspqueue_write(queue, 0, 1, &dbuf, sizeof(rsp), (const uint8_t *) &rsp, DSPQUEUE_TIMEOUT_NONE);
if (err != 0) {
FARF(ERROR, "dspqueue_write failed: 0x%08x", (unsigned) err);
break;
}
process_opbatch(ctx, &req, &dbuf);
}
vtcm_release(ctx);
}
static void htp_packet_callback(dspqueue_t queue, int error, void * context) {
(void) queue;
(void) error;
struct htp_handle * h = (struct htp_handle *) context;
if (h && h->ctx) {
process_ops(h->ctx);
}
}
static void htp_main_thread(void * context) {
struct htp_context * ctx = (struct htp_context *) context;
FARF(HIGH, "htp-main-thread: started");
while (!atomic_load(&ctx->killed)) {
uint32_t flags = 0;
uint32_t num_buffers = 0;
uint32_t message_length = 0;
int err = dspqueue_peek(ctx->dsp_queue, &flags, &num_buffers, &message_length, 50000);
if (err == 0) {
process_ops(ctx);
} else if (err == AEE_EWOULDBLOCK || err == AEE_EEXPIRED) {
continue;
} else {
FARF(ERROR, "dspqueue_peek failed: 0x%08x", (unsigned) err);
break;
}
}
FARF(HIGH, "htp-main-thread: stopped");
}

467
ggml/src/ggml-hexagon/htp/matmul-ops.c
View File

@@ -92,10 +92,10 @@ struct htp_mm_context {
// Per thread quant tasks
// Precomputed block-parallel quantization values
worker_callback_t quant_task_func;
uint32_t quant_ib_first[MAX_NUM_WORKERS];
uint32_t quant_ib_last[MAX_NUM_WORKERS];
uint32_t quant_r[MAX_NUM_WORKERS];
uint32_t quant_c[MAX_NUM_WORKERS];
uint32_t quant_ib_first[WORK_QUEUE_MAX_N_THREADS];
uint32_t quant_ib_last[WORK_QUEUE_MAX_N_THREADS];
uint32_t quant_r[WORK_QUEUE_MAX_N_THREADS];
uint32_t quant_c[WORK_QUEUE_MAX_N_THREADS];
uint32_t n_quant_tasks;
uint32_t n_quant_rows_per_thread;
atomic_uint quant_barrier;
@@ -254,7 +254,7 @@ static void hvx_mm_4d(unsigned int nth, unsigned int ith, void * data) {
return;
}
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir0_start);
const uint32_t blck_0 = 64;
@@ -309,7 +309,7 @@ static void hvx_mm_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void
const uint32_t src0_start_row = src0_nrows_per_thread * ith; \
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \
\
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
\
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \
const uint32_t n_prefetch = kparams->n_prefetch; \
@@ -410,7 +410,7 @@ static void hvx_mv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void
const uint32_t src0_start_row = src0_nrows_per_thread * ith; \
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows); \
\
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
\
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \
const uint32_t n_prefetch = kparams->n_prefetch; \
@@ -523,7 +523,7 @@ static void hvx_mm_qkv_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith,
uint8_t * restrict vtcm_src3_ptr = mmctx->vtcm_src3 + mmctx->vtcm_src3_size_per_thread * ith; \
uint8_t * restrict src1_data = mmctx->vtcm_src1; \
\
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
\
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params; \
const uint32_t n_prefetch = kparams->n_prefetch; \
@@ -699,7 +699,7 @@ static void hvx_mm_ffn_2d_repacked_##SUFFIX(unsigned int nth, unsigned int ith,
uint8_t * restrict vtcm_src2_ptr = mmctx->vtcm_src2 + mmctx->vtcm_src2_size_per_thread * ith; \
uint8_t * restrict src1_data = mmctx->vtcm_src1; \
\
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
\
const uint8_t * restrict src0_row = (const uint8_t *) src0->data; \
const uint8_t * restrict src2_row = (const uint8_t *) src2->data; \
@@ -820,7 +820,7 @@ static void name(unsigned int nth, unsigned int ith, void * data) {
return; \
} \
\
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, ir_first); \
\
uint8_t * restrict dst = mmctx->vtcm_src1; \
@@ -846,7 +846,7 @@ QUANTIZE_IMPL(quantize_f16_f16_flat, "quantize-f16-f16", quantize_f16_f
static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, void * data) {
struct htp_mm_context * mmctx = data;
struct htp_ops_context * octx = mmctx->octx;
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);
const struct htp_tensor * src = octx->src[1];
@@ -870,7 +870,7 @@ static void quantize_f32_q8_0_tiled_block(unsigned int nth, unsigned int ith, vo
static void quantize_f32_q8_1_tiled_block(unsigned int nth, unsigned int ith, void * data) {
struct htp_mm_context * mmctx = data;
struct htp_ops_context * octx = mmctx->octx;
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, mmctx->quant_ib_first[ith]);
const struct htp_tensor * src = octx->src[1];
@@ -944,7 +944,7 @@ static void hvx_mm_2d(unsigned int nth, unsigned int ith, void * data) {
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
const uint32_t src0_end_row_x2 = src0_start_row + ((src0_end_row - src0_start_row) & ~1U);
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
const size_t dst_row_size = nb1;
const size_t src0_row_size = nb01;
@@ -1040,7 +1040,7 @@ static void hvx_mv_2d(unsigned int nth, unsigned int ith, void * data) {
const uint32_t src0_start_row = src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
const size_t dst_row_size = nb1;
const size_t src0_row_size = nb01;
@@ -1155,7 +1155,7 @@ static void hvx_mm_id(unsigned int nth, unsigned int ith, void * data) {
return;
}
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
const uint32_t n_prefetch = kparams->n_prefetch;
@@ -1244,7 +1244,7 @@ static void hvx_mv_id(unsigned int nth, unsigned int ith, void * data) {
return;
}
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL;
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
const uint32_t n_prefetch = kparams->n_prefetch;
@@ -1338,6 +1338,9 @@ static int hvx_mm_init_vec_dot(struct htp_mm_context * mmctx, enum htp_data_type
static int hvx_mm_matmul(struct htp_ops_context * octx) {
htp_matmul_tensors_preamble;
struct htp_thread_trace * tr = &octx->ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
struct htp_mm_context mmctx_struct = {0};
struct htp_mm_context * mmctx = &mmctx_struct;
mmctx->octx = octx;
@@ -1557,9 +1560,6 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
mmctx->vtcm_src0_stride = src0_row_size_padded;
mmctx->vtcm_src1_stride = src1_row_size;
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)
return HTP_STATUS_OK;
if (need_quant) {
mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks;
mmctx->quant_task_func = quant_task_func;
@@ -1570,8 +1570,9 @@ static int hvx_mm_matmul(struct htp_ops_context * octx) {
mmctx->n_quant_tasks = 0;
}
const uint32_t n_matmul_jobs = octx->n_threads;
worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, octx->n_threads);
return HTP_STATUS_OK;
}
@@ -1874,7 +1875,7 @@ static void hvx_mm_ffn_2d(unsigned int nth, unsigned int ith, void * data) {
#define DEQUANTIZE_WORKER_LOOP_IMPL(SUFFIX) \
static void dequantize_tiled_worker_loop_##SUFFIX(unsigned int n, unsigned int i, void *data) { \
tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data; \
struct htp_thread_trace * tr = state->traces ? &state->traces[i] : NULL; \
struct htp_thread_trace * tr = &state->traces[i]; \
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i); \
for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) { \
int start = task_id * state->n_tiles_per_task; \
@@ -1892,7 +1893,7 @@ DEQUANTIZE_WORKER_LOOP_IMPL(q8_0)
static void convert_f16_worker_loop(unsigned int n, unsigned int i, void *data) {
tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data;
struct htp_thread_trace * tr = state->traces ? &state->traces[i] : NULL;
struct htp_thread_trace * tr = &state->traces[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, i);
for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) {
int start = task_id * state->n_tiles_per_task;
@@ -1905,7 +1906,7 @@ static void convert_f16_worker_loop(unsigned int n, unsigned int i, void *data)
static void quantize_f32_worker_loop(unsigned int n, unsigned int i, void *data) {
tiled_dequantize_state_t *state = (tiled_dequantize_state_t *)data;
struct htp_thread_trace * tr = state->traces ? &state->traces[i] : NULL;
struct htp_thread_trace * tr = &state->traces[i];
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_QUANT, i);
for (unsigned int task_id = i; task_id < (unsigned int)state->n_tasks; task_id += n) {
@@ -1920,7 +1921,7 @@ static void quantize_f32_worker_loop(unsigned int n, unsigned int i, void *data)
static void transfer_output_chunk_worker_fn(unsigned int n, unsigned int i, void *data) {
output_transfer_task_state_t *st = (output_transfer_task_state_t *) data;
struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL;
struct htp_thread_trace * tr = &st->traces[i];
int start_chunk_idx = i * st->n_chunks_per_task;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, start_chunk_idx);
@@ -1955,6 +1956,170 @@ typedef struct {
uint32_t dma_step_rows_shift;
} activation_transfer_task_state_t;
typedef struct {
__fp16 *dst;
const float *src;
uint32_t n_rows;
uint32_t k_block;
uint32_t k_stride;
uint32_t k_valid;
uint32_t n_col_chunks;
struct fastdiv_values n_threads_div;
float *vtcm_f32_act;
size_t vtcm_f32_act_bytes;
struct htp_thread_trace *traces;
struct htp_context *ctx;
uint32_t dma_step_rows;
uint32_t dma_step_rows_shift;
} activation_transfer_col_chunk_state_t;
static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk(
dma_queue *dma_q,
__fp16 *restrict vtcm_dst,
const float *restrict src,
uint32_t n_rows,
uint32_t k_block,
uint32_t k_stride,
uint32_t k_chunk_valid,
uint32_t c_first,
uint32_t c_len,
float *thread_f32_act,
struct htp_thread_trace *tr,
uint32_t dma_step_rows,
uint32_t dma_step_rows_shift) {
const uint32_t R = dma_step_rows;
const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS);
const uint32_t n_steps = n_rows_padded >> dma_step_rows_shift;
// Push step 0
if (n_steps > 0 && n_rows > 0) {
uint32_t nrows_to_fetch = hex_smin(n_rows, R);
dma_queue_push(dma_q, dma_make_ptr(thread_f32_act, src + c_first),
c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch);
}
// Push step 1
if (n_steps > 1) {
uint32_t next_r = R * 1;
if (next_r < n_rows) {
uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R);
const float *next_src = src + next_r * k_stride + c_first;
float *next_buf = thread_f32_act + 1 * R * c_len;
dma_queue_push(dma_q, dma_make_ptr(next_buf, next_src),
c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch);
}
}
for (uint32_t s = 0; s < n_steps; ++s) {
uint32_t r = s << dma_step_rows_shift;
float *curr_buf = thread_f32_act;
if (r < n_rows) {
curr_buf = (float *) dma_queue_pop(dma_q).dst;
}
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, r);
for (uint32_t p = 0; p < (R >> 1); ++p) {
uint32_t row_idx = r + (p << 1);
float *pair_buf = curr_buf + (p << 1) * c_len;
bool r0_valid = ((row_idx + 0) < n_rows);
bool r1_valid = ((row_idx + 1) < n_rows);
transfer_activation_row_pair_fp32_to_fp16_col_chunk(
vtcm_dst, pair_buf, pair_buf + c_len, row_idx, k_block, c_first, c_len, k_chunk_valid, r0_valid, r1_valid
);
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, r);
// Push step s + 2
uint32_t next_s = s + 2;
uint32_t next_r = next_s << dma_step_rows_shift;
if (next_r < n_rows) {
uint32_t nrows_to_fetch = hex_smin(n_rows - next_r, R);
const float *next_src = src + next_r * k_stride + c_first;
dma_queue_push(dma_q, dma_make_ptr(curr_buf, next_src),
c_len * sizeof(float), k_stride * sizeof(float), k_chunk_valid * sizeof(float), nrows_to_fetch);
}
}
}
static void transfer_activation_chunk_fp32_to_fp16_col_chunk(
__fp16 *restrict vtcm_dst,
const float *restrict src,
uint32_t n_rows,
uint32_t k_block,
uint32_t k_stride,
uint32_t c_first,
uint32_t c_len,
uint32_t k_chunk_valid) {
const uint32_t n_rows_padded = hex_align_up(n_rows, HTP_MM_HMX_TILE_N_ROWS);
const uint32_t n_rows_tiled = (n_rows / HTP_MM_HMX_TILE_N_ROWS) * HTP_MM_HMX_TILE_N_ROWS;
uint32_t r = 0;
#pragma unroll(2)
for (r = 0; r < n_rows_tiled; r += 2) {
const float *ptr_in0 = src + (r + 0) * k_stride + c_first;
const float *ptr_in1 = src + (r + 1) * k_stride + c_first;
transfer_activation_row_pair_fp32_to_fp16_col_chunk(
vtcm_dst, ptr_in0, ptr_in1, r, k_block, c_first, c_len, k_chunk_valid, true, true
);
}
for (; r < n_rows_padded; r += 2) {
const bool row0_valid = r < n_rows;
const bool row1_valid = (r + 1) < n_rows;
const float *ptr_in0 = row0_valid ? (src + (r + 0) * k_stride + c_first) : NULL;
const float *ptr_in1 = row1_valid ? (src + (r + 1) * k_stride + c_first) : NULL;
transfer_activation_row_pair_fp32_to_fp16_col_chunk(
vtcm_dst, ptr_in0, ptr_in1, r, k_block, c_first, c_len, k_chunk_valid, row0_valid, row1_valid
);
}
}
static void transfer_activation_chunk_col_chunk_worker_fn(unsigned int n, unsigned int i, void *data) {
activation_transfer_col_chunk_state_t *st = (activation_transfer_col_chunk_state_t *) data;
struct htp_thread_trace * tr = &st->traces[i];
uint32_t n_blocks = st->k_block / 32;
uint32_t b_first = fastdiv(n_blocks * i, &st->n_threads_div);
uint32_t b_last = fastdiv(n_blocks * (i + 1), &st->n_threads_div);
uint32_t c_first = b_first * 32;
uint32_t c_last = b_last * 32;
uint32_t c_len = c_last - c_first;
if (c_len == 0) {
return;
}
uint32_t k_chunk_valid = 0;
if (st->k_valid > c_first) {
k_chunk_valid = hex_smin(st->k_valid, c_last) - c_first;
}
__fp16 *dst = st->dst;
const float *src = st->src;
if (st->vtcm_f32_act) {
size_t thread_scratch_bytes = hex_align_down(fastdiv(st->vtcm_f32_act_bytes, &st->n_threads_div), 128);
float *thread_f32_act = (float *)((char *)st->vtcm_f32_act + i * thread_scratch_bytes);
transfer_activation_chunk_fp32_to_fp16_dma_pipelined_col_chunk(
st->ctx->dma[i], dst, src, st->n_rows, st->k_block, st->k_stride, k_chunk_valid,
c_first, c_len, thread_f32_act, tr, st->dma_step_rows, st->dma_step_rows_shift
);
} else {
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, c_first);
transfer_activation_chunk_fp32_to_fp16_col_chunk(
dst, src, st->n_rows, st->k_block, st->k_stride, c_first, c_len, k_chunk_valid
);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, c_first);
}
}
static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined(
dma_queue *dma_q,
__fp16 *restrict vtcm_dst,
@@ -2024,7 +2189,7 @@ static void transfer_activation_chunk_fp32_to_fp16_dma_pipelined(
static void transfer_activation_chunk_worker_fn(unsigned int n, unsigned int i, void *data) {
activation_transfer_task_state_t *st = (activation_transfer_task_state_t *) data;
struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL;
struct htp_thread_trace * tr = &st->traces[i];
for (unsigned int task_id = i; task_id < (unsigned int)st->n_tasks; task_id += n) {
int chunk_idx = task_id * st->n_chunks_per_task;
@@ -2085,15 +2250,16 @@ typedef struct {
static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigned int i, void *data) {
activation_transfer_gathered_task_state_t *st = data;
struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL;
struct htp_thread_trace * tr = &st->traces[i];
int chunk_idx = i;
int chunk_size = st->n_chunks_per_task;
int start_row = st->start_row + chunk_idx * chunk_size;
int vtcm_start_row = chunk_idx * chunk_size;
int start_row = st->start_row + vtcm_start_row;
int n_rows = hex_smin(st->cne1 - start_row, chunk_size);
if (n_rows > 0) {
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx);
transfer_activation_chunk_fp32_to_fp16_gathered(
st->dst, st->src, start_row, n_rows, st->k_block,
st->dst, st->src, start_row, vtcm_start_row, n_rows, st->k_block,
st->matrix_rows, st->cur_a, st->mapping_stride,
st->ne11, &st->ne11_div, st->nb11, st->nb12, st->cne1, st->k_valid);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx);
@@ -2102,15 +2268,16 @@ static void transfer_activation_chunk_gathered_worker_fn(unsigned int n, unsigne
static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, unsigned int i, void *data) {
activation_transfer_gathered_task_state_t *st = data;
struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL;
struct htp_thread_trace * tr = &st->traces[i];
int chunk_idx = i;
int chunk_size = st->n_chunks_per_task;
int start_row = st->start_row + chunk_idx * chunk_size;
int vtcm_start_row = chunk_idx * chunk_size;
int start_row = st->start_row + vtcm_start_row;
int n_rows = hex_smin(st->cne1 - start_row, chunk_size);
if (n_rows > 0) {
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx);
transfer_activation_chunk_fp32_to_fp16_gathered_flat(
st->dst, st->src, start_row, n_rows, st->k_block,
st->dst, st->src, start_row, vtcm_start_row, n_rows, st->k_block,
st->matrix_rows, st->cur_a, st->mapping_stride,
st->nb12, st->cne1, st->k_valid);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, chunk_idx);
@@ -2119,15 +2286,16 @@ static void transfer_activation_chunk_gathered_worker_flat_fn(unsigned int n, un
static void transfer_output_chunk_scattered_worker_fn(unsigned int n, unsigned int i, void *data) {
output_transfer_scattered_task_state_t *st = data;
struct htp_thread_trace * tr = st->traces ? &st->traces[i] : NULL;
struct htp_thread_trace * tr = &st->traces[i];
int chunk_idx = i;
int chunk_size = st->n_chunks_per_task;
int start_row = st->start_row + chunk_idx * chunk_size;
int vtcm_start_row = chunk_idx * chunk_size;
int start_row = st->start_row + vtcm_start_row;
int n_rows = hex_smin(st->cne1 - start_row, chunk_size);
if (n_rows > 0) {
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, chunk_idx);
transfer_output_chunk_fp16_to_fp32_scattered(
st->dst, st->vtcm_src, start_row, n_rows, st->n_cols,
st->dst, st->vtcm_src, start_row, vtcm_start_row, n_rows, st->n_cols,
st->matrix_rows, st->cur_a, st->mapping_stride,
st->dst_nb1, st->dst_nb2, st->cne1);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, chunk_idx);
@@ -2210,42 +2378,81 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst,
}
}
static void transfer_activation_chunk_threaded(
struct htp_context *ctx,
__fp16 *dst,
const float *src,
int n_rows,
int k_block,
int k_stride,
int n_threads,
int k_valid,
float *vtcm_f32_act,
size_t vtcm_f32_act_bytes) {
struct activation_transfer_params {
struct htp_context * ctx;
__fp16 * dst;
const float * src;
int n_rows;
int k_block;
int k_stride;
int n_threads;
const struct fastdiv_values * act_threads_div;
const struct fastdiv_values * k_div;
int k_valid;
float * vtcm_f32_act;
size_t vtcm_f32_act_bytes;
};
static void transfer_activation_chunk_threaded(const struct activation_transfer_params * params) {
struct htp_context * ctx = params->ctx;
__fp16 * dst = params->dst;
const float * src = params->src;
int n_rows = params->n_rows;
int k_block = params->k_block;
int k_stride = params->k_stride;
int n_threads = params->n_threads;
const struct fastdiv_values * act_threads_div = params->act_threads_div;
const struct fastdiv_values * k_div = params->k_div;
int k_valid = params->k_valid;
float * vtcm_f32_act = params->vtcm_f32_act;
size_t vtcm_f32_act_bytes = params->vtcm_f32_act_bytes;
if (n_rows <= 0) {
return;
}
const size_t n_tasks = (n_rows + 31) >> 5;
if (n_threads > 1 && k_block > 32 && n_tasks < (size_t) n_threads) {
// Calculate step rows parameters for column-chunked dma pipelining
uint32_t dma_step_rows = 2;
uint32_t dma_step_rows_shift = 1;
if (vtcm_f32_act && vtcm_f32_act_bytes > 0 && k_block > 0) {
size_t thread_scratch_bytes = hex_align_down(fastdiv(vtcm_f32_act_bytes, act_threads_div), 128);
size_t thread_scratch_elements = thread_scratch_bytes / sizeof(float);
size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div);
if (dma_step_rows_max >= 4) {
dma_step_rows = 4;
dma_step_rows_shift = 2;
}
}
activation_transfer_col_chunk_state_t col_state;
col_state.dst = dst;
col_state.src = src;
col_state.n_rows = n_rows;
col_state.k_block = k_block;
col_state.k_stride = k_stride;
col_state.k_valid = k_valid;
col_state.n_col_chunks = n_threads;
col_state.n_threads_div = *act_threads_div;
col_state.vtcm_f32_act = vtcm_f32_act;
col_state.vtcm_f32_act_bytes = vtcm_f32_act_bytes;
col_state.traces = ctx->trace;
col_state.ctx = ctx;
col_state.dma_step_rows = dma_step_rows;
col_state.dma_step_rows_shift = dma_step_rows_shift;
worker_pool_run_func(ctx->worker_pool, transfer_activation_chunk_col_chunk_worker_fn, &col_state, n_threads);
return;
}
assert(k_block % HTP_MM_HMX_TILE_N_COLS == 0 && k_stride % HTP_MM_HMX_TILE_N_COLS == 0);
size_t n_tot_chunks = n_rows;
size_t n_chunks_per_task = (n_threads == 1) ? n_tot_chunks : 32; // must be multiple of 32 to ensure correct destination address
uint32_t dma_step_rows = 2;
uint32_t dma_step_rows_shift = 1;
if (vtcm_f32_act && vtcm_f32_act_bytes > 0 && k_block > 0) {
size_t thread_scratch_elements = vtcm_f32_act_bytes / (n_threads * sizeof(float));
size_t dma_step_rows_max = (thread_scratch_elements / 2) / k_block;
if (dma_step_rows_max >= 4) {
dma_step_rows = 4;
dma_step_rows_shift = 2;
} else {
dma_step_rows = 2;
dma_step_rows_shift = 1;
}
}
activation_transfer_task_state_t state;
state.n_tasks = (n_tot_chunks + n_chunks_per_task - 1) / n_chunks_per_task;
state.n_tasks = (n_threads == 1) ? 1 : hmx_ceil_div(n_tot_chunks, 32);
state.n_tot_chunks = n_tot_chunks;
state.n_chunks_per_task = n_chunks_per_task;
state.dst = dst;
@@ -2258,7 +2465,18 @@ static void transfer_activation_chunk_threaded(
state.vtcm_f32_act = vtcm_f32_act;
int active_threads = hex_smin(n_threads, (int)state.n_tasks);
state.vtcm_f32_act_bytes_per_thread = (vtcm_f32_act_bytes / active_threads) & ~127u;
state.vtcm_f32_act_bytes_per_thread = hex_align_down(vtcm_f32_act_bytes / active_threads, 128);
uint32_t dma_step_rows = 2;
uint32_t dma_step_rows_shift = 1;
if (vtcm_f32_act && state.vtcm_f32_act_bytes_per_thread > 0 && k_block > 0) {
size_t thread_scratch_elements = state.vtcm_f32_act_bytes_per_thread / sizeof(float);
size_t dma_step_rows_max = fastdiv(thread_scratch_elements / 2, k_div);
if (dma_step_rows_max >= 4) {
dma_step_rows = 4;
dma_step_rows_shift = 2;
}
}
state.dma_step_rows = dma_step_rows;
state.dma_step_rows_shift = dma_step_rows_shift;
@@ -2321,9 +2539,14 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
int pipeline,
int n_threads,
int act_threads,
const struct fastdiv_values * act_threads_div,
const struct fastdiv_values * k_div,
int tile_size,
int aligned_tile_size,
int vtcm_size) {
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
if (k % 32 != 0 || n % 32 != 0) { return -1; }
if (!hex_is_aligned(dst, VLEN) || !hex_is_aligned(activation, VLEN)) { return -1; }
@@ -2393,6 +2616,8 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
int n_chunk_cnt = hmx_ceil_div(n, n_chunk_n_cols);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
if (pipeline) {
// --- Asynchronous Pipelined Loop ---
hmx_matmul_job_t job_slots[2]; // persistent double-buffered job descriptors
@@ -2403,7 +2628,21 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
void *vtcm_weight_bufs[2] = { vtcm_scratch0, vtcm_scratch1 };
void *vtcm_output_bufs[2] = { vtcm_output, vtcm_scratch2 };
transfer_activation_chunk_threaded(ctx, vtcm_f16_act, activation + mr * act_stride, n_rows, k, act_stride, act_threads, k_valid, vtcm_f32_act, L.act_f32_bytes);
struct activation_transfer_params act_params = {
.ctx = ctx,
.dst = vtcm_f16_act,
.src = activation + mr * act_stride,
.n_rows = (int) n_rows,
.k_block = k,
.k_stride = act_stride,
.n_threads = act_threads,
.act_threads_div = act_threads_div,
.k_div = k_div,
.k_valid = k_valid,
.vtcm_f32_act = vtcm_f32_act,
.vtcm_f32_act_bytes = L.act_f32_bytes,
};
transfer_activation_chunk_threaded(&act_params);
// Prologue: push A0 and optionally A1 (if n_chunk_cnt > 1)
const size_t n_cols_A0 = hex_smin(n - 0 * n_chunk_n_cols, n_chunk_n_cols);
@@ -2480,7 +2719,21 @@ static int hmx_mm_2d_f32(struct htp_context *ctx,
for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) {
const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows);
transfer_activation_chunk_threaded(ctx, vtcm_f16_act, activation + mr * act_stride, n_rows, k, act_stride, act_threads, k_valid, vtcm_f32_act, L.act_f32_bytes);
struct activation_transfer_params act_params = {
.ctx = ctx,
.dst = vtcm_f16_act,
.src = activation + mr * act_stride,
.n_rows = (int) n_rows,
.k_block = k,
.k_stride = act_stride,
.n_threads = act_threads,
.act_threads_div = act_threads_div,
.k_div = k_div,
.k_valid = k_valid,
.vtcm_f32_act = vtcm_f32_act,
.vtcm_f32_act_bytes = L.act_f32_bytes,
};
transfer_activation_chunk_threaded(&act_params);
// A0: Pre-fetch the first weight chunk (nc = 0)
if (n > 0) {
@@ -2570,7 +2823,8 @@ static inline const float *hmx_mm_src2_batch_ptr(const hmx_mm_f16_f32_batched_pa
static int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx,
const hmx_mm_f16_f32_batched_params_t *params,
int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size) {
int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size,
const struct fastdiv_values * act_threads_div, const struct fastdiv_values * k_div) {
int ret = 0;
for (int b3 = 0; b3 < params->ne13 && ret == 0; ++b3) {
for (int b2 = 0; b2 < params->ne12 && ret == 0; ++b2) {
@@ -2582,14 +2836,17 @@ static int hmx_mm_f16_f32_batched_simple(struct htp_context *ctx,
params->act_stride, params->weight_stride * (int)sizeof(__fp16),
HTP_TYPE_F16, params->k, params->dst_stride, params->src2_stride, params->n,
m_chunk, n_chunk, pipeline, n_threads, act_threads,
0, 0, vtcm_size);
act_threads_div, k_div, 0, 0, vtcm_size);
}
}
return ret;
}
static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_batched_params_t *params,
int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads, int vtcm_size) {
int m_chunk, int n_chunk, int pipeline, int n_threads, int act_threads,
const struct fastdiv_values * act_threads_div,
const struct fastdiv_values * k_div,
int vtcm_size) {
if (params->act_stride < params->k || params->weight_stride < params->k || params->dst_stride < params->n) { return -1; }
if (params->ne02 <= 0 || params->ne03 <= 0 || params->ne12 <= 0 || params->ne13 <= 0) { return -1; }
if (params->ne12 % params->ne02 != 0 || params->ne13 % params->ne03 != 0) { return -1; }
@@ -2604,9 +2861,12 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_
// Grouped path is only valid if group_size > 1 and it fits within VTCM budget.
bool run_grouped = (group_size > 1 && (size_t)vtcm_size <= vtcm_budget);
if (!run_grouped) {
return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size);
return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div);
}
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
const size_t vec_dot_size = params->k * sizeof(__fp16);
const bool use_dma_activation = (params->act_stride > params->k);
@@ -2622,7 +2882,8 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_
if (L.total_bytes > vtcm_budget) {
FARF(HIGH, "%s: grouped layout overflowed VTCM, falling back to simple batched loop", __func__);
return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
return hmx_mm_f16_f32_batched_simple(ctx, params, m_chunk, n_chunk, pipeline, n_threads, act_threads, vtcm_size, act_threads_div, k_div);
}
uint8_t * const base = (uint8_t *) ctx->vtcm_base;
@@ -2644,6 +2905,8 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_
const size_t fp16_row_bytes = (size_t) params->k * sizeof(__fp16);
const size_t weight_row_bytes = (size_t) params->weight_stride * sizeof(__fp16);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
hmx_matmul_job_t job;
for (int b3 = 0; b3 < params->ne13; ++b3) {
@@ -2662,9 +2925,21 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_
for (int g = 0; g < group_size; ++g) {
const float *activation_chunk = hmx_mm_activation_batch_ptr(params, b2_base + g, b3) + mr * params->act_stride;
__fp16 *vtcm_act_g = vtcm_f16_act + (size_t) g * L.act_head_stride;
transfer_activation_chunk_threaded(ctx, vtcm_act_g,
activation_chunk, (int) n_rows,
params->k, params->act_stride, act_threads, params->k, vtcm_f32_act, L.act_f32_bytes);
struct activation_transfer_params act_params = {
.ctx = ctx,
.dst = vtcm_act_g,
.src = activation_chunk,
.n_rows = (int) n_rows,
.k_block = params->k,
.k_stride = params->act_stride,
.n_threads = act_threads,
.act_threads_div = act_threads_div,
.k_div = k_div,
.k_valid = params->k,
.vtcm_f32_act = vtcm_f32_act,
.vtcm_f32_act_bytes = L.act_f32_bytes,
};
transfer_activation_chunk_threaded(&act_params);
}
// Prologue: Push A0 and A1 (if exists)
@@ -2835,6 +3110,9 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
const struct mmid_row_mapping *matrix_rows,
int cur_a,
int mapping_stride) {
struct htp_thread_trace * tr = &ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
const int cne1 = m;
const int m_padded = hex_align_up(m, 32);
@@ -2913,6 +3191,8 @@ static int hmx_mm_id_2d_f32(struct htp_context *ctx,
hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00));
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
hmx_matmul_job_t job;
for (size_t mr = 0; mr < (size_t) m_padded; mr += m_chunk_n_rows) {
@@ -2980,10 +3260,6 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k
const int act_stride = (int)(src1->nb[1] / sizeof(float));
const int wgt_stride = (int)(src0->nb[1] / sizeof(__fp16));
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
return HTP_STATUS_OK;
}
const float * src2_ptr = NULL;
uint32_t src2_stride = 0;
size_t src2_nb2 = 0;
@@ -3027,6 +3303,8 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k
kparams->m_chunk, kparams->n_chunk,
kparams->pipeline, n_threads,
kparams->n_act_threads,
&kparams->div_n_act_threads,
&kparams->div_ne00_padded,
kparams->vtcm_size);
} else {
ret = hmx_mm_2d_f32(
@@ -3035,6 +3313,8 @@ static int hmx_mm_op_matmul(struct htp_ops_context * octx, const struct htp_mm_k
(int)(dst->nb[1] / sizeof(float)), src2_stride, (int)dst->ne[0],
kparams->m_chunk, kparams->n_chunk, kparams->pipeline, n_threads,
kparams->n_act_threads,
&kparams->div_n_act_threads,
&kparams->div_ne00_padded,
kparams->tile_size, kparams->aligned_tile_size, kparams->vtcm_size
);
}
@@ -3103,6 +3383,10 @@ static int hvx_mm_matmul_id(
bool must_free_mapping
) {
htp_matmul_tensors_preamble;
struct htp_thread_trace * tr = &octx->ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
const struct htp_mm_kernel_params * kparams = (const struct htp_mm_kernel_params *) octx->kernel_params;
const struct htp_tensor * restrict ids = octx->src[2];
const size_t src0_row_size = nb01;
@@ -3175,8 +3459,9 @@ static int hvx_mm_matmul_id(
mmctx->n_quant_tasks = n_quant_tasks;
atomic_init(&mmctx->quant_barrier, n_quant_tasks);
const uint32_t n_matmul_jobs = octx->n_threads;
worker_pool_run_func(octx->ctx->worker_pool, matmul_id_job_func, mmctx, n_matmul_jobs);
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
worker_pool_run_func(octx->ctx->worker_pool, matmul_id_job_func, mmctx, octx->n_threads);
if (must_free_mapping) free(mapping_buf);
return HTP_STATUS_OK;
@@ -3185,6 +3470,9 @@ static int hvx_mm_matmul_id(
int op_matmul_id(struct htp_ops_context * octx) {
htp_matmul_tensors_preamble;
struct htp_thread_trace * tr = &octx->ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
struct htp_mm_context mmctx_struct = {0};
struct htp_mm_context * mmctx = &mmctx_struct;
mmctx->octx = octx;
@@ -3262,10 +3550,7 @@ int op_matmul_id(struct htp_ops_context * octx) {
}
}
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE) {
if (must_free_mapping) free(mapping_buf);
return HTP_STATUS_OK;
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
if (kparams->n_hmx) {
return hmx_mm_op_matmul_id(octx, mmctx, matrix_row_counts, matrix_rows, mapping_buf, must_free_mapping);
@@ -3275,6 +3560,9 @@ int op_matmul_id(struct htp_ops_context * octx) {
}
int op_matmul_qkv(struct htp_ops_context * octx) {
struct htp_thread_trace * tr = &octx->ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
const struct htp_tensor * restrict src0 = octx->src[0]; // Wk
const struct htp_tensor * restrict src1 = octx->src[1]; // x
const struct htp_tensor * restrict src2 = octx->src[2]; // Wv
@@ -3379,9 +3667,6 @@ int op_matmul_qkv(struct htp_ops_context * octx) {
mmctx->vtcm_src3_size_per_thread = L.src3_bytes / octx->n_threads;
mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads;
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)
return HTP_STATUS_OK;
mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks;
mmctx->quant_task_func = quant_task_func;
mmctx->n_quant_tasks = n_quant_tasks;
@@ -3413,12 +3698,18 @@ int op_matmul_qkv(struct htp_ops_context * octx) {
} else {
matmul_job_func = hvx_mm_qkv_2d;
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs);
return HTP_STATUS_OK;
}
int op_matmul_ffn(struct htp_ops_context * octx) {
struct htp_thread_trace * tr = &octx->ctx->trace[0];
htp_trace_event_start(tr, HTP_TRACE_EVT_INIT, 0);
const struct htp_tensor * restrict src0 = octx->src[0]; // Wgate
const struct htp_tensor * restrict src1 = octx->src[1]; // y
const struct htp_tensor * restrict src2 = octx->src[2]; // Wup
@@ -3516,9 +3807,6 @@ int op_matmul_ffn(struct htp_ops_context * octx) {
mmctx->vtcm_src2_size_per_thread = L.src2_bytes / octx->n_threads;
mmctx->vtcm_dst_size_per_thread = L.dst_bytes / octx->n_threads;
if (octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)
return HTP_STATUS_OK;
mmctx->n_quant_rows_per_thread = (src1_nrows + n_quant_tasks - 1) / n_quant_tasks;
mmctx->quant_task_func = quant_task_func;
mmctx->n_quant_tasks = n_quant_tasks;
@@ -3550,6 +3838,9 @@ int op_matmul_ffn(struct htp_ops_context * octx) {
} else {
matmul_job_func = hvx_mm_ffn_2d;
}
htp_trace_event_stop(tr, HTP_TRACE_EVT_INIT, 0);
worker_pool_run_func(octx->ctx->worker_pool, matmul_job_func, mmctx, n_matmul_jobs);
return HTP_STATUS_OK;

132
ggml/src/ggml-hexagon/htp/matmul-ops.h
View File

@@ -95,6 +95,8 @@ struct htp_mm_kernel_params {
struct fastdiv_values div_r2;
struct fastdiv_values div_r3;
struct fastdiv_values div_ne11;
struct fastdiv_values div_n_act_threads;
struct fastdiv_values div_ne00_padded;
};
#if defined(__cplusplus)
@@ -643,6 +645,136 @@ static inline size_t htp_mm_hmx_get_batched_vtcm_size(
return L.total_bytes;
}
static inline bool htp_mm_hmx_solve_batched_params(
int wtype,
uint32_t k,
uint32_t ne01_padded,
uint32_t ne11,
uint32_t group_size,
bool use_dma_activation,
int n_threads,
bool pipeline,
size_t vtcm_budget,
size_t * m_chunk_out,
size_t * n_chunk_out,
int * act_threads_out,
size_t * vtcm_size_out
) {
size_t best_mblocks = SIZE_MAX;
int best_act_threads = 0;
size_t best_m_chunk = 0;
size_t best_n_chunk = 0;
size_t best_vtcm_size = 0;
int act_threads = n_threads;
while (act_threads >= 1) {
size_t group_overhead = 256;
size_t group_size_per_n, group_size_per_m, group_size_per_mn;
htp_mm_hmx_get_batched_chunk_costs(k, group_size, &group_size_per_n, &group_size_per_m, &group_size_per_mn);
size_t m_chunk_candidate = 0;
size_t n_chunk_candidate = 0;
size_t vtcm_size_candidate = 0;
if (htp_mm_hmx_compute_chunks(vtcm_budget, group_overhead, group_size_per_n, group_size_per_m, group_size_per_mn, hex_align_up(ne11, 32), ne01_padded,
(size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) ne11 * HTP_MM_HMX_COST_A_CONVERT,
&m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) {
size_t exact_size = htp_mm_hmx_get_batched_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, group_size, use_dma_activation, pipeline, act_threads);
if (exact_size <= vtcm_budget) {
size_t mblocks = ((size_t) ne11 + m_chunk_candidate - 1) / m_chunk_candidate;
if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) {
best_mblocks = mblocks;
best_act_threads = act_threads;
best_m_chunk = m_chunk_candidate;
best_n_chunk = n_chunk_candidate;
best_vtcm_size = exact_size;
}
}
}
if (act_threads == 1) {
act_threads = 0;
} else {
act_threads /= 2;
}
}
if (best_act_threads > 0) {
*m_chunk_out = best_m_chunk;
*n_chunk_out = best_n_chunk;
*vtcm_size_out = best_vtcm_size;
*act_threads_out = best_act_threads;
return true;
}
return false;
}
static inline bool htp_mm_hmx_solve_2d_params(
int wtype,
uint32_t k,
uint32_t m_id_rows,
uint32_t ne01_padded,
uint32_t ne11_padded,
uint32_t m_for_cost,
int n_threads,
bool pipeline,
bool is_matmul_id,
uint32_t aligned_tile_size,
size_t vtcm_budget,
size_t * m_chunk_out,
size_t * n_chunk_out,
int * act_threads_out,
size_t * vtcm_size_out
) {
size_t best_mblocks = SIZE_MAX;
int best_act_threads = 0;
size_t best_m_chunk = 0;
size_t best_n_chunk = 0;
size_t best_vtcm_size = 0;
const int m_for_chunks = is_matmul_id ? hex_align_up(m_id_rows, 32) : ne11_padded;
int act_threads = n_threads;
while (act_threads >= 1) {
size_t simple_2d_overhead = 256;
size_t simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn;
htp_mm_hmx_get_2d_chunk_costs(wtype, k, pipeline, aligned_tile_size, &simple_2d_size_per_n, &simple_2d_size_per_m, &simple_2d_size_per_mn);
size_t m_chunk_candidate = 0;
size_t n_chunk_candidate = 0;
size_t vtcm_size_candidate = 0;
if (htp_mm_hmx_compute_chunks(vtcm_budget, simple_2d_overhead, simple_2d_size_per_n, simple_2d_size_per_m, simple_2d_size_per_mn, m_for_chunks, ne01_padded,
(size_t) ne01_padded * HTP_MM_HMX_COST_W_DEQUANT, (size_t) m_for_cost * HTP_MM_HMX_COST_A_CONVERT,
&m_chunk_candidate, &n_chunk_candidate, &vtcm_size_candidate) == 0) {
size_t exact_size = htp_mm_hmx_get_2d_vtcm_size(wtype, k, m_chunk_candidate, n_chunk_candidate, pipeline, is_matmul_id ? 0 : act_threads, aligned_tile_size);
if (exact_size <= vtcm_budget) {
size_t mblocks = ((size_t) m_for_cost + m_chunk_candidate - 1) / m_chunk_candidate;
if (mblocks < best_mblocks || (mblocks == best_mblocks && act_threads > best_act_threads)) {
best_mblocks = mblocks;
best_act_threads = act_threads;
best_m_chunk = m_chunk_candidate;
best_n_chunk = n_chunk_candidate;
best_vtcm_size = exact_size;
}
}
}
if (act_threads == 1) {
act_threads = 0;
} else {
act_threads /= 2;
}
}
if (best_act_threads > 0) {
*m_chunk_out = best_m_chunk;
*n_chunk_out = best_n_chunk;
*vtcm_size_out = best_vtcm_size;
*act_threads_out = best_act_threads;
return true;
}
return false;
}
#ifdef __cplusplus
}
#endif

11
ggml/src/ggml-hexagon/htp/rope-ops.c
View File

@@ -18,6 +18,7 @@
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-ops.h"
#include "htp-tensor.h"
// Redefined the rope type constants as we can't include ggml.h
#define HTP_ROPE_TYPE_NORMAL 0
@@ -712,17 +713,11 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) {
}
int op_rope(struct htp_ops_context * octx) {
int err = HTP_STATUS_OK;
switch (octx->src[0]->type) {
case HTP_TYPE_F32:
err = execute_op_rope_f32(octx);
break;
return execute_op_rope_f32(octx);
default:
err = HTP_STATUS_NO_SUPPORT;
break;
return HTP_STATUS_NO_SUPPORT;
}
return err;
}

15
ggml/src/ggml-hexagon/htp/unary-ops.c
View File

@@ -19,6 +19,7 @@
#include "ggml-common.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "htp-vtcm.h"
#include "hex-profile.h"
@@ -397,7 +398,7 @@ static void unary_task_f32_##NAME(unsigned int nth, unsigned int ith, void * dat
struct htp_ops_context * octx = uctx->octx; \
const struct htp_tensor * src = octx->src[0]; \
const struct htp_tensor * dst = octx->dst; \
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
\
htp_unary_preamble; \
\
@@ -558,7 +559,7 @@ static void unary_task_f32_tiled_##NAME(unsigned int nth, unsigned int ith, void
struct htp_ops_context * octx = uctx->octx; \
const struct htp_tensor * src = octx->src[0]; \
const struct htp_tensor * dst = octx->dst; \
struct htp_thread_trace * tr = octx->ctx ? &octx->ctx->trace[ith] : NULL; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
\
htp_unary_preamble; \
\
@@ -922,17 +923,11 @@ static int execute_op_unary_f32(struct htp_ops_context * octx) {
}
int op_unary(struct htp_ops_context * octx) {
int err = HTP_STATUS_OK;
switch (octx->src[0]->type) {
case HTP_TYPE_F32:
err = execute_op_unary_f32(octx);
break;
return execute_op_unary_f32(octx);
default:
err = HTP_STATUS_NO_SUPPORT;
break;
return HTP_STATUS_NO_SUPPORT;
}
return err;
}

244
ggml/src/ggml-hexagon/htp/work-queue.c Normal file
View File

@@ -0,0 +1,244 @@
#include "work-queue.h"
#include "hex-utils.h"
#include <qurt.h>
#include <qurt_hvx.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "HAP_farf.h"
#define LOWEST_USABLE_QURT_PRIO (254)
// internal structure kept in thread-local storage per instance of work queue
typedef struct {
work_queue_t queue;
unsigned int id;
} worker_context_t;
struct work_queue_task_s {
work_queue_func_t func;
void * data;
unsigned int n_threads;
atomic_uint barrier;
};
// internal structure kept in thread-local storage per instance of work queue
struct work_queue_s {
atomic_uint seqn; // seqno used to detect new jobs
atomic_uint idx_read; // Updated by producer (pop/reclaim)
unsigned int idx_write; // Updated by producer (push)
uint32_t idx_mask;
uint32_t capacity;
qurt_thread_t thread[WORK_QUEUE_MAX_N_THREADS]; // thread ID's of the workers
worker_context_t context[WORK_QUEUE_MAX_N_THREADS]; // worker contexts
void * stack[WORK_QUEUE_MAX_N_THREADS]; // thread stack pointers
unsigned int n_threads; // total threads (workers + main)
unsigned int n_workers; // number of active threads (just workers)
atomic_bool active; // workers are polling/active
atomic_bool killed; // threads need to exit
bool external_mem; // memory owned externally
struct work_queue_task_s queue[] __attribute__((aligned(HEX_L2_LINE_SIZE)));
};
static void work_queue_thread(void * context) {
worker_context_t * me = (worker_context_t *) context;
work_queue_t q = me->queue;
FARF(HIGH, "work-queue: thread %u started", me->id);
unsigned int prev_seqn = 0;
while (!atomic_load_explicit(&q->killed, memory_order_relaxed)) {
unsigned int seqn = atomic_load_explicit(&q->seqn, memory_order_acquire);
if (seqn == prev_seqn) {
if (atomic_load_explicit(&q->active, memory_order_relaxed)) {
hex_pause();
} else {
qurt_futex_wait(&q->seqn, prev_seqn);
}
continue;
}
prev_seqn = seqn;
// Process all active tasks in the queue
unsigned int ir = atomic_load_explicit(&q->idx_read, memory_order_relaxed);
unsigned int iw = q->idx_write;
while (ir != iw) {
struct work_queue_task_s * task = &q->queue[ir];
unsigned int n = task->n_threads;
unsigned int i = me->id;
if (i < n) {
task->func(n, i, task->data);
atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release);
} else {
while (atomic_load_explicit(&task->barrier, memory_order_relaxed) > 0) {
hex_pause();
}
}
ir = (ir + 1) & q->idx_mask;
}
}
FARF(HIGH, "work-queue: thread %u stopped", me->id);
}
bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) {
if (n > q->n_threads) {
FARF(ERROR, "work-queue: invalid number of jobs %u for n-threads %u", n, q->n_threads);
return false;
}
unsigned int ir = atomic_load_explicit(&q->idx_read, memory_order_relaxed);
unsigned int iw = q->idx_write;
if (((iw + 1) & q->idx_mask) == ir) {
FARF(ERROR, "work-queue-push: queue is full\n");
return false;
}
struct work_queue_task_s * task = &q->queue[iw];
task->func = func;
task->data = data;
task->n_threads = n;
atomic_store_explicit(&task->barrier, n, memory_order_relaxed);
q->idx_write = (iw + 1) & q->idx_mask;
// publish job to workers (already awake and polling)
atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release);
// main thread runs job #0
func(n, 0, data);
atomic_fetch_sub_explicit(&task->barrier, 1, memory_order_release);
while (atomic_load_explicit(&task->barrier, memory_order_relaxed) > 0) {
hex_pause();
}
atomic_thread_fence(memory_order_acquire);
atomic_store_explicit(&q->idx_read, (ir + 1) & q->idx_mask, memory_order_relaxed);
return true;
}
size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size) {
capacity = hex_ceil_pow2(capacity);
uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0;
size_t size_stacks = stack_size * n_workers;
size_t size_q = hex_align_up(sizeof(struct work_queue_s) + capacity * sizeof(struct work_queue_task_s), HEX_L2_LINE_SIZE);
return size_stacks + size_q;
}
size_t work_queue_alignof(void) {
return 4096;
}
work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size) {
capacity = hex_ceil_pow2(capacity);
uint32_t n_workers = n_threads > 1 ? n_threads - 1 : 0;
unsigned char * mem_blob = (unsigned char *) ptr;
work_queue_t q = (work_queue_t) (mem_blob + stack_size * n_workers);
memset(q, 0, sizeof(struct work_queue_s) + capacity * sizeof(struct work_queue_task_s));
q->n_threads = n_threads;
q->n_workers = n_workers;
q->external_mem = true;
q->capacity = capacity;
for (unsigned int i = 0; i < n_workers; i++) {
q->stack[i] = mem_blob; mem_blob += stack_size;
q->thread[i] = 0;
q->context[i].id = i + 1;
q->context[i].queue = q;
}
atomic_init(&q->idx_read, 0);
atomic_init(&q->seqn, 0);
atomic_init(&q->active, false);
q->idx_write = 0;
q->idx_mask = capacity - 1;
q->killed = 0;
for (int i = 0; i < (int) capacity; i++) {
atomic_init(&q->queue[i].barrier, 0);
q->queue[i].func = NULL;
q->queue[i].data = NULL;
q->queue[i].n_threads = 0;
}
// launch the workers
qurt_thread_attr_t attr;
qurt_thread_attr_init(&attr);
for (unsigned int i = 0; i < n_workers; i++) {
qurt_thread_attr_set_stack_addr(&attr, q->stack[i]);
qurt_thread_attr_set_stack_size(&attr, stack_size);
char thread_name[32];
snprintf(thread_name, sizeof(thread_name), "work-queue:%u", i);
qurt_thread_attr_set_name(&attr, thread_name);
// set up priority - by default, match the creating thread's prio
int prio = qurt_thread_get_priority(qurt_thread_get_id());
if (prio < 1) {
prio = 1;
}
if (prio > LOWEST_USABLE_QURT_PRIO) {
prio = LOWEST_USABLE_QURT_PRIO;
}
qurt_thread_attr_set_priority(&attr, prio);
int err = qurt_thread_create(&q->thread[i], &attr, work_queue_thread, (void *) &q->context[i]);
if (err) {
FARF(ERROR, "Could not launch worker threads!");
work_queue_free(q);
return NULL;
}
}
return q;
}
void work_queue_free(work_queue_t q) {
if (!q) { return; }
atomic_store_explicit(&q->killed, 1, memory_order_relaxed);
atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release);
qurt_futex_wake(&q->seqn, q->n_workers);
for (unsigned int i = 0; i < q->n_workers; i++) {
if (q->thread[i]) {
int status;
(void) qurt_thread_join(q->thread[i], &status);
}
}
}
void work_queue_wakeup(work_queue_t q) {
if (!atomic_load_explicit(&q->active, memory_order_relaxed)) {
atomic_store_explicit(&q->active, true, memory_order_release);
// Increment seqn and wake workers to transition them out of sleep
atomic_fetch_add_explicit(&q->seqn, 1, memory_order_release);
qurt_futex_wake(&q->seqn, q->n_workers);
}
}
void work_queue_suspend(work_queue_t q) {
atomic_store_explicit(&q->active, false, memory_order_release);
}

38
ggml/src/ggml-hexagon/htp/work-queue.h Normal file
View File

@@ -0,0 +1,38 @@
#ifndef HTP_WORK_QUEUE_H
#define HTP_WORK_QUEUE_H
#include <stdbool.h>
#include <stdint.h>
#include <stddef.h>
typedef void (*work_queue_func_t)(unsigned int n, unsigned int i, void *);
struct work_queue_s;
typedef struct work_queue_s * work_queue_t;
#define WORK_QUEUE_MAX_N_THREADS 10
size_t work_queue_sizeof(uint32_t n_threads, uint32_t capacity, uint32_t stack_size);
size_t work_queue_alignof(void);
work_queue_t work_queue_init(void * ptr, uint32_t n_threads, uint32_t capacity, uint32_t stack_size);
void work_queue_free(work_queue_t q);
void work_queue_wakeup(work_queue_t q);
void work_queue_suspend(work_queue_t q);
bool work_queue_run_async(work_queue_t q, work_queue_func_t func, void * data, unsigned int n);
static inline bool work_queue_run(work_queue_t q, work_queue_func_t func, void * data, unsigned int n) {
if (n <= 1) {
func(n, 0, data);
return true;
}
return work_queue_run_async(q, func, data, n);
}
// Legacy compatibility
typedef work_queue_func_t worker_callback_t;
#define worker_pool_run_func work_queue_run
#define worker_pool work_queue
#endif // #ifndef HTP_WORK_QUEUE_H

305
ggml/src/ggml-hexagon/htp/worker-pool.c
View File

@@ -1,305 +0,0 @@
#include "worker-pool.h"
#include "hex-utils.h"
#include <qurt.h>
#include <qurt_hvx.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "HAP_farf.h"
#define LOWEST_USABLE_QURT_PRIO (254)
struct worker_pool_s;
// internal structure kept in thread-local storage per instance of worker pool
typedef struct {
struct worker_pool_s * pool;
unsigned int id;
} worker_context_t;
// internal structure kept in thread-local storage per instance of worker pool
typedef struct worker_pool_s {
worker_pool_job_t job[MAX_NUM_WORKERS]; // list of job descriptors
qurt_thread_t thread[MAX_NUM_WORKERS]; // thread ID's of the workers
worker_context_t context[MAX_NUM_WORKERS]; // worker contexts
void * stack[MAX_NUM_WORKERS]; // thread stack pointers
unsigned int n_threads; // number of workers in this pool
atomic_uint seqn; // seqno used to detect new jobs
atomic_uint next_job; // next job index
atomic_uint n_pending; // number of pending jobs
atomic_uint n_jobs; // number of current jobs
atomic_bool killed; // threads need to exit
} worker_pool_t;
static void worker_pool_main(void * context) {
worker_context_t * me = (worker_context_t *) context;
worker_pool_t * pool = me->pool;
FARF(HIGH, "worker-pool: thread %u started", me->id);
unsigned int prev_seqn = 0;
unsigned int poll_cnt = WORKER_POOL_POLL_COUNT;
while (!atomic_load(&pool->killed)) {
unsigned int seqn = atomic_load(&pool->seqn);
if (seqn == prev_seqn) {
// drop HVX context while spinning
if (poll_cnt > 1 && poll_cnt == WORKER_POOL_POLL_COUNT) {
qurt_hvx_unlock();
}
if (--poll_cnt) {
hex_pause();
continue;
}
qurt_futex_wait(&pool->seqn, prev_seqn);
poll_cnt = WORKER_POOL_POLL_COUNT;
continue;
}
prev_seqn = seqn;
poll_cnt = WORKER_POOL_POLL_COUNT;
// New job
unsigned int n = atomic_load(&pool->n_jobs);
unsigned int i = atomic_fetch_add(&pool->next_job, 1);
if (i >= n) {
// Spurious wakeup
continue;
}
pool->job[i].func(n, i, pool->job[i].data);
atomic_fetch_sub(&pool->n_pending, 1);
}
FARF(HIGH, "worker-pool: thread %u stopped", me->id);
}
AEEResult worker_pool_init_with_stack_size(worker_pool_context_t * context, uint32_t n_threads, uint32_t stack_size) {
int err = 0;
if (NULL == context) {
FARF(ERROR, "NULL context passed to worker_pool_init().");
return AEE_EBADPARM;
}
// Allocations
int size = (stack_size * n_threads) + (sizeof(worker_pool_t));
unsigned char * mem_blob = (unsigned char *) malloc(size);
if (!mem_blob) {
FARF(ERROR, "Could not allocate memory for worker pool!!");
return AEE_ENOMEMORY;
}
worker_pool_t * me = (worker_pool_t *) (mem_blob + stack_size * n_threads);
// name for the first worker, useful in debugging threads
char name[19];
snprintf(name, 12, "0x%8x:", (int) me);
strcat(name, "worker0");
me->n_threads = n_threads;
// initializations
for (unsigned int i = 0; i < me->n_threads; i++) {
me->stack[i] = NULL;
me->thread[i] = 0;
me->context[i].id = i;
me->context[i].pool = me;
}
// initialize job queue
me->n_pending = 0;
me->n_jobs = 0;
me->next_job = 0;
me->seqn = 0;
me->killed = 0;
// launch the workers
qurt_thread_attr_t attr;
qurt_thread_attr_init(&attr);
for (unsigned int i = 0; i < me->n_threads; i++) {
// set up stack
me->stack[i] = mem_blob;
mem_blob += stack_size;
qurt_thread_attr_set_stack_addr(&attr, me->stack[i]);
qurt_thread_attr_set_stack_size(&attr, stack_size);
// set up name
qurt_thread_attr_set_name(&attr, name);
name[17] = (name[17] + 1);
// name threads context:worker0, context:worker1, .. (recycle at 9, but num threads should be less than that anyway)
if (name[17] > '9') {
name[17] = '0';
}
// set up priority - by default, match the creating thread's prio
int prio = qurt_thread_get_priority(qurt_thread_get_id());
if (prio < 1) {
prio = 1;
}
if (prio > LOWEST_USABLE_QURT_PRIO) {
prio = LOWEST_USABLE_QURT_PRIO;
}
qurt_thread_attr_set_priority(&attr, prio);
// launch
err = qurt_thread_create(&me->thread[i], &attr, worker_pool_main, (void *) &me->context[i]);
if (err) {
FARF(ERROR, "Could not launch worker threads!");
worker_pool_release((worker_pool_context_t *) &me);
return AEE_EQURTTHREADCREATE;
}
}
*context = (worker_pool_context_t *) me;
return AEE_SUCCESS;
}
AEEResult worker_pool_init(worker_pool_context_t * context, uint32_t n_threads) {
return worker_pool_init_with_stack_size(context, n_threads, WORKER_THREAD_STACK_SZ);
}
// clean up worker pool
void worker_pool_release(worker_pool_context_t * context) {
worker_pool_t * me = (worker_pool_t *) *context;
// if no worker pool exists, return error.
if (NULL == me) {
return;
}
atomic_store(&me->killed, 1);
atomic_fetch_add(&me->seqn, 1);
qurt_futex_wake(&me->seqn, me->n_threads);
// de-initializations
for (unsigned int i = 0; i < me->n_threads; i++) {
if (me->thread[i]) {
int status;
(void) qurt_thread_join(me->thread[i], &status);
}
}
// free allocated memory (were allocated as a single buffer starting at stack[0])
if (me->stack[0]) {
free(me->stack[0]);
}
*context = NULL;
}
// run jobs
AEEResult worker_pool_run_jobs(worker_pool_context_t context, worker_pool_job_t * job, unsigned int n) {
worker_pool_t * me = (worker_pool_t *) context;
if (NULL == me) {
FARF(ERROR, "worker-pool: invalid context");
return AEE_EBADPARM;
}
if (n > me->n_threads) {
FARF(ERROR, "worker-pool: invalid number of jobs %u for n-threads %u", n, me->n_threads);
return AEE_EBADPARM;
}
memcpy(me->job, job, sizeof(worker_pool_job_t) * n);
if (n > 1) {
atomic_store(&me->next_job, 1);
atomic_store(&me->n_jobs, n);
atomic_store(&me->n_pending, n - 1);
// wake up workers
atomic_fetch_add(&me->seqn, 1);
qurt_futex_wake(&me->seqn, n - 1);
}
// main thread runs job #0
me->job[0].func(n, 0, me->job[0].data);
if (n > 1) {
while (atomic_load(&me->n_pending))
;
}
return 0;
}
// run func
AEEResult worker_pool_run_func(worker_pool_context_t context, worker_callback_t func, void * data, unsigned int n) {
worker_pool_job_t job[n];
for (unsigned int i = 0; i < n; i++) {
job[i].func = func;
job[i].data = data;
}
return worker_pool_run_jobs(context, job, n);
}
AEEResult worker_pool_set_thread_priority(worker_pool_context_t context, unsigned int prio) {
worker_pool_t * me = (worker_pool_t *) context;
// if no worker pool exists, return error.
if (!me) {
return AEE_ENOMORE;
}
int result = AEE_SUCCESS;
if (prio < 1) {
prio = 1;
}
if (prio > LOWEST_USABLE_QURT_PRIO) {
prio = LOWEST_USABLE_QURT_PRIO;
}
for (unsigned int i = 0; i < me->n_threads; i++) {
int res = qurt_thread_set_priority(me->thread[i], (unsigned short) prio);
if (0 != res) {
result = AEE_EBADPARM;
FARF(ERROR, "QURT failed to set priority of thread %d, ERROR = %d", me->thread[i], res);
}
}
return result;
}
AEEResult worker_pool_retrieve_thread_id(worker_pool_context_t context, unsigned int * tids) {
worker_pool_t * me = (worker_pool_t *) context;
if (!me) {
FARF(ERROR, "worker-pool: invalid context");
return AEE_EBADPARM;
;
}
for (int i = 0; i < me->n_threads; i++) {
tids[i] = me->thread[i];
}
return AEE_SUCCESS;
}
AEEResult worker_pool_get_thread_priority(worker_pool_context_t context, unsigned int * prio) {
worker_pool_t * me = (worker_pool_t *) context;
if (!me) {
FARF(ERROR, "worker-pool: invalid context");
return AEE_EBADPARM;
}
int priority = qurt_thread_get_priority(me->thread[0]);
if (priority > 0) {
*prio = priority;
return 0;
} else {
*prio = 0;
return AEE_EBADSTATE;
}
}

65
ggml/src/ggml-hexagon/htp/worker-pool.h
View File

@@ -1,65 +0,0 @@
#ifndef HTP_WORKER_POOL_H
#define HTP_WORKER_POOL_H
// MACRO enables function to be visible in shared-library case.
#define WORKERPOOL_API __attribute__((visibility("default")))
#include <AEEStdDef.h>
#include <AEEStdErr.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
/// signature of callbacks to be invoked by worker threads
typedef void (*worker_callback_t)(unsigned int n, unsigned int i, void *);
/// Typedef of worker_pool context
typedef void * worker_pool_context_t;
/// descriptor for requested callback
typedef struct {
worker_callback_t func;
void * data;
} worker_pool_job_t;
#define WORKER_THREAD_STACK_SZ (2 * 16384)
/// Maximum supported number of worker threads.
#define MAX_NUM_WORKERS 10
#if __HVX_ARCH__ > 79
#define WORKER_POOL_POLL_COUNT 2000
#else
#define WORKER_POOL_POLL_COUNT 1
#endif
// Initialize worker pool.
WORKERPOOL_API AEEResult worker_pool_init(worker_pool_context_t * context, uint32_t n_threads);
// Initialize worker pool with custom stack size
WORKERPOOL_API AEEResult worker_pool_init_with_stack_size(worker_pool_context_t * context,
uint32_t n_threads,
uint32_t stack_size);
// Kill worker threads and release worker pool resources
WORKERPOOL_API void worker_pool_release(worker_pool_context_t * context);
// Run jobs with the worker pool.
WORKERPOOL_API AEEResult worker_pool_run_jobs(worker_pool_context_t context, worker_pool_job_t * job, unsigned int n);
WORKERPOOL_API AEEResult worker_pool_run_func(worker_pool_context_t context,
worker_callback_t func,
void * data,
unsigned int n);
WORKERPOOL_API AEEResult worker_pool_set_thread_priority(worker_pool_context_t context, unsigned int prio);
WORKERPOOL_API AEEResult worker_pool_get_thread_priority(worker_pool_context_t context, unsigned int * prio);
WORKERPOOL_API AEEResult worker_pool_retrieve_thread_id(worker_pool_context_t context, unsigned int * tids);
#ifdef __cplusplus
}
#endif
#endif // #ifndef HTP_WORKER_POOL_H