diff --git a/ggml/src/ggml-hexagon/htp/concat-ops.c b/ggml/src/ggml-hexagon/htp/concat-ops.c index 1bb9ce5e3e..4dc1463934 100644 --- a/ggml/src/ggml-hexagon/htp/concat-ops.c +++ b/ggml/src/ggml-hexagon/htp/concat-ops.c @@ -1,15 +1,17 @@ +#include "dma-queue.h" #include "hex-common.h" +#include "hex-cpy-dma.h" +#include "hex-fastdiv.h" #include "hex-profile.h" +#include "hexagon_protos.h" +#include "hexagon_types.h" #include "htp-ctx.h" #include "htp-ops.h" #include "htp-tensor.h" -#include "hexagon_types.h" -#include "hexagon_protos.h" -#include "hvx_hexagon_protos.h" -#include "dma-queue.h" #include "htp-vtcm.h" #include "hvx-utils.h" -#include "hex-fastdiv.h" +#include "hvx_hexagon_protos.h" + #include struct htp_concat_context { @@ -285,73 +287,63 @@ static void concat_generic(unsigned int nth, unsigned int ith, void * data) { } } -static bool concat_dim1_contiguous_dma(struct htp_ops_context * octx, int dim, uint32_t type_size) { +static bool concat_dma(struct htp_ops_context * octx, int dim, uint32_t type_size) { + if (dim < 0 || dim >= HTP_OP_MAX_DIMS) { + return false; + } + const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; - if (dim != 1 || octx->ctx->mdev.count > 1 || + // Not partitioned across devices: the row/element-split paths handle that. + if (octx->ctx->mdev.count > 1 || (dst->type != HTP_TYPE_F32 && dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_I32) || - src0->type != dst->type || src1->type != dst->type || - src0->ne[0] != dst->ne[0] || src1->ne[0] != dst->ne[0] || - src0->ne[2] != dst->ne[2] || src1->ne[2] != dst->ne[2] || - src0->ne[3] != dst->ne[3] || src1->ne[3] != dst->ne[3] || - dst->ne[1] != src0->ne[1] + src1->ne[1] || - !htp_tensor_is_contiguous(src0, type_size) || - !htp_tensor_is_contiguous(src1, type_size) || - !htp_tensor_is_contiguous(dst, type_size)) { + src0->type != dst->type || src1->type != dst->type || src0->nb[0] != type_size || src1->nb[0] != type_size || + dst->nb[0] != type_size || (size_t) dst->ne[0] * type_size > DMA_MAX_SIZE_24B || + dst->nb[1] > DMA_MAX_STRIDE_24B || src0->nb[1] > DMA_MAX_STRIDE_24B || src1->nb[1] > DMA_MAX_STRIDE_24B) { return false; } - const uint32_t src0_row_size = src0->ne[0] * type_size; - const uint32_t src1_row_size = src1->ne[0] * type_size; - - // v75+ dma_queue_push() writes a 2D descriptor directly and does not split overflow. -#if __HVX_ARCH__ >= 75 - if (src0_row_size > 0xffffffu || src1_row_size > 0xffffffu || - src0->nb[1] > 0xffffffu || src1->nb[1] > 0xffffffu || dst->nb[1] > 0xffffffu || - src0->ne[1] > UINT16_MAX || src1->ne[1] > UINT16_MAX) { - return false; - } -#endif - - dma_queue * q = octx->ctx->dma[0]; - - for (uint32_t i3 = 0; i3 < dst->ne[3]; ++i3) { - for (uint32_t i2 = 0; i2 < dst->ne[2]; ++i2) { - dma_addr_t dst_addr = dst->data + i3 * dst->nb[3] + i2 * dst->nb[2]; - dma_addr_t src0_addr = src0->data + i3 * src0->nb[3] + i2 * src0->nb[2]; - dma_addr_t src1_addr = src1->data + i3 * src1->nb[3] + i2 * src1->nb[2]; - - if (!dma_queue_push(q, dma_make_data(dst_addr, src0_addr), dst->nb[1], src0->nb[1], src0_row_size, src0->ne[1])) { - dma_queue_flush(q); - dma_queue_push(q, dma_make_data(dst_addr, src0_addr), dst->nb[1], src0->nb[1], src0_row_size, src0->ne[1]); - } - - dst_addr += src0->ne[1] * dst->nb[1]; - if (!dma_queue_push(q, dma_make_data(dst_addr, src1_addr), dst->nb[1], src1->nb[1], src1_row_size, src1->ne[1])) { - dma_queue_flush(q); - dma_queue_push(q, dma_make_data(dst_addr, src1_addr), dst->nb[1], src1->nb[1], src1_row_size, src1->ne[1]); - } + for (int d = 0; d < HTP_OP_MAX_DIMS; d++) { + const uint32_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d]; + if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) { + return false; } } + // The two views of dst, shaped like the sources. + struct htp_tensor view0 = *dst; + struct htp_tensor view1 = *dst; + for (int d = 0; d < HTP_OP_MAX_DIMS; d++) { + view0.ne[d] = src0->ne[d]; + view1.ne[d] = src1->ne[d]; + } + view1.data += (uint64_t) src0->ne[dim] * dst->nb[dim]; + + dma_queue * q = octx->ctx->dma[0]; + + cpy_dma_sametype_sameshape(q, &view0, src0, type_size); + cpy_dma_sametype_sameshape(q, &view1, src1, type_size); dma_queue_flush(q); return true; } int op_concat(struct htp_ops_context * octx) { + int dim = octx->op_params[0]; + if (dim < 0 || dim >= HTP_OP_MAX_DIMS) { + return HTP_STATUS_NO_SUPPORT; + } + const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; const struct htp_tensor * dst = octx->dst; - int dim = octx->op_params[0]; - const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2; bool is_src1_transposed = (src1->nb[0] > src1->nb[1]); bool is_src0_transposed = (src0->nb[0] > src0->nb[1]); - if (concat_dim1_contiguous_dma(octx, dim, type_size)) { + if (concat_dma(octx, dim, type_size)) { return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c index aef3c04533..df38d3eed3 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.c +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c @@ -11,12 +11,12 @@ #define GGML_COMMON_DECL_C #include "ggml-common.h" +#include "hex-cpy-dma.h" #include "htp-ctx.h" -#include "htp-ops.h" -#include "htp-ops.h" -#include "hvx-utils.h" -#include "htp-tensor.h" #include "htp-fence.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "hvx-utils.h" struct htp_copy_context { struct htp_ops_context * octx; @@ -49,30 +49,6 @@ struct htp_copy_context { struct fastdiv_values div_ne02_ne01_ne00; }; -static inline void cpy_dma_sametype_reshape_contig( - dma_queue * dma_q, - dma_addr_t dst, - dma_addr_t src0, - uint32_t total_bytes -) { - if (total_bytes == 0) { - return; - } - - const uint32_t max_chunk = DMA_SAFE_CHUNK_SIZE; - while (total_bytes > 0) { - const uint32_t chunk = MIN(total_bytes, max_chunk); - if (!dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/ 1)) { - dma_queue_flush(dma_q); - dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/ 1); - } - dst += chunk; - src0 += chunk; - total_bytes -= chunk; - } - dma_queue_flush(dma_q); -} - #define cpy_preamble \ const struct htp_tensor *src0 = octx->src[0]; \ const struct htp_tensor *dst = octx->dst; \ @@ -112,6 +88,7 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo dma_addr_t dst_addr = dst->data + ir0 * ne00 * ELEM_SIZE; \ dma_addr_t src0_addr = src0->data + ir0 * ne00 * ELEM_SIZE; \ cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (ir1 - ir0) * ne00 * ELEM_SIZE); \ + dma_queue_flush(dma_q); \ return; \ } \ const uint32_t ne02_ne01 = ne02 * ne01; \ @@ -157,6 +134,7 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void dma_addr_t dst_addr = dst->data + th_start * ELEM_SIZE; \ dma_addr_t src0_addr = src0->data + th_start * ELEM_SIZE; \ cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (th_end - th_start) * ELEM_SIZE); \ + dma_queue_flush(dma_q); \ return; \ } \ \ @@ -381,67 +359,6 @@ static void cpy_thread_f32_i32_sameshape(unsigned int nth, unsigned int ith, voi } } -static inline void cpy_dma_push_2d_chunked( - dma_queue * dma_q, - dma_addr_t dst, - dma_addr_t src, - size_t dst_stride, - size_t src_stride, - size_t row_size, - uint32_t nrows -) { - while (nrows > 0) { - const uint32_t cur_rows = MIN(nrows, DMA_MAX_NROWS); - if (!dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows)) { - dma_queue_flush(dma_q); - dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows); - } - dst += cur_rows * dst_stride; - src += cur_rows * src_stride; - nrows -= cur_rows; - } -} - -static inline void cpy_dma_sametype_sameshape( - struct htp_ops_context * octx, - const struct htp_tensor * dst, - const struct htp_tensor * src0, - uint32_t elem_size, - uint32_t ne00, uint32_t ne01, uint32_t ne02, uint32_t ne03, - uint32_t nb01, uint32_t nb02, uint32_t nb03, - uint32_t nb1, uint32_t nb2, uint32_t nb3 -) { - const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size); - - dma_queue * dma_q = octx->ctx->dma[0]; - - if (contiguous) { - cpy_dma_sametype_reshape_contig(dma_q, dst->data, src0->data, ne00 * elem_size * ne01 * ne02 * ne03); - return; - } - - const bool contiguous_outer = - (ne02 == 1 || (nb02 == ne01 * nb01 && nb2 == ne01 * nb1)) && - (ne03 == 1 || (nb03 == ne02 * nb02 && nb3 == ne02 * nb2)); - - if (contiguous_outer) { - uint32_t total_rows = ne01 * ne02 * ne03; - cpy_dma_push_2d_chunked(dma_q, dst->data, src0->data, nb1, nb01, ne00 * elem_size, total_rows); - dma_queue_flush(dma_q); - return; - } - - for (uint32_t i03 = 0; i03 < ne03; i03++) { - for (uint32_t i02 = 0; i02 < ne02; i02++) { - dma_addr_t dst_data = dst->data + i02 * nb2 + i03 * nb3; - dma_addr_t src0_data = src0->data + i02 * nb02 + i03 * nb03; - cpy_dma_push_2d_chunked(dma_q, dst_data, src0_data, nb1, nb01, ne00 * elem_size, ne01); - } - } - - dma_queue_flush(dma_q); -} - static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) { cpy_preamble; *use_dma = false; @@ -551,7 +468,8 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) { if (sametype && (octx->ctx->mdev.count <= 1 || htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) { if (octx->ctx->mdev.idx == 0) { *use_dma = true; - cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3); + cpy_dma_sametype_sameshape(octx->ctx->dma[0], dst, src0, ct.src0_type_size); + dma_queue_flush(octx->ctx->dma[0]); } } else { work_queue_func_t copy_fun = NULL; @@ -582,6 +500,7 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) { if (octx->ctx->mdev.count <= 1 && dst_is_contiguous && src_is_contiguous) { *use_dma = true; cpy_dma_sametype_reshape_contig(octx->ctx->dma[0], dst->data, src0->data, total_elems * ct.dst_type_size); + dma_queue_flush(octx->ctx->dma[0]); return HTP_STATUS_OK; } diff --git a/ggml/src/ggml-hexagon/htp/hex-cpy-dma.h b/ggml/src/ggml-hexagon/htp/hex-cpy-dma.h new file mode 100644 index 0000000000..c87f495e28 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/hex-cpy-dma.h @@ -0,0 +1,126 @@ +#ifndef HEX_CPY_DMA_H +#define HEX_CPY_DMA_H + +// DDR<->DDR DMA copies of same-type, same-shape tensors with arbitrary strides. +// Used by CPY for the copy itself and by CONCAT, which is two such copies into +// two views of its destination. Every helper only pushes descriptors; the +// caller flushes the queue when it needs the data. + +#include "dma-queue.h" +#include "hex-common.h" +#include "htp-tensor.h" + +#include +#include + +// Contiguous byte run, as 1d transfers of at most DMA_SAFE_CHUNK_SIZE each. +static inline void cpy_dma_sametype_reshape_contig(dma_queue * dma_q, + dma_addr_t dst, + dma_addr_t src0, + uint32_t total_bytes) { + if (total_bytes == 0) { + return; + } + + const uint32_t max_chunk = DMA_SAFE_CHUNK_SIZE; + while (total_bytes > 0) { + const uint32_t chunk = MIN(total_bytes, max_chunk); + if (!dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1)) { + dma_queue_flush(dma_q); + dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1); + } + dst += chunk; + src0 += chunk; + total_bytes -= chunk; + } +} + +// One 2d transfer, split at the 16-bit nrows field. +static inline void cpy_dma_push_2d_chunked(dma_queue * dma_q, + dma_addr_t dst, + dma_addr_t src, + size_t dst_stride, + size_t src_stride, + size_t row_size, + uint32_t nrows) { + if (row_size == 0 || nrows == 0) { + return; + } + + while (nrows > 0) { + const uint32_t cur_rows = MIN(nrows, DMA_MAX_NROWS); + if (!dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows)) { + dma_queue_flush(dma_q); + dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows); + } + dst += cur_rows * dst_stride; + src += cur_rows * src_stride; + nrows -= cur_rows; + } +} + +// Copy src0 into dst: same type, same ne[], any nb[] above dim 0, dim 0 dense on +// both sides (nb[0] == elem_size). +static inline void cpy_dma_sametype_sameshape(dma_queue * dma_q, + const struct htp_tensor * dst, + const struct htp_tensor * src0, + uint32_t elem_size) { + const uint32_t ne00 = src0->ne[0]; + const uint32_t ne01 = src0->ne[1]; + const uint32_t ne02 = src0->ne[2]; + const uint32_t ne03 = src0->ne[3]; + + if (ne00 == 0 || ne01 == 0 || ne02 == 0 || ne03 == 0) { + return; + } + + const uint32_t nb01 = src0->nb[1]; + const uint32_t nb02 = src0->nb[2]; + const uint32_t nb03 = src0->nb[3]; + + const uint32_t nb1 = dst->nb[1]; + const uint32_t nb2 = dst->nb[2]; + const uint32_t nb3 = dst->nb[3]; + + const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size); + + if (contiguous) { + cpy_dma_sametype_reshape_contig(dma_q, dst->data, src0->data, ne00 * elem_size * ne01 * ne02 * ne03); + return; + } + + // The single-descriptor path flattens (i01,i02,i03) into one row index, so every + // row must sit at a constant stride: nb01 on the source, nb1 on the destination. + // Walk the outer dims and require each to continue that progression. A dim of + // extent 1 spans no rows, so it is skipped -- but its own stride must NOT then be + // used to justify the next dim's stride, which is what comparing nb03 against + // ne02*nb02 did: ggml leaves the stride of an extent-1 dim meaningless, so a view + // could pass the check while its rows were nowhere near that stride. + uint32_t exp_src = ne01 * nb01; + uint32_t exp_dst = ne01 * nb1; + bool contiguous_outer = true; + if (ne02 != 1) { + contiguous_outer = contiguous_outer && (nb02 == exp_src) && (nb2 == exp_dst); + } + exp_src *= ne02; + exp_dst *= ne02; + if (ne03 != 1) { + contiguous_outer = contiguous_outer && (nb03 == exp_src) && (nb3 == exp_dst); + } + + if (contiguous_outer) { + uint32_t total_rows = ne01 * ne02 * ne03; + cpy_dma_push_2d_chunked(dma_q, dst->data, src0->data, nb1, nb01, ne00 * elem_size, total_rows); + return; + } + + for (uint32_t i03 = 0; i03 < ne03; i03++) { + for (uint32_t i02 = 0; i02 < ne02; i02++) { + dma_addr_t dst_data = dst->data + i02 * nb2 + i03 * nb3; + dma_addr_t src0_data = src0->data + i02 * nb02 + i03 * nb03; + cpy_dma_push_2d_chunked(dma_q, dst_data, src0_data, nb1, nb01, ne00 * elem_size, ne01); + } + } +} + +#endif /* HEX_CPY_DMA_H */