upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-03 03:17:32 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into xsn/decision

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-10-01 20:17:16 +02:00
parent 93ed62dce6 e358d59178
commit 9efdb2f5d7
6 changed files with 187 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-hexagon/htp/concat-ops.c
+40 -48
View File
@@ -1,15 +1,17 @@
#include "dma-queue.h"
#include "hex-common.h"
#include "hex-cpy-dma.h"
#include "hex-fastdiv.h"
#include "hex-profile.h"
#include "hexagon_protos.h"
#include "hexagon_types.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hexagon_types.h"
#include "hexagon_protos.h"
#include "hvx_hexagon_protos.h"
#include "dma-queue.h"
#include "htp-vtcm.h"
#include "hvx-utils.h"
#include "hex-fastdiv.h"
#include "hvx_hexagon_protos.h"
#include <string.h>
struct htp_concat_context {
@@ -285,73 +287,63 @@ static void concat_generic(unsigned int nth, unsigned int ith, void * data) {
}
}
static bool concat_dim1_contiguous_dma(struct htp_ops_context * octx, int dim, uint32_t type_size) {
static bool concat_dma(struct htp_ops_context * octx, int dim, uint32_t type_size) {
if (dim < 0 || dim >= HTP_OP_MAX_DIMS) {
return false;
}
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
if (dim != 1 || octx->ctx->mdev.count > 1 ||
// Not partitioned across devices: the row/element-split paths handle that.
if (octx->ctx->mdev.count > 1 ||
(dst->type != HTP_TYPE_F32 && dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_I32) ||
src0->type != dst->type || src1->type != dst->type ||
src0->ne[0] != dst->ne[0] || src1->ne[0] != dst->ne[0] ||
src0->ne[2] != dst->ne[2] || src1->ne[2] != dst->ne[2] ||
src0->ne[3] != dst->ne[3] || src1->ne[3] != dst->ne[3] ||
dst->ne[1] != src0->ne[1] + src1->ne[1] ||
!htp_tensor_is_contiguous(src0, type_size) ||
!htp_tensor_is_contiguous(src1, type_size) ||
!htp_tensor_is_contiguous(dst, type_size)) {
src0->type != dst->type || src1->type != dst->type || src0->nb[0] != type_size || src1->nb[0] != type_size ||
dst->nb[0] != type_size || (size_t) dst->ne[0] * type_size > DMA_MAX_SIZE_24B ||
dst->nb[1] > DMA_MAX_STRIDE_24B || src0->nb[1] > DMA_MAX_STRIDE_24B || src1->nb[1] > DMA_MAX_STRIDE_24B) {
return false;
}
const uint32_t src0_row_size = src0->ne[0] * type_size;
const uint32_t src1_row_size = src1->ne[0] * type_size;
// v75+ dma_queue_push() writes a 2D descriptor directly and does not split overflow.
#if __HVX_ARCH__ >= 75
if (src0_row_size > 0xffffffu || src1_row_size > 0xffffffu ||
src0->nb[1] > 0xffffffu || src1->nb[1] > 0xffffffu || dst->nb[1] > 0xffffffu ||
src0->ne[1] > UINT16_MAX || src1->ne[1] > UINT16_MAX) {
return false;
}
#endif
dma_queue * q = octx->ctx->dma[0];
for (uint32_t i3 = 0; i3 < dst->ne[3]; ++i3) {
for (uint32_t i2 = 0; i2 < dst->ne[2]; ++i2) {
dma_addr_t dst_addr = dst->data + i3 * dst->nb[3] + i2 * dst->nb[2];
dma_addr_t src0_addr = src0->data + i3 * src0->nb[3] + i2 * src0->nb[2];
dma_addr_t src1_addr = src1->data + i3 * src1->nb[3] + i2 * src1->nb[2];
if (!dma_queue_push(q, dma_make_data(dst_addr, src0_addr), dst->nb[1], src0->nb[1], src0_row_size, src0->ne[1])) {
dma_queue_flush(q);
dma_queue_push(q, dma_make_data(dst_addr, src0_addr), dst->nb[1], src0->nb[1], src0_row_size, src0->ne[1]);
}
dst_addr += src0->ne[1] * dst->nb[1];
if (!dma_queue_push(q, dma_make_data(dst_addr, src1_addr), dst->nb[1], src1->nb[1], src1_row_size, src1->ne[1])) {
dma_queue_flush(q);
dma_queue_push(q, dma_make_data(dst_addr, src1_addr), dst->nb[1], src1->nb[1], src1_row_size, src1->ne[1]);
}
for (int d = 0; d < HTP_OP_MAX_DIMS; d++) {
const uint32_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d];
if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) {
return false;
}
}
// The two views of dst, shaped like the sources.
struct htp_tensor view0 = *dst;
struct htp_tensor view1 = *dst;
for (int d = 0; d < HTP_OP_MAX_DIMS; d++) {
view0.ne[d] = src0->ne[d];
view1.ne[d] = src1->ne[d];
}
view1.data += (uint64_t) src0->ne[dim] * dst->nb[dim];
dma_queue * q = octx->ctx->dma[0];
cpy_dma_sametype_sameshape(q, &view0, src0, type_size);
cpy_dma_sametype_sameshape(q, &view1, src1, type_size);
dma_queue_flush(q);
return true;
}
int op_concat(struct htp_ops_context * octx) {
int dim = octx->op_params[0];
if (dim < 0 || dim >= HTP_OP_MAX_DIMS) {
return HTP_STATUS_NO_SUPPORT;
}
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
int dim = octx->op_params[0];
const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
bool is_src1_transposed = (src1->nb[0] > src1->nb[1]);
bool is_src0_transposed = (src0->nb[0] > src0->nb[1]);
if (concat_dim1_contiguous_dma(octx, dim, type_size)) {
if (concat_dma(octx, dim, type_size)) {
return HTP_STATUS_OK;
}
ggml/src/ggml-hexagon/htp/cpy-ops.c
+9 -90
View File
@@ -11,12 +11,12 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-cpy-dma.h"
#include "htp-ctx.h"
#include "htp-ops.h"
#include "htp-ops.h"
#include "hvx-utils.h"
#include "htp-tensor.h"
#include "htp-fence.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "hvx-utils.h"
struct htp_copy_context {
struct htp_ops_context * octx;
@@ -49,30 +49,6 @@ struct htp_copy_context {
struct fastdiv_values div_ne02_ne01_ne00;
};
static inline void cpy_dma_sametype_reshape_contig(
dma_queue * dma_q,
dma_addr_t dst,
dma_addr_t src0,
uint32_t total_bytes
) {
if (total_bytes == 0) {
return;
}
const uint32_t max_chunk = DMA_SAFE_CHUNK_SIZE;
while (total_bytes > 0) {
const uint32_t chunk = MIN(total_bytes, max_chunk);
if (!dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/ 1)) {
dma_queue_flush(dma_q);
dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/ 1);
}
dst += chunk;
src0 += chunk;
total_bytes -= chunk;
}
dma_queue_flush(dma_q);
}
#define cpy_preamble \
const struct htp_tensor *src0 = octx->src[0]; \
const struct htp_tensor *dst = octx->dst; \
@@ -112,6 +88,7 @@ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, vo
dma_addr_t dst_addr = dst->data + ir0 * ne00 * ELEM_SIZE; \
dma_addr_t src0_addr = src0->data + ir0 * ne00 * ELEM_SIZE; \
cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (ir1 - ir0) * ne00 * ELEM_SIZE); \
dma_queue_flush(dma_q); \
return; \
} \
const uint32_t ne02_ne01 = ne02 * ne01; \
@@ -157,6 +134,7 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void
dma_addr_t dst_addr = dst->data + th_start * ELEM_SIZE; \
dma_addr_t src0_addr = src0->data + th_start * ELEM_SIZE; \
cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (th_end - th_start) * ELEM_SIZE); \
dma_queue_flush(dma_q); \
return; \
} \
\
@@ -381,67 +359,6 @@ static void cpy_thread_f32_i32_sameshape(unsigned int nth, unsigned int ith, voi
}
}
static inline void cpy_dma_push_2d_chunked(
dma_queue * dma_q,
dma_addr_t dst,
dma_addr_t src,
size_t dst_stride,
size_t src_stride,
size_t row_size,
uint32_t nrows
) {
while (nrows > 0) {
const uint32_t cur_rows = MIN(nrows, DMA_MAX_NROWS);
if (!dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows)) {
dma_queue_flush(dma_q);
dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows);
}
dst += cur_rows * dst_stride;
src += cur_rows * src_stride;
nrows -= cur_rows;
}
}
static inline void cpy_dma_sametype_sameshape(
struct htp_ops_context * octx,
const struct htp_tensor * dst,
const struct htp_tensor * src0,
uint32_t elem_size,
uint32_t ne00, uint32_t ne01, uint32_t ne02, uint32_t ne03,
uint32_t nb01, uint32_t nb02, uint32_t nb03,
uint32_t nb1, uint32_t nb2, uint32_t nb3
) {
const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size);
dma_queue * dma_q = octx->ctx->dma[0];
if (contiguous) {
cpy_dma_sametype_reshape_contig(dma_q, dst->data, src0->data, ne00 * elem_size * ne01 * ne02 * ne03);
return;
}
const bool contiguous_outer =
(ne02 == 1 || (nb02 == ne01 * nb01 && nb2 == ne01 * nb1)) &&
(ne03 == 1 || (nb03 == ne02 * nb02 && nb3 == ne02 * nb2));
if (contiguous_outer) {
uint32_t total_rows = ne01 * ne02 * ne03;
cpy_dma_push_2d_chunked(dma_q, dst->data, src0->data, nb1, nb01, ne00 * elem_size, total_rows);
dma_queue_flush(dma_q);
return;
}
for (uint32_t i03 = 0; i03 < ne03; i03++) {
for (uint32_t i02 = 0; i02 < ne02; i02++) {
dma_addr_t dst_data = dst->data + i02 * nb2 + i03 * nb3;
dma_addr_t src0_data = src0->data + i02 * nb02 + i03 * nb03;
cpy_dma_push_2d_chunked(dma_q, dst_data, src0_data, nb1, nb01, ne00 * elem_size, ne01);
}
}
dma_queue_flush(dma_q);
}
static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
cpy_preamble;
*use_dma = false;
@@ -551,7 +468,8 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
if (sametype && (octx->ctx->mdev.count <= 1 || htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
if (octx->ctx->mdev.idx == 0) {
*use_dma = true;
cpy_dma_sametype_sameshape(octx, dst, src0, ct.src0_type_size, ne00, ne01, ne02, ne03, nb01, nb02, nb03, nb1, nb2, nb3);
cpy_dma_sametype_sameshape(octx->ctx->dma[0], dst, src0, ct.src0_type_size);
dma_queue_flush(octx->ctx->dma[0]);
}
} else {
work_queue_func_t copy_fun = NULL;
@@ -582,6 +500,7 @@ static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
if (octx->ctx->mdev.count <= 1 && dst_is_contiguous && src_is_contiguous) {
*use_dma = true;
cpy_dma_sametype_reshape_contig(octx->ctx->dma[0], dst->data, src0->data, total_elems * ct.dst_type_size);
dma_queue_flush(octx->ctx->dma[0]);
return HTP_STATUS_OK;
}
ggml/src/ggml-hexagon/htp/hex-cpy-dma.h
+126
View File
@@ -0,0 +1,126 @@
#ifndef HEX_CPY_DMA_H
#define HEX_CPY_DMA_H
// DDR<->DDR DMA copies of same-type, same-shape tensors with arbitrary strides.
// Used by CPY for the copy itself and by CONCAT, which is two such copies into
// two views of its destination. Every helper only pushes descriptors; the
// caller flushes the queue when it needs the data.
#include "dma-queue.h"
#include "hex-common.h"
#include "htp-tensor.h"
#include <stddef.h>
#include <stdint.h>
// Contiguous byte run, as 1d transfers of at most DMA_SAFE_CHUNK_SIZE each.
static inline void cpy_dma_sametype_reshape_contig(dma_queue * dma_q,
dma_addr_t dst,
dma_addr_t src0,
uint32_t total_bytes) {
if (total_bytes == 0) {
return;
}
const uint32_t max_chunk = DMA_SAFE_CHUNK_SIZE;
while (total_bytes > 0) {
const uint32_t chunk = MIN(total_bytes, max_chunk);
if (!dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1)) {
dma_queue_flush(dma_q);
dma_queue_push(dma_q, dma_make_data(dst, src0), chunk, chunk, chunk, /*nrows=*/1);
}
dst += chunk;
src0 += chunk;
total_bytes -= chunk;
}
}
// One 2d transfer, split at the 16-bit nrows field.
static inline void cpy_dma_push_2d_chunked(dma_queue * dma_q,
dma_addr_t dst,
dma_addr_t src,
size_t dst_stride,
size_t src_stride,
size_t row_size,
uint32_t nrows) {
if (row_size == 0 || nrows == 0) {
return;
}
while (nrows > 0) {
const uint32_t cur_rows = MIN(nrows, DMA_MAX_NROWS);
if (!dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows)) {
dma_queue_flush(dma_q);
dma_queue_push(dma_q, dma_make_data(dst, src), dst_stride, src_stride, row_size, cur_rows);
}
dst += cur_rows * dst_stride;
src += cur_rows * src_stride;
nrows -= cur_rows;
}
}
// Copy src0 into dst: same type, same ne[], any nb[] above dim 0, dim 0 dense on
// both sides (nb[0] == elem_size).
static inline void cpy_dma_sametype_sameshape(dma_queue * dma_q,
const struct htp_tensor * dst,
const struct htp_tensor * src0,
uint32_t elem_size) {
const uint32_t ne00 = src0->ne[0];
const uint32_t ne01 = src0->ne[1];
const uint32_t ne02 = src0->ne[2];
const uint32_t ne03 = src0->ne[3];
if (ne00 == 0 || ne01 == 0 || ne02 == 0 || ne03 == 0) {
return;
}
const uint32_t nb01 = src0->nb[1];
const uint32_t nb02 = src0->nb[2];
const uint32_t nb03 = src0->nb[3];
const uint32_t nb1 = dst->nb[1];
const uint32_t nb2 = dst->nb[2];
const uint32_t nb3 = dst->nb[3];
const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size);
if (contiguous) {
cpy_dma_sametype_reshape_contig(dma_q, dst->data, src0->data, ne00 * elem_size * ne01 * ne02 * ne03);
return;
}
// The single-descriptor path flattens (i01,i02,i03) into one row index, so every
// row must sit at a constant stride: nb01 on the source, nb1 on the destination.
// Walk the outer dims and require each to continue that progression. A dim of
// extent 1 spans no rows, so it is skipped -- but its own stride must NOT then be
// used to justify the next dim's stride, which is what comparing nb03 against
// ne02*nb02 did: ggml leaves the stride of an extent-1 dim meaningless, so a view
// could pass the check while its rows were nowhere near that stride.
uint32_t exp_src = ne01 * nb01;
uint32_t exp_dst = ne01 * nb1;
bool contiguous_outer = true;
if (ne02 != 1) {
contiguous_outer = contiguous_outer && (nb02 == exp_src) && (nb2 == exp_dst);
}
exp_src *= ne02;
exp_dst *= ne02;
if (ne03 != 1) {
contiguous_outer = contiguous_outer && (nb03 == exp_src) && (nb3 == exp_dst);
}
if (contiguous_outer) {
uint32_t total_rows = ne01 * ne02 * ne03;
cpy_dma_push_2d_chunked(dma_q, dst->data, src0->data, nb1, nb01, ne00 * elem_size, total_rows);
return;
}
for (uint32_t i03 = 0; i03 < ne03; i03++) {
for (uint32_t i02 = 0; i02 < ne02; i02++) {
dma_addr_t dst_data = dst->data + i02 * nb2 + i03 * nb3;
dma_addr_t src0_data = src0->data + i02 * nb02 + i03 * nb03;
cpy_dma_push_2d_chunked(dma_q, dst_data, src0_data, nb1, nb01, ne00 * elem_size, ne01);
}
}
}
#endif /* HEX_CPY_DMA_H */
src/llama-memory-hybrid-idx.cpp
+5 -3
View File
@@ -762,10 +762,12 @@ void llama_memory_hybrid_idx_context::kpool_build_state(const llama_ubatch & uba
}
}
// a ubatch touches at most t_s/kpool + 1 pools of a sequence with t_s tokens: pad to that bound so the
// graph keeps its shape as the count moves, e.g. between 0 and n_seq while several sequences decode
// a ubatch touches at most t_s/kpool + 1 pools per sequence, pad to that bound so the graph keeps its shape
// as the count moves; reserve sizes the list for every pool the cache can hold, so never pad past n_pool_max
const auto * idx = mem->get_mem_idx();
const uint32_t n_pool_max = idx->get_size() / kpool * idx->get_n_seq_max();
const uint32_t bound = ubatch.n_tokens/kpool + ubatch.n_seqs_unq;
st.n_new_g = std::max({st.n_new, 1u, std::min(bound, kpool_pad(st.n_pool_real) - 1)});
st.n_new_g = std::max({st.n_new, 1u, std::min({bound, kpool_pad(st.n_pool_real) - 1, n_pool_max})});
}
const llama_memory_hybrid_idx_context::kpool_state & llama_memory_hybrid_idx_context::kpool_cur() const {
tests/fusion/MTL.csv
+1 -1
View File
@@ -246,7 +246,7 @@ qwen3vl ,0 ,any ,RMS_NORM+MUL , 9
qwen3vlmoe ,1 ,any ,MUL+ADD , 4
qwen3vlmoe ,1 ,any ,RMS_NORM+MUL , 9
qwen3vlmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
qwen4exp ,0 ,any ,ADD+ADD+ADD , 1
qwen4exp ,0 ,any ,ADD+ADD+ADD , 2
qwen4exp ,0 ,any ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 9
qwen4exp ,0 ,any ,GATED_DELTA_NET+CPY , 1
qwen4exp ,0 ,any ,MUL+ADD , 4
1 # test-fusion baseline for device MTL
246 qwen3vlmoe ,1 ,any ,MUL+ADD , 4
247 qwen3vlmoe ,1 ,any ,RMS_NORM+MUL , 9
248 qwen3vlmoe ,1 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2
249 qwen4exp ,0 ,any ,ADD+ADD+ADD , 1 qwen4exp ,0 ,any ,ADD+ADD+ADD , 2
250 qwen4exp ,0 ,any ,ADD+ADD+ADD+ADD+ADD+ADD+ADD , 9
251 qwen4exp ,0 ,any ,GATED_DELTA_NET+CPY , 1
252 qwen4exp ,0 ,any ,MUL+ADD , 4
tests/test-recurrent-state-rollback.cpp
+6 -6
View File
@@ -34,10 +34,10 @@ static const char * test_status_str(test_status status) {
return "";
}
static bool decode_tokens(llama_context * ctx, const std::vector<llama_token> & tokens, uint32_t count) {
static bool decode_tokens(llama_context * ctx, const std::vector<llama_token> & tokens) {
common_batch batch(ctx);
for (uint32_t pos = 0; pos < count; ++pos) {
batch.add(tokens[pos], pos, 0, pos + 1 == count);
for (uint32_t pos = 0; pos < tokens.size(); ++pos) {
batch.add(tokens[pos], pos, 0, pos + 1 == tokens.size());
}
return llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get()) == 0;
}
@@ -74,7 +74,7 @@ static llama_context_ptr init_ctx(llama_model * model, llama_context_params cpar
// Use a full ubatch so buffer discovery preserves prefill allocation sizes.
const uint32_t n_tokens = llama_n_ubatch(ctx.get());
if (!decode_tokens(ctx.get(), std::vector<llama_token>(n_tokens, 0), n_tokens)) {
if (!decode_tokens(ctx.get(), std::vector<llama_token>(n_tokens, 0))) {
return nullptr;
}
llama_synchronize(ctx.get());
@@ -347,7 +347,7 @@ static test_status test_rollback(const common_params & params, llama_model * mod
// Decode the full prompt on the source, then roll back three positions.
// Replaying them crosses DSV4's ratio-4 compressor boundary.
// Rollback leaves the recurrent memory in a snapshot state (rs_idx != 0).
if (!decode_tokens(ctx_src.get(), tokens, n_tokens)) {
if (!decode_tokens(ctx_src.get(), tokens)) {
LOG_ERR("%s: failed to decode prompt\n", __func__);
return test_status::FAIL;
}
@@ -432,7 +432,7 @@ static test_status test_rollback(const common_params & params, llama_model * mod
t = 0;
}
}
if (!decode_tokens(ctx_dirty.get(), noise, n_tokens)) {
if (!decode_tokens(ctx_dirty.get(), noise)) {
LOG_ERR("%s: dirty prompt decode failed\n", __func__);
return test_status::FAIL;
}