diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index df8313e819..b31e92e2da 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -857,7 +857,11 @@ struct ggml_tensor * llama_model_loader::require_tensor_meta(const std::string & return tensor; } -const struct ggml_tensor * llama_model_loader::check_tensor_dims(const std::string & name, const std::vector & ne, bool required) const { +const struct ggml_tensor * llama_model_loader::check_tensor_dims( + const std::string & name, + const std::vector & ne, + bool required, + bool allow_reshape) const { const struct ggml_tensor * cur = get_tensor_meta(name.c_str()); if (cur == NULL) { @@ -867,21 +871,33 @@ const struct ggml_tensor * llama_model_loader::check_tensor_dims(const std::stri throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str())); } - { - bool is_ok = true; + bool is_ok = true; + + if (allow_reshape) { + // check total number of elements only + const int64_t ncur = ggml_nelements(cur); + int64_t nexp = 1; + for (size_t i = 0; i < ne.size(); ++i) { + nexp *= ne[i]; + } + if (ncur != nexp) { + is_ok = false; + } + } else { for (size_t i = 0; i < GGML_MAX_DIMS; ++i) { if ((i < ne.size() && ne[i] != cur->ne[i]) || (i >= ne.size() && cur->ne[i] != 1)) { is_ok = false; break; } } - if (!is_ok) { - throw std::runtime_error( - format("%s: tensor '%s' has wrong shape; expected %s, got %s", - __func__, name.c_str(), - llama_format_tensor_shape(ne).c_str(), - llama_format_tensor_shape(cur).c_str())); - } + } + + if (!is_ok) { + throw std::runtime_error( + format("%s: tensor '%s' has wrong shape; expected %s, got %s", + __func__, name.c_str(), + llama_format_tensor_shape(ne).c_str(), + llama_format_tensor_shape(cur).c_str())); } return cur; @@ -1246,11 +1262,25 @@ struct ggml_tensor * llama_model_loader::create_tensor( return ret; } - ggml_tensor * t_meta = get_tensor_meta(tn.str().c_str()); - ggml_backend_buffer_type_t buft = buft_for_tensor(t_meta); - if (buft == nullptr) { - return nullptr; // return type is ggml_tensor * + LLAMA_LOG_DEBUG("%s: loading tensor %s\n", __func__, tn.str().c_str()); + const struct ggml_tensor * cur = check_tensor_dims(tn.str(), ne, !(flags & TENSOR_NOT_REQUIRED), flags & TENSOR_ALLOW_RESHAPE); + if (cur == NULL) { + return NULL; } + + ggml_tensor t_meta = *cur; + if (flags & TENSOR_ALLOW_RESHAPE) { + for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) { + t_meta.ne[dim] = dim < ne.size() ? ne.begin()[dim] : 1; + t_meta.nb[dim] = dim == 0 ? ggml_type_size(t_meta.type) : t_meta.ne[dim-1]*t_meta.nb[dim-1]; + } + } + + ggml_backend_buffer_type_t buft = buft_for_tensor(&t_meta); + if (buft == nullptr) { + return nullptr; + } + ggml_context * ctx = ctx_for_buft(buft); // if duplicated, check if the original tensor was allocated in the same buffer type context and avoid creating a new one @@ -1261,20 +1291,13 @@ struct ggml_tensor * llama_model_loader::create_tensor( } } - LLAMA_LOG_DEBUG("%s: loading tensor %s\n", __func__, tn.str().c_str()); - const struct ggml_tensor * cur = check_tensor_dims(tn.str(), ne, !(flags & TENSOR_NOT_REQUIRED)); - - if (cur == NULL) { - return NULL; - } - const bool duplicated = flags & TENSOR_DUPLICATED; - struct ggml_tensor * tensor = ggml_dup_tensor(ctx, cur); - ggml_set_name(tensor, ggml_get_name(cur)); + struct ggml_tensor * tensor = ggml_dup_tensor(ctx, &t_meta); + ggml_set_name(tensor, ggml_get_name(&t_meta)); if (duplicated) { - size_data += ggml_nbytes(cur); + size_data += ggml_nbytes(&t_meta); } else { n_created++; } @@ -1282,34 +1305,6 @@ struct ggml_tensor * llama_model_loader::create_tensor( return tensor; } -struct ggml_tensor * llama_model_loader::create_tensor_as_view(struct ggml_context * ctx, struct ggml_tensor * base, const std::string & name, const std::initializer_list & ne, size_t offset, bool required) { - const struct ggml_tensor * cur = check_tensor_dims(name, ne, required); - - if (cur == NULL) { - return NULL; - } - - if (cur->type != base->type) { - throw std::runtime_error(format("%s: tensor '%s' has wrong type; expected %s, got %s", __func__, name.c_str(), ggml_type_name(base->type), ggml_type_name(cur->type))); - } - - std::array dims; - for (size_t i = 0; i < GGML_MAX_DIMS; ++i) { - dims[i] = i < ne.size() ? ne.begin()[i] : 1; - } - - struct ggml_tensor * tensor = ggml_view_4d(ctx, base, - dims[0], dims[1], dims[2], dims[3], - cur->nb[1], cur->nb[2], cur->nb[3], - offset); - - ggml_set_name(tensor, name.c_str()); - - n_created++; - - return tensor; -} - void llama_model_loader::done_getting_tensors(bool partial) const { if (n_created > n_tensors) { throw std::runtime_error(format("%s: too many tensors created; expected %d, got %d", __func__, n_tensors, n_created)); diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h index 7ad3807822..d6b31c2311 100644 --- a/src/llama-model-loader.h +++ b/src/llama-model-loader.h @@ -67,6 +67,7 @@ struct llama_model_loader { static const int TENSOR_DUPLICATED = 1 << 1; static const int TENSOR_SKIP = 1 << 2; static const int TENSOR_SKIP_IF_VIRTUAL = 1 << 3; + static const int TENSOR_ALLOW_RESHAPE = 1 << 4; int n_kv = 0; int n_tensors = 0; @@ -177,14 +178,16 @@ struct llama_model_loader { struct ggml_tensor * require_tensor_meta(const std::string & name) const; - const struct ggml_tensor * check_tensor_dims(const std::string & name, const std::vector & ne, bool required) const; + const struct ggml_tensor * check_tensor_dims( + const std::string & name, + const std::vector & ne, + bool required, + bool allow_reshape) const; struct ggml_tensor * create_tensor( const llama_hparams & hparams, const buft_list_t * buft_list_cpu, const buft_list_t * buft_list_input, const buft_list_t * buft_list_output, const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags); - struct ggml_tensor * create_tensor_as_view(struct ggml_context * ctx, struct ggml_tensor * base, const std::string & name, const std::initializer_list & ne, size_t offset, bool required = true); - void done_getting_tensors(bool partial = false) const; void init_mappings(bool prefetch = true, llama_mlocks * mlock_mmaps = nullptr); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 8fff1a4326..333f506de5 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -2867,7 +2867,8 @@ llama_model_base::llama_model_base(const struct llama_model_params & params) : l TENSOR_DUPLICATED (llama_model_loader::TENSOR_DUPLICATED), TENSOR_NOT_REQUIRED (llama_model_loader::TENSOR_NOT_REQUIRED), TENSOR_SKIP (llama_model_loader::TENSOR_SKIP), - TENSOR_SKIP_IF_VIRTUAL(llama_model_loader::TENSOR_SKIP_IF_VIRTUAL) {} + TENSOR_SKIP_IF_VIRTUAL(llama_model_loader::TENSOR_SKIP_IF_VIRTUAL), + TENSOR_ALLOW_RESHAPE (llama_model_loader::TENSOR_ALLOW_RESHAPE) {} ggml_tensor * llama_model_base::create_tensor(const LLM_TN_IMPL & tn, const std::initializer_list & ne, int flags) { GGML_ASSERT(ml != nullptr); diff --git a/src/llama-model.h b/src/llama-model.h index 056a6efa59..6b9e94a0a6 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -719,6 +719,7 @@ struct llama_model_base : public llama_model { const int TENSOR_NOT_REQUIRED; const int TENSOR_SKIP; const int TENSOR_SKIP_IF_VIRTUAL; + const int TENSOR_ALLOW_RESHAPE; explicit llama_model_base(const llama_model_params & params); virtual ~llama_model_base() = default; diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index e68dc49b6d..89cd461765 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -114,7 +114,9 @@ void llama_model_deepseek4::load_arch_tensors(llama_model_loader & ml) { layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head}, flags); layer.wkv = create_tensor(tn(LLM_TENSOR_ATTN_KV, "weight", i), {n_embd, n_embd_head}, flags); layer.attn_kv_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_NORM, "weight", i), {n_embd_head}, flags); - layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank * o_groups}, flags); + // for wo_a, the shape in the file is (n_head * n_embd_head / o_groups, o_lora_rank*o_groups) + // so we reshape here, to avoid reshaping the tensor in the graph + layer.wo_a = create_tensor(tn(LLM_TENSOR_ATTN_OUT_A, "weight", i), {n_head * n_embd_head / o_groups, o_lora_rank, o_groups}, flags | TENSOR_ALLOW_RESHAPE); layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT_B, "weight", i), {o_groups * o_lora_rank, n_embd}, flags); layer.hc_attn_fn = create_tensor(tn(LLM_TENSOR_HC_ATTN_FN, "weight", i), {hc_dim, hc_mix_dim}, flags); @@ -1258,7 +1260,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_attention_impl( out = ggml_reshape_3d(ctx0, out, o_group_dim, n_groups, nt); out = ggml_permute(ctx0, out, 0, 2, 1, 3); - ggml_tensor * oa = ggml_mul_mat(ctx0, ggml_reshape_3d(ctx0, layer.wo_a, layer.wo_a->ne[0], o_lora_rank, n_groups), out); + ggml_tensor * oa = ggml_mul_mat(ctx0, layer.wo_a, out); cb(oa, "attn_wo_a", il); oa = ggml_permute(ctx0, oa, 0, 2, 1, 3); oa = ggml_cont_2d(ctx0, oa, o_lora_rank*n_groups, nt);