upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-02 19:07:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

support lev & kev

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-10-01 22:14:48 +02:00
parent 82f7f147a7
commit 8536d4fb08
10 changed files with 483 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-context.cpp
+45 -17
View File
@@ -414,6 +414,10 @@ struct server_slot {
if (pooling == LLAMA_POOLING_TYPE_RANK && llama_get_causal_attn(ctx_tgt)) {
return true;
}
// a decision task reads its outputs from the last batch
if (task->type == SERVER_TASK_TYPE_DECISION) {
return true;
}
return false;
}
@@ -2232,21 +2236,39 @@ private:
res->scores.push_back(logits[label]);
}
} else {
// the prompt is evaluated in one batch, the n-th output of this slot is the n-th prompt token
// the outputs of this slot in this batch are the last tokens of the prompt
std::vector<int32_t> idx;
for (int i = 0; i < batch.size(); ++i) {
if (batch.tokens[i].output && batch.tokens[i].seq_id == slot.id) {
idx.push_back(i);
}
}
GGML_ASSERT(decision.column >= 0 && decision.column < llama_model_n_embd_out(model_tgt));
const int32_t pos_first = slot.prompt.n_tokens() - (int32_t) idx.size();
auto get_embd = [&](int32_t pos) -> const float * {
const int32_t i = pos - pos_first;
return i >= 0 && i < (int32_t) idx.size() ? llama_get_embeddings_ith(slot.ctx_tgt, idx[i]) : nullptr;
};
const int32_t n_embd_out = llama_model_n_embd_out(model_tgt);
const int32_t n_pointer = n_embd_out / 2;
const float * embd_q = decision.pointer >= 0 ? get_embd(decision.pointer) : nullptr;
GGML_ASSERT(decision.column >= 0 && decision.column < n_embd_out);
for (const int32_t marker : decision.markers) {
const float * embd = marker >= 0 && marker < (int32_t) idx.size() ? llama_get_embeddings_ith(slot.ctx_tgt, idx[marker]) : nullptr;
if (embd == nullptr) {
send_error(slot, "failed to get embeddings", ERROR_TYPE_SERVER);
const float * embd = get_embd(marker);
if (embd == nullptr || (decision.pointer >= 0 && embd_q == nullptr)) {
send_error(slot, "failed to get embeddings, the question and its options must fit in one batch", ERROR_TYPE_SERVER);
return;
}
res->scores.push_back(embd[decision.column]);
if (decision.pointer < 0) {
res->scores.push_back(embd[decision.column]);
continue;
}
float dot = 0.0f;
for (int32_t i = 0; i < n_pointer; i++) {
dot += embd_q[i] * embd[n_pointer + i];
}
res->scores.push_back(dot / sqrtf((float) n_pointer));
}
}
@@ -5339,16 +5361,17 @@ void server_routes::init_routes() {
return res;
}
// one task per question
// one task per variant of each question
auto & rd = res->rd;
{
std::vector<server_task> tasks;
tasks.reserve(questions.size());
for (const auto & question : questions) {
server_task task = server_task(SERVER_TASK_TYPE_DECISION);
task.id = rd.get_new_id();
decision.fill_task(state, question, files, ctx_server.mctx, ctx_server.init_opt, task);
tasks.push_back(std::move(task));
for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
server_task task = server_task(SERVER_TASK_TYPE_DECISION);
task.id = rd.get_new_id();
decision.fill_task(state, question, variant, files, ctx_server.mctx, ctx_server.init_opt, task);
tasks.push_back(std::move(task));
}
}
if (decision.can_share_prompt()) {
tasks = server_decision_group_tasks(std::move(tasks), params.n_parallel);
@@ -5367,11 +5390,16 @@ void server_routes::init_routes() {
json answers = json::object();
int32_t n_tokens = 0;
for (size_t i = 0; i < questions.size(); i++) {
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[i].get());
GGML_ASSERT(result != nullptr);
answers[questions[i].id] = decision.format_answer(questions[i], result->scores);
n_tokens += result->n_tokens;
size_t i_result = 0;
for (const auto & question : questions) {
std::vector<std::vector<float>> scores;
for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[i_result++].get());
GGML_ASSERT(result != nullptr);
scores.push_back(result->scores);
n_tokens += result->n_tokens;
}
answers[question.id] = decision.format_answer(question, scores);
}
res->ok(json{