mirror of
https://github.com/ggml-org/llama.cpp.git
synced 2026-10-02 02:47:26 -05:00
695 lines
26 KiB
C++
695 lines
26 KiB
C++
#include "server-decision.h"
|
|
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <stdexcept>
|
|
|
|
static const char * decision_question_type_name(server_decision_question_type type) {
|
|
switch (type) {
|
|
case SERVER_DECISION_QUESTION_CHOICE: return "choice";
|
|
case SERVER_DECISION_QUESTION_SCORE: return "score";
|
|
case SERVER_DECISION_QUESTION_NOUL: return "noul";
|
|
}
|
|
return "";
|
|
}
|
|
|
|
static std::string decision_meta_str(const llama_model * model, const std::string & key) {
|
|
char buf[256];
|
|
const int32_t n = llama_model_meta_val_str(model, key.c_str(), buf, sizeof(buf));
|
|
return n < 0 ? "" : std::string(buf);
|
|
}
|
|
|
|
//
|
|
// model-specific setup
|
|
//
|
|
|
|
void server_decision_context::init(const llama_model * model) {
|
|
*this = server_decision_context(); // the model can be reloaded
|
|
|
|
const common_decision_type model_type = common_get_decision_type(model);
|
|
if (model_type == COMMON_DECISION_TYPE_NONE) {
|
|
return;
|
|
}
|
|
|
|
const std::string prefix = decision_meta_str(model, "general.architecture") + ".decision.";
|
|
const std::string type_name = decision_meta_str(model, prefix + "type");
|
|
|
|
vocab = llama_model_get_vocab(model);
|
|
|
|
const char * tmpl_src = llama_model_chat_template(model, "systemone");
|
|
if (tmpl_src == nullptr) {
|
|
throw std::runtime_error("decision model has no \"systemone\" template");
|
|
}
|
|
tmpl = std::make_shared<const common_chat_template>(tmpl_src, "", "");
|
|
|
|
const std::string prefix_temp = prefix + "temperature.";
|
|
for (int32_t i = 0; i < llama_model_meta_count(model); i++) {
|
|
char key[256];
|
|
char val[64];
|
|
if (llama_model_meta_key_by_index(model, i, key, sizeof(key)) < 0 || !string_starts_with(key, prefix_temp)) {
|
|
continue;
|
|
}
|
|
if (llama_model_meta_val_str_by_index(model, i, val, sizeof(val)) < 0) {
|
|
continue;
|
|
}
|
|
const float temp = std::strtof(val, nullptr);
|
|
if (temp <= 0.0f) {
|
|
throw std::runtime_error(string_format("invalid decision temperature: %s = %s", key, val));
|
|
}
|
|
temperatures[key + prefix_temp.size()] = temp;
|
|
}
|
|
|
|
if (model_type == COMMON_DECISION_TYPE_OPENJEV) {
|
|
// one letter per option, each must be a single token
|
|
const std::string letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
|
|
for (const char c : letters) {
|
|
const auto toks = common_tokenize(vocab, std::string(1, c), false, false);
|
|
if (toks.size() != 1) {
|
|
throw std::runtime_error(string_format("decision label '%c' is not a single token", c));
|
|
}
|
|
labels.push_back(toks[0]);
|
|
}
|
|
n_options_max = labels.size();
|
|
noul_true_first = true;
|
|
} else if (model_type == COMMON_DECISION_TYPE_LAYA) {
|
|
token_marker = llama_vocab_mask(vocab);
|
|
token_sep = llama_vocab_sep(vocab);
|
|
if (token_marker == LLAMA_TOKEN_NULL || token_sep == LLAMA_TOKEN_NULL) {
|
|
throw std::runtime_error("decision model has no mask or sep token");
|
|
}
|
|
text_marker = common_token_to_piece(vocab, token_marker, true);
|
|
|
|
const std::string val = decision_meta_str(model, prefix + "max_head_tokens");
|
|
max_head_tokens = std::strtoul(val.c_str(), nullptr, 10);
|
|
if (max_head_tokens == 0) {
|
|
throw std::runtime_error("decision model has no valid max_head_tokens");
|
|
}
|
|
n_options_max = 255;
|
|
} else if (model_type == COMMON_DECISION_TYPE_CLEF) {
|
|
n_options_max = 255;
|
|
noul_true_first = true;
|
|
} else {
|
|
throw std::runtime_error("unsupported decision model type: " + type_name);
|
|
}
|
|
type = model_type;
|
|
|
|
SRV_INF("decision model type: %s\n", type_name.c_str());
|
|
}
|
|
|
|
//
|
|
// request parsing
|
|
//
|
|
|
|
std::vector<server_decision_question> server_decision_context::parse_questions(const json & body) const {
|
|
if (!body.contains("state") || body.at("state").is_null()) {
|
|
throw std::invalid_argument("\"state\" must be provided");
|
|
}
|
|
if (!body.contains("questions") || !body.at("questions").is_object() || body.at("questions").empty()) {
|
|
throw std::invalid_argument("\"questions\" must be a non-empty object");
|
|
}
|
|
|
|
std::vector<server_decision_question> questions;
|
|
for (const auto & [id, q] : body.at("questions").items()) {
|
|
auto err = [&id = id](const std::string & msg) {
|
|
return std::invalid_argument("questions." + id + ": " + msg);
|
|
};
|
|
if (!q.is_object()) {
|
|
throw err("must be an object");
|
|
}
|
|
if (!q.contains("instructions") || q.at("instructions").is_null()) {
|
|
throw err("\"instructions\" must be provided");
|
|
}
|
|
|
|
server_decision_question question;
|
|
question.id = id;
|
|
question.instructions = q.at("instructions");
|
|
|
|
const std::string type_name = json_value(q, "type", std::string());
|
|
const json criteria = q.contains("criteria") ? q.at("criteria") : json();
|
|
|
|
if (type_name == "choice") {
|
|
question.type = SERVER_DECISION_QUESTION_CHOICE;
|
|
if (!criteria.is_object() || criteria.empty()) {
|
|
throw err("\"criteria\" must be a non-empty object");
|
|
}
|
|
for (const auto & [key, description] : criteria.items()) {
|
|
question.options.push_back({key, description});
|
|
}
|
|
} else if (type_name == "score") {
|
|
question.type = SERVER_DECISION_QUESTION_SCORE;
|
|
if (!criteria.is_array() || criteria.size() < 2 || criteria.size() > 10) {
|
|
throw err("\"criteria\" must be an array of 2 to 10 levels");
|
|
}
|
|
for (size_t i = 0; i < criteria.size(); i++) {
|
|
question.options.push_back({std::to_string(i), criteria.at(i)});
|
|
}
|
|
} else if (type_name == "noul") {
|
|
question.type = SERVER_DECISION_QUESTION_NOUL;
|
|
if (!criteria.is_null() && !criteria.is_object()) {
|
|
throw err("\"criteria\" must be an object");
|
|
}
|
|
for (const char * key : {"false", "true"}) {
|
|
question.options.push_back({key, criteria.is_object() && criteria.contains(key) ? criteria.at(key) : json()});
|
|
}
|
|
if (noul_true_first) {
|
|
std::swap(question.options[0], question.options[1]);
|
|
}
|
|
} else {
|
|
throw err("\"type\" must be one of: choice, score, noul");
|
|
}
|
|
|
|
if (question.options.size() > n_options_max) {
|
|
throw err(string_format("too many options (%zu), this model supports at most %zu", question.options.size(), n_options_max));
|
|
}
|
|
|
|
questions.push_back(std::move(question));
|
|
}
|
|
return questions;
|
|
}
|
|
|
|
//
|
|
// images
|
|
//
|
|
|
|
static const size_t DECISION_MAX_IMAGES = 8;
|
|
|
|
static void decision_load_image(const json & url, std::vector<raw_buffer> & files) {
|
|
if (!url.is_string() || !string_starts_with(url.get<std::string>(), "data:image/")) {
|
|
throw std::invalid_argument("images must be data URLs (data:image/...;base64,...)");
|
|
}
|
|
if (files.size() >= DECISION_MAX_IMAGES) {
|
|
throw std::invalid_argument(string_format("too many images, the maximum is %zu", DECISION_MAX_IMAGES));
|
|
}
|
|
handle_media(files, url.get<std::string>(), "");
|
|
}
|
|
|
|
json server_decision_context::parse_state(const json & body, std::vector<raw_buffer> & files) const {
|
|
if (body.contains("images") && !body.at("images").is_null()) {
|
|
if (!body.at("images").is_array()) {
|
|
throw std::invalid_argument("\"images\" must be an array");
|
|
}
|
|
for (const auto & url : body.at("images")) {
|
|
decision_load_image(url, files);
|
|
}
|
|
}
|
|
|
|
const json & state = body.at("state");
|
|
const bool is_wrapped = state.is_object() && state.contains("messages");
|
|
const json & messages = is_wrapped ? state.at("messages") : state;
|
|
if (!messages.is_array()) {
|
|
return state;
|
|
}
|
|
|
|
// chat messages: take the image parts out of the content
|
|
json messages_out = json::array();
|
|
for (const auto & msg : messages) {
|
|
if (!msg.is_object() || !msg.contains("content") || !msg.at("content").is_array()) {
|
|
messages_out.push_back(msg);
|
|
continue;
|
|
}
|
|
json content = json::array();
|
|
for (const auto & part : msg.at("content")) {
|
|
if (part.is_object() && json_value(part, "type", std::string()) == "image_url" && part.contains("image_url")) {
|
|
const json & image_url = part.at("image_url");
|
|
decision_load_image(image_url.is_object() && image_url.contains("url") ? image_url.at("url") : image_url, files);
|
|
} else {
|
|
content.push_back(part);
|
|
}
|
|
}
|
|
json msg_out = msg;
|
|
msg_out["content"] = content;
|
|
messages_out.push_back(msg_out);
|
|
}
|
|
|
|
if (!is_wrapped) {
|
|
return messages_out;
|
|
}
|
|
json state_out = state;
|
|
state_out["messages"] = messages_out;
|
|
return state_out;
|
|
}
|
|
|
|
//
|
|
// prompt
|
|
//
|
|
|
|
// replace text in all strings of a JSON value
|
|
static json decision_replace_text(const json & val, const std::string & search, const std::string & replace) {
|
|
if (val.is_string()) {
|
|
std::string str = val.get<std::string>();
|
|
string_replace_all(str, search, replace);
|
|
return str;
|
|
}
|
|
if (val.is_array()) {
|
|
json out = json::array();
|
|
for (const auto & item : val) {
|
|
out.push_back(decision_replace_text(item, search, replace));
|
|
}
|
|
return out;
|
|
}
|
|
if (val.is_object()) {
|
|
json out = json::object();
|
|
for (const auto & [key, item] : val.items()) {
|
|
out[key] = decision_replace_text(item, search, replace);
|
|
}
|
|
return out;
|
|
}
|
|
return val;
|
|
}
|
|
|
|
std::string server_decision_context::render(const json & state, const server_decision_question & question, size_t n_images) const {
|
|
json options = json::array();
|
|
for (const auto & opt : question.options) {
|
|
options.push_back(json{
|
|
{"key", opt.key},
|
|
{"description", opt.description},
|
|
});
|
|
}
|
|
|
|
// the template is given raw JSON values, it serializes the ones that are not strings
|
|
json inp = json{
|
|
{"type", decision_question_type_name(question.type)},
|
|
{"instructions", question.instructions},
|
|
{"state", state},
|
|
{"options", options},
|
|
};
|
|
|
|
// the input must not contain the marker of the options
|
|
if (!text_marker.empty()) {
|
|
inp = decision_replace_text(inp, text_marker, " ");
|
|
}
|
|
|
|
// the template puts one media marker per image
|
|
json images = json::array();
|
|
if (n_images > 0) {
|
|
inp = decision_replace_text(inp, get_media_marker(), " ");
|
|
for (size_t i = 0; i < n_images; i++) {
|
|
images.push_back(get_media_marker());
|
|
}
|
|
}
|
|
inp["images"] = images;
|
|
|
|
jinja::context ctx(tmpl->source());
|
|
jinja::global_from_json(ctx, inp, false);
|
|
jinja::runtime runtime(ctx);
|
|
const jinja::value results = runtime.execute(tmpl->prog);
|
|
return jinja::runtime::gather_string_parts(results)->as_string().str();
|
|
}
|
|
|
|
void server_decision_context::fill_task(
|
|
const json & state,
|
|
const server_decision_question & question,
|
|
const std::vector<raw_buffer> & files,
|
|
mtmd_context * mctx,
|
|
const mtmd_helper_init_opt & init_opt,
|
|
server_task & task) const {
|
|
const std::string prompt = render(state, question, files.size());
|
|
|
|
if (type == COMMON_DECISION_TYPE_OPENJEV) {
|
|
task.decision.labels.assign(labels.begin(), labels.begin() + question.options.size());
|
|
if (!files.empty()) {
|
|
task.tokens = process_mtmd_prompt(mctx, prompt, files, init_opt);
|
|
return;
|
|
}
|
|
}
|
|
|
|
llama_tokens tokens = common_tokenize(vocab, prompt, false, true);
|
|
if (type == COMMON_DECISION_TYPE_LAYA) {
|
|
fill_task_laya(tokens, question, task);
|
|
}
|
|
task.tokens = server_tokens(tokens, false);
|
|
}
|
|
|
|
// the prompt is: [cls] question [sep] ([marker] option)* [sep] state [sep]
|
|
// options and question are cut to fit max_head_tokens, the same way the model was trained
|
|
void server_decision_context::fill_task_laya(llama_tokens & tokens, const server_decision_question & question, server_task & task) const {
|
|
const size_t n_options = question.options.size();
|
|
|
|
std::vector<size_t> markers;
|
|
for (size_t i = 0; i < tokens.size(); i++) {
|
|
if (tokens[i] == token_marker) {
|
|
markers.push_back(i);
|
|
}
|
|
}
|
|
const auto invalid = std::runtime_error("unexpected layout of the decision prompt");
|
|
if (markers.size() != n_options || markers[0] < 2 || tokens[markers[0] - 1] != token_sep || tokens.back() != token_sep) {
|
|
throw invalid;
|
|
}
|
|
const size_t head_end = markers[0] - 1;
|
|
const size_t opts_end = std::find(tokens.begin() + markers.back(), tokens.end(), token_sep) - tokens.begin();
|
|
if (opts_end + 1 >= tokens.size()) {
|
|
throw invalid;
|
|
}
|
|
|
|
// marker + text of each option
|
|
std::vector<llama_tokens> options;
|
|
size_t n_options_tokens = 0;
|
|
auto set_max = [&](size_t n_max) {
|
|
n_options_tokens = 0;
|
|
for (auto & opt : options) {
|
|
opt.resize(std::min(opt.size(), n_max));
|
|
n_options_tokens += opt.size();
|
|
}
|
|
};
|
|
for (size_t i = 0; i < n_options; i++) {
|
|
const size_t end = i + 1 < n_options ? markers[i + 1] : opts_end;
|
|
options.emplace_back(tokens.begin() + markers[i], tokens.begin() + end);
|
|
}
|
|
set_max(max_option_tokens + 1);
|
|
if (n_options_tokens + 16 > max_head_tokens) {
|
|
// too many or too long options, shrink them evenly
|
|
set_max(std::max((size_t) 4, (max_head_tokens - std::min(max_head_tokens, (size_t) 16)) / n_options));
|
|
}
|
|
const size_t n_question_max = std::max((size_t) 8, max_head_tokens - std::min(max_head_tokens, n_options_tokens));
|
|
|
|
llama_tokens out;
|
|
out.push_back(tokens[0]);
|
|
out.insert(out.end(), tokens.begin() + 1, tokens.begin() + std::min(head_end, 1 + n_question_max));
|
|
out.push_back(token_sep);
|
|
for (const auto & opt : options) {
|
|
task.decision.markers.push_back(out.size());
|
|
out.insert(out.end(), opt.begin(), opt.end());
|
|
}
|
|
out.insert(out.end(), tokens.begin() + opts_end, tokens.end());
|
|
tokens = std::move(out);
|
|
|
|
// the output has one score per question type
|
|
task.decision.column = question.type;
|
|
}
|
|
|
|
//
|
|
// joint prompt (clef)
|
|
//
|
|
|
|
// the template separates the pieces of the prompt with these, they are tokenized one by one
|
|
static const std::string CLEF_MARKER = "<<clef:";
|
|
static const std::string CLEF_PIECE_SEP = "<<clef:sep>>";
|
|
static const std::string CLEF_PIECE_STATE = "<<clef:state>>";
|
|
static const std::string CLEF_PIECE_QUESTION = "<<clef:question>>";
|
|
static const std::string CLEF_PIECE_OPTION = "<<clef:option>>";
|
|
|
|
// same value with the keys of all objects sorted
|
|
static json decision_sort_keys(const json & val) {
|
|
if (val.is_array()) {
|
|
json out = json::array();
|
|
for (const auto & item : val) {
|
|
out.push_back(decision_sort_keys(item));
|
|
}
|
|
return out;
|
|
}
|
|
if (val.is_object()) {
|
|
std::map<std::string, json> sorted;
|
|
for (const auto & [key, item] : val.items()) {
|
|
sorted[key] = decision_sort_keys(item);
|
|
}
|
|
json out = json::object();
|
|
for (const auto & [key, item] : sorted) {
|
|
out[key] = item;
|
|
}
|
|
return out;
|
|
}
|
|
return val;
|
|
}
|
|
|
|
// strings are used as is, other values are compact JSON with sorted keys
|
|
static std::string clef_render(const json & val) {
|
|
return val.is_string() ? val.get<std::string>() : decision_sort_keys(val).dump();
|
|
}
|
|
|
|
std::vector<size_t> server_decision_context::prompt_order(const server_decision_question & question) const {
|
|
std::vector<size_t> order(question.options.size());
|
|
for (size_t i = 0; i < order.size(); i++) {
|
|
order[i] = i;
|
|
}
|
|
if (type == COMMON_DECISION_TYPE_CLEF && question.type == SERVER_DECISION_QUESTION_CHOICE) {
|
|
std::sort(order.begin(), order.end(), [&](size_t a, size_t b) {
|
|
return question.options[a].key < question.options[b].key;
|
|
});
|
|
}
|
|
return order;
|
|
}
|
|
|
|
void server_decision_context::fill_task_joint(const json & state, const std::vector<server_decision_question> & questions, server_task & task) const {
|
|
auto clean = [](const json & val) {
|
|
return decision_replace_text(val, CLEF_MARKER, "<<clef ");
|
|
};
|
|
|
|
json inp_questions = json::array();
|
|
for (const auto & question : questions) {
|
|
json options = json::array();
|
|
for (const size_t i : prompt_order(question)) {
|
|
const auto & opt = question.options[i];
|
|
|
|
json description = opt.description;
|
|
if (description.is_null() && question.type == SERVER_DECISION_QUESTION_NOUL) {
|
|
description = opt.key == "true"
|
|
? "The proposition is true or the answer is yes."
|
|
: "The proposition is false or the answer is no.";
|
|
}
|
|
|
|
// keys in sorted order
|
|
json semantics = json::object();
|
|
if (!description.is_null()) {
|
|
semantics["description"] = decision_sort_keys(description);
|
|
}
|
|
semantics["option_id"] = opt.key;
|
|
|
|
options.push_back(json{{"text", semantics.dump()}});
|
|
}
|
|
|
|
inp_questions.push_back(json{
|
|
{"id", question.id},
|
|
{"type", decision_question_type_name(question.type)},
|
|
{"instructions", clef_render(question.instructions)},
|
|
{"options", options},
|
|
});
|
|
}
|
|
|
|
const json inp = clean(json{
|
|
{"state", clef_render(state)},
|
|
{"questions", inp_questions},
|
|
});
|
|
|
|
jinja::context ctx(tmpl->source());
|
|
jinja::global_from_json(ctx, inp, false);
|
|
jinja::runtime runtime(ctx);
|
|
const jinja::value results = runtime.execute(tmpl->prog);
|
|
const std::string prompt = jinja::runtime::gather_string_parts(results)->as_string().str();
|
|
|
|
// the model was trained with the pieces tokenized one by one
|
|
llama_tokens tokens;
|
|
int32_t i_question = -1;
|
|
size_t n_questions = 0;
|
|
size_t n_options = 0;
|
|
for (std::string piece : string_split(prompt, CLEF_PIECE_SEP)) {
|
|
enum { PIECE_TEXT, PIECE_QUESTION, PIECE_OPTION } kind = PIECE_TEXT;
|
|
if (string_starts_with(piece, CLEF_PIECE_QUESTION)) {
|
|
piece = piece.substr(CLEF_PIECE_QUESTION.size());
|
|
kind = PIECE_QUESTION;
|
|
} else if (string_starts_with(piece, CLEF_PIECE_OPTION)) {
|
|
piece = piece.substr(CLEF_PIECE_OPTION.size());
|
|
kind = PIECE_OPTION;
|
|
} else if (string_starts_with(piece, CLEF_PIECE_STATE)) {
|
|
piece = piece.substr(CLEF_PIECE_STATE.size());
|
|
}
|
|
|
|
const int32_t start = tokens.size();
|
|
const llama_tokens piece_tokens = common_tokenize(vocab, piece, false, true);
|
|
tokens.insert(tokens.end(), piece_tokens.begin(), piece_tokens.end());
|
|
const int32_t end = tokens.size();
|
|
|
|
if (kind == PIECE_TEXT) {
|
|
continue;
|
|
}
|
|
if (start == end) {
|
|
throw std::invalid_argument("the instructions and the options of a question must not be empty");
|
|
}
|
|
// see llama_batch_ext_set_decision_order(): question of type noul, choice, score, or option
|
|
int32_t order = 4;
|
|
if (kind == PIECE_QUESTION) {
|
|
i_question++;
|
|
switch (questions.at(i_question).type) {
|
|
case SERVER_DECISION_QUESTION_NOUL: order = 1; break;
|
|
case SERVER_DECISION_QUESTION_CHOICE: order = 2; break;
|
|
case SERVER_DECISION_QUESTION_SCORE: order = 3; break;
|
|
}
|
|
n_questions++;
|
|
} else {
|
|
// the score of option i is returned at row i
|
|
task.decision.markers.push_back(n_options);
|
|
n_options++;
|
|
}
|
|
task.decision.order.resize(tokens.size(), 0);
|
|
std::fill(task.decision.order.begin() + start, task.decision.order.end(), order);
|
|
}
|
|
task.decision.order.resize(tokens.size(), 0);
|
|
|
|
size_t n_options_exp = 0;
|
|
for (const auto & question : questions) {
|
|
n_options_exp += question.options.size();
|
|
}
|
|
if (n_questions != questions.size() || n_options != n_options_exp) {
|
|
throw std::runtime_error("unexpected layout of the decision prompt");
|
|
}
|
|
|
|
task.decision.column = 0;
|
|
task.tokens = server_tokens(tokens, false);
|
|
}
|
|
|
|
std::vector<std::vector<float>> server_decision_context::split_scores(
|
|
const std::vector<server_decision_question> & questions,
|
|
const std::vector<float> & scores) const {
|
|
std::vector<std::vector<float>> result;
|
|
size_t offset = 0;
|
|
for (const auto & question : questions) {
|
|
const auto order = prompt_order(question);
|
|
if (offset + order.size() > scores.size()) {
|
|
throw std::runtime_error("decision result does not match the number of options");
|
|
}
|
|
std::vector<float> cur(order.size());
|
|
for (size_t i = 0; i < order.size(); i++) {
|
|
cur[order[i]] = scores[offset + i];
|
|
}
|
|
offset += order.size();
|
|
result.push_back(std::move(cur));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
//
|
|
// answer
|
|
//
|
|
|
|
float server_decision_context::get_temperature(const server_decision_question & question) const {
|
|
const size_t n = question.options.size();
|
|
const std::string type_name = decision_question_type_name(question.type);
|
|
const std::string bucket = n <= 2 ? "2" : n <= 5 ? "3_5" : n <= 10 ? "6_10" : "11";
|
|
|
|
for (const auto & name : {type_name + "." + bucket, type_name}) {
|
|
const auto it = temperatures.find(name);
|
|
if (it != temperatures.end()) {
|
|
return it->second;
|
|
}
|
|
}
|
|
return 1.0f;
|
|
}
|
|
|
|
// confidence formulas are the ones published by TypeSafe
|
|
|
|
static double decision_confidence_choice(const std::vector<double> & probs) {
|
|
if (probs.size() < 2) {
|
|
return 1.0;
|
|
}
|
|
const double uniform = 1.0 / probs.size();
|
|
const double p_max = *std::max_element(probs.begin(), probs.end());
|
|
return std::max(0.0, (p_max - uniform) / (1.0 - uniform));
|
|
}
|
|
|
|
static double decision_confidence_score(const std::vector<double> & probs) {
|
|
if (probs.size() < 2) {
|
|
return 1.0;
|
|
}
|
|
const size_t n = probs.size();
|
|
const size_t mode = std::max_element(probs.begin(), probs.end()) - probs.begin();
|
|
|
|
// mean distance to the mode, relative to the one of a uniform distribution around its center
|
|
double dist = 0.0;
|
|
double dist_uniform = 0.0;
|
|
for (size_t i = 0; i < n; i++) {
|
|
dist += probs[i] * std::fabs((double) i - (double) mode);
|
|
dist_uniform += std::fabs((double) i - (n - 1) / 2.0) / n;
|
|
}
|
|
return std::max(0.0, 1.0 - dist / dist_uniform);
|
|
}
|
|
|
|
json server_decision_context::format_answer(const server_decision_question & question, const std::vector<float> & scores) const {
|
|
const size_t n = question.options.size();
|
|
if (scores.size() != n) {
|
|
throw std::runtime_error("decision result does not match the number of options");
|
|
}
|
|
|
|
// softmax over the options
|
|
const float temperature = get_temperature(question);
|
|
const float score_max = *std::max_element(scores.begin(), scores.end());
|
|
std::vector<double> probs(n);
|
|
double sum = 0.0;
|
|
for (size_t i = 0; i < n; i++) {
|
|
probs[i] = std::exp((double) (scores[i] - score_max) / temperature);
|
|
sum += probs[i];
|
|
}
|
|
for (auto & p : probs) {
|
|
p /= sum;
|
|
}
|
|
|
|
json answer = json{{"type", decision_question_type_name(question.type)}};
|
|
|
|
if (question.type == SERVER_DECISION_QUESTION_NOUL) {
|
|
for (size_t i = 0; i < n; i++) {
|
|
if (question.options[i].key == "true") {
|
|
answer["noul"] = probs[i];
|
|
}
|
|
}
|
|
return answer;
|
|
}
|
|
|
|
json probabilities = json::object();
|
|
for (size_t i = 0; i < n; i++) {
|
|
probabilities[question.options[i].key] = probs[i];
|
|
}
|
|
|
|
if (question.type == SERVER_DECISION_QUESTION_CHOICE) {
|
|
const size_t best = std::max_element(probs.begin(), probs.end()) - probs.begin();
|
|
answer["choice"] = question.options[best].key;
|
|
answer["probabilities"] = probabilities;
|
|
answer["confidence"] = decision_confidence_choice(probs);
|
|
} else {
|
|
double expected = 0.0;
|
|
json legend = json::object();
|
|
for (size_t i = 0; i < n; i++) {
|
|
expected += i * probs[i];
|
|
legend[question.options[i].key] = question.options[i].description;
|
|
}
|
|
answer["score"] = expected;
|
|
answer["legend"] = legend;
|
|
answer["probabilities"] = probabilities;
|
|
answer["confidence"] = decision_confidence_score(probs);
|
|
}
|
|
return answer;
|
|
}
|
|
|
|
//
|
|
// shared prompt prefix
|
|
//
|
|
|
|
std::vector<server_task> server_decision_group_tasks(std::vector<server_task> && tasks, size_t n_slots) {
|
|
n_slots = std::max(n_slots, (size_t) 1);
|
|
|
|
std::vector<server_task> groups;
|
|
for (size_t i = 0; i < tasks.size(); i += n_slots) {
|
|
const size_t end = std::min(tasks.size(), i + n_slots);
|
|
server_task & parent = tasks[i];
|
|
|
|
// every task must have at least one token of its own to evaluate
|
|
size_t n_shared = parent.tokens.size() - 1;
|
|
for (size_t j = i + 1; j < end; j++) {
|
|
n_shared = std::min(n_shared, parent.tokens.get_common_prefix(tasks[j].tokens));
|
|
n_shared = std::min(n_shared, tasks[j].tokens.size() - 1);
|
|
}
|
|
|
|
if (end - i < 2 || n_shared == 0) {
|
|
for (size_t j = i; j < end; j++) {
|
|
groups.push_back(std::move(tasks[j]));
|
|
}
|
|
continue;
|
|
}
|
|
|
|
parent.n_tokens_shared = n_shared;
|
|
for (size_t j = i + 1; j < end; j++) {
|
|
tasks[j].id_parent = parent.id;
|
|
parent.child_tasks.push_back(std::move(tasks[j]));
|
|
}
|
|
groups.push_back(std::move(parent));
|
|
}
|
|
return groups;
|
|
}
|