upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-30 06:30:49 -05:00
Code Issues Packages Projects Releases Wiki Activity

llama: move suppress_tokens handling to common/sampling (#26276)

Browse Source 
* llama: move suppress_tokens handling to common/sampling

* address security issues

* rm has_logit_bias
This commit is contained in:
Xuan-Son Nguyen
2026-07-29 18:02:30 +02:00
committed by GitHub
parent caa596ab3f
commit afeebe103b
5 changed files with 32 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
common/common.h
View File

@@ -294,10 +294,6 @@ struct common_params_sampling {
bool backend_sampling = false;
bool has_logit_bias() const {
return !logit_bias.empty();
}
// print the parameters into a string
std::string print() const;
};

15
common/sampling.cpp
View File

@@ -310,8 +310,19 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, st
}
}
if (params.has_logit_bias()) {
samplers.push_back(llama_sampler_init_logit_bias(llama_vocab_n_tokens(vocab), params.logit_bias.size(), params.logit_bias.data()));
// logit bias: user biases + model suppress tokens (-INFINITY)
{
std::vector<llama_logit_bias> merged = params.logit_bias;
int32_t n_suppress = 0;
const llama_token * suppress = llama_vocab_get_suppress_tokens(vocab, &n_suppress);
for (int32_t i = 0; i < n_suppress; ++i) {
merged.push_back({ suppress[i], -INFINITY });
}
if (!merged.empty()) {
samplers.push_back(llama_sampler_init_logit_bias(llama_vocab_n_tokens(vocab), merged.size(), merged.data()));
}
}
if (params.mirostat == 0) {

3
include/llama.h
View File

@@ -1102,6 +1102,9 @@ extern "C" {
LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab);
LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab);
// model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens)
LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens);
LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab);
LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab);
LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab);

17
src/llama-vocab.cpp
View File

@@ -2578,7 +2578,14 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
if (suppress_idx != -1) {
const int n = gguf_get_arr_n(ctx, suppress_idx);
const int32_t * data = (const int32_t *) gguf_get_arr_data(ctx, suppress_idx);
suppress_tokens.assign(data, data + n);
// drop out-of-range ids
suppress_tokens.reserve(n);
for (int i = 0; i < n; ++i) {
const int32_t id = data[i];
if (id >= 0 && id < (int) id_to_token.size()) {
suppress_tokens.push_back(id);
}
}
}
}
@@ -4205,6 +4212,14 @@ bool llama_vocab_get_add_sep(const struct llama_vocab * vocab) {
return vocab->get_add_sep();
}
const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens) {
const std::vector<llama_token> & tokens = vocab->get_suppress_tokens();
if (n_suppress_tokens) {
*n_suppress_tokens = (int32_t) tokens.size();
}
return tokens.data();
}
llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab) {
return vocab->token_fim_pre();
}

37
src/models/gemma4.cpp
View File

@@ -142,33 +142,6 @@ static ggml_tensor * ggml_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, in
idx * x->ne[0] * x->ne[1] * ggml_element_size(x));
}
// TODO @ngxson : maybe improve this in the future
class llm_graph_input_logits_bias : public llm_graph_input_i {
public:
llm_graph_input_logits_bias(const llama_vocab & vocab) {
arr.resize(vocab.n_tokens(), 0.0f);
for (llama_token id : vocab.get_suppress_tokens()) {
if (0 <= id && id < (int32_t)vocab.n_tokens()) {
arr[id] = -INFINITY;
}
}
}
virtual ~llm_graph_input_logits_bias() = default;
void set_input(const llama_ubatch * /*ubatch*/) override {
const int64_t n_vocab = arr.size();
ggml_backend_tensor_set(logits_bias, arr.data(), 0, n_vocab*ggml_element_size(logits_bias));
}
bool can_reuse(const llm_graph_params & /*params*/) override {
return true;
}
ggml_tensor * logits_bias = nullptr; // F32 [n_vocab]
std::vector<float> arr;
};
llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_params & params) :
llm_graph_context(params),
model(model),
@@ -429,16 +402,6 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para
cur = ggml_scale(ctx0, cur, hparams.f_final_logit_softcapping);
}
// apply logits bias if needed (e.g. for gemma4_unified patch)
// this is to mirror the suppress_tokens patch on transformers, to avoid model from outputing <image|> and <audio|> tokens (which is a known issue related to the checkpoint)
// TODO: maybe handle this inside the sampling system in the future
if (!model.vocab.get_suppress_tokens().empty()) {
auto inp_bias = std::make_unique<llm_graph_input_logits_bias>(model.vocab);
inp_bias->logits_bias = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, inp_bias->arr.size());
cur = ggml_add(ctx0, cur, inp_bias->logits_bias);
res->add_input(std::move(inp_bias));
}
cb(cur, "result_output", -1);
res->t_logits = cur;