upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-29 09:27:33 -05:00
Code Issues Packages Projects Releases Wiki Activity

Add FP8 KV-cache support

Browse Source
This commit is contained in:
Oliver Simons
2026-09-04 20:51:13 +02:00
parent cef6c528f8
commit b6ccff9b0e
10 changed files with 146 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-model.cpp
+10
View File
@@ -1524,6 +1524,16 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
for (int i = 0; i < n_layer_all; ++i) {
auto & layer = layers[i];
if (hparams.has_kv(i)) {
layer.k_cache_scale = create_tensor(tn(LLM_TENSOR_ATTN_K, "k_scale", i), {1}, TENSOR_NOT_REQUIRED);
layer.v_cache_scale = create_tensor(tn(LLM_TENSOR_ATTN_V, "v_scale", i), {1}, TENSOR_NOT_REQUIRED);
if ((layer.k_cache_scale && layer.k_cache_scale->type != GGML_TYPE_F32) ||
(layer.v_cache_scale && layer.v_cache_scale->type != GGML_TYPE_F32)) {
throw std::runtime_error(format("KV cache scales for layer %d must be F32", i));
}
}
// attention weight scales (per-tensor, shape {1})
if (!layer.wq_s && layer.wq) {
layer.wq_s = create_tensor(tn(LLM_TENSOR_ATTN_Q, "scale", i), {1}, TENSOR_NOT_REQUIRED);