upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-09-30 09:57:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

only allow is_first_load to populate it

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-08-20 12:33:28 +02:00
parent c47322cc8a
commit 166cad8af6
2 changed files with 24 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-models.cpp
+21 -15
View File
@@ -672,21 +672,24 @@ void server_models::load_models() {
apply_hidden();
log_available_models();
std::vector<std::string> models_to_load;
for (const auto & [name, inst] : mapping) {
std::string val;
if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
models_to_load.push_back(name);
// skipped on reload, see startup_models
if (startup_models.has_value()) {
std::vector<std::string> models_to_load;
for (const auto & [name, inst] : mapping) {
std::string val;
if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
models_to_load.push_back(name);
}
}
if ((int)models_to_load.size() > base_params.models_max) {
throw std::runtime_error(string_format(
"number of models to load on startup (%zu) exceeds models_max (%d)",
models_to_load.size(), base_params.models_max));
}
}
if ((int)models_to_load.size() > base_params.models_max) {
throw std::runtime_error(string_format(
"number of models to load on startup (%zu) exceeds models_max (%d)",
models_to_load.size(), base_params.models_max));
}
// to be lazy-loaded after main() setup phase is completed
startup_models = std::move(models_to_load);
// to be lazy-loaded after main() setup phase is completed
startup_models = std::move(models_to_load);
}
lk.unlock();
} else {
@@ -880,8 +883,11 @@ void server_models::load_startup_models() {
std::vector<std::string> to_load;
{
std::lock_guard<std::mutex> lk(mutex);
to_load = std::move(startup_models);
startup_models.clear();
if (!startup_models.has_value()) {
return; // already drained
}
to_load = std::move(*startup_models);
startup_models.reset();
}
for (const auto & name : to_load) {
SRV_INF("(startup) loading model %s\n", name.c_str());
tools/server/server-models.h
+3 -2
View File
@@ -136,8 +136,9 @@ private:
// if true, the next get_meta() will trigger a reload of model list
bool need_reload = false;
// models marked with load-on-startup
std::vector<std::string> startup_models;
// models marked with load-on-startup, unset once load_startup_models() drains it
// using std::optional to make sure it's only called once after is_first_load, load_startup_models will unset it
std::optional<std::vector<std::string>> startup_models{std::in_place};
// conv_id -> model name that currently serves its stream session, lets the resumable stream
// routes go straight to the owning child instead of polling every one. populated when