diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp
index 6202e2aad3..323173ffb7 100644
--- a/tools/server/server-models.cpp
+++ b/tools/server/server-models.cpp
@@ -672,21 +672,24 @@ void server_models::load_models() {
         apply_hidden();
         log_available_models();
 
-        std::vector<std::string> models_to_load;
-        for (const auto & [name, inst] : mapping) {
-            std::string val;
-            if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
-                models_to_load.push_back(name);
+        // skipped on reload, see startup_models
+        if (startup_models.has_value()) {
+            std::vector<std::string> models_to_load;
+            for (const auto & [name, inst] : mapping) {
+                std::string val;
+                if (inst.meta.preset.get_option(COMMON_ARG_PRESET_LOAD_ON_STARTUP, val) && common_arg_utils::is_truthy(val)) {
+                    models_to_load.push_back(name);
+                }
+            }
+            if ((int)models_to_load.size() > base_params.models_max) {
+                throw std::runtime_error(string_format(
+                    "number of models to load on startup (%zu) exceeds models_max (%d)",
+                    models_to_load.size(), base_params.models_max));
             }
-        }
-        if ((int)models_to_load.size() > base_params.models_max) {
-            throw std::runtime_error(string_format(
-                "number of models to load on startup (%zu) exceeds models_max (%d)",
-                models_to_load.size(), base_params.models_max));
-        }
 
-        // to be lazy-loaded after main() setup phase is completed
-        startup_models = std::move(models_to_load);
+            // to be lazy-loaded after main() setup phase is completed
+            startup_models = std::move(models_to_load);
+        }
 
         lk.unlock();
     } else {
@@ -880,8 +883,11 @@ void server_models::load_startup_models() {
     std::vector<std::string> to_load;
     {
         std::lock_guard<std::mutex> lk(mutex);
-        to_load = std::move(startup_models);
-        startup_models.clear();
+        if (!startup_models.has_value()) {
+            return; // already drained
+        }
+        to_load = std::move(*startup_models);
+        startup_models.reset();
     }
     for (const auto & name : to_load) {
         SRV_INF("(startup) loading model %s\n", name.c_str());
diff --git a/tools/server/server-models.h b/tools/server/server-models.h
index 0a3e3b3a42..6f737d8c74 100644
--- a/tools/server/server-models.h
+++ b/tools/server/server-models.h
@@ -136,8 +136,9 @@ private:
     // if true, the next get_meta() will trigger a reload of model list
     bool need_reload = false;
 
-    // models marked with load-on-startup
-    std::vector<std::string> startup_models;
+    // models marked with load-on-startup, unset once load_startup_models() drains it
+    // using std::optional to make sure it's only called once after is_first_load, load_startup_models will unset it
+    std::optional<std::vector<std::string>> startup_models{std::in_place};
 
     // conv_id -> model name that currently serves its stream session, lets the resumable stream
     // routes go straight to the owning child instead of polling every one. populated when