wip

ggml/src/ggml-backend-reg.cpp

@@ -106,6 +106,8 @@ static std::string path_str(const fs::path & path) {
 struct ggml_backend_reg_entry {
     ggml_backend_reg_t reg;
     dl_handle_ptr handle;
+    // we won't actually unload the backend because dlclose() may cause issues on some backends (e.g. CUDA), so we mark it here and reuse the handle later
+    bool unloaded = false;
 };
 
 struct ggml_backend_registry {
@@ -113,6 +115,10 @@ struct ggml_backend_registry {
     std::vector<ggml_backend_dev_t> devices;
 
     ggml_backend_registry() {
+        load_all_static();
+    }
+
+    void load_all_static() {
 #ifdef GGML_USE_CUDA
         register_backend(ggml_backend_cuda_reg());
 #endif
@@ -183,6 +189,13 @@ struct ggml_backend_registry {
 
         for (auto & entry : backends) {
             if (entry.reg == reg) {
+                if (entry.unloaded) {
+                    // reuse if needed
+                    entry.unloaded = false;
+                    for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); i++) {
+                        register_device(ggml_backend_reg_dev_get(reg, i));
+                    }
+                }
                 return;
             }
         }
@@ -267,11 +280,16 @@ struct ggml_backend_registry {
             return;
         }
 
+        if (it->unloaded) {
+            return;
+        }
+
         if (!silent) {
             GGML_LOG_DEBUG("%s: unloading %s backend\n", __func__, ggml_backend_reg_name(reg));
         }
 
-        // remove devices
+        // remove devices from the registry's bookkeeping
+        // note: the ggml_backend_dev_t objects are kept so the backend can be resumed later
         devices.erase(
             std::remove_if(devices.begin(), devices.end(),
                             [reg](ggml_backend_dev_t dev) { return ggml_backend_dev_backend_reg(dev) == reg; }),
@@ -282,8 +300,8 @@ struct ggml_backend_registry {
             reg->iface.free(reg);
         }
 
-        // remove backend
+        // mark as unloaded, so the handle can be reused later
-        backends.erase(it);
+        it->unloaded = true;
     }
 };
 
@@ -558,6 +576,7 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
 }
 
 void ggml_backend_load_all() {
+    get_reg().load_all_static();
     ggml_backend_load_all_from_path(nullptr);
 }
 

ggml/src/ggml-cuda/ggml-cuda.cu

@@ -5729,19 +5729,12 @@ static void ggml_backend_cuda_reg_free(ggml_backend_reg_t reg) {
 
     for (ggml_backend_dev_t dev : ctx->devices) {
         int device = ((ggml_backend_cuda_device_context *) dev->context)->device;
-        delete (ggml_backend_cuda_device_context *) dev->context;
-        delete dev;
 
         cudaError_t err = cudaSetDevice(device);
         if (err == cudaSuccess) {
             cudaDeviceReset();
         }
     }
-
-    delete ctx;
-    reg->context = nullptr;
-
-    ggml_backend_cuda_reg_initialized = false;
 }
 
 static const ggml_backend_reg_i ggml_backend_cuda_reg_interface = {

src/llama.cpp

@@ -96,9 +96,7 @@ void llama_backend_init(void) {
         ggml_free(ctx);
     }
 
-    if (!ggml_backend_reg_count()) {
+    ggml_backend_load_all();
-        ggml_backend_load_all();
-    }
 }
 
 void llama_numa_init(enum ggml_numa_strategy numa) {
@@ -115,9 +113,8 @@ void llama_numa_init(enum ggml_numa_strategy numa) {
 
 void llama_backend_free(void) {
     ggml_quantize_free();
-    while (ggml_backend_reg_count() > 0) {
+    for (size_t i = 0; i < ggml_backend_reg_count(); i++) {
-        auto * reg = ggml_backend_reg_get(0);
+        ggml_backend_unload(ggml_backend_reg_get(i));
-        ggml_backend_unload(reg);
     }
 }