mirror of
https://github.com/leejet/stable-diffusion.cpp.git
synced 2026-09-25 07:27:53 -05:00
feat: enable single-GPU auto-fit with tiered parameter placement (#1942)
This commit is contained in:
leejet
+322
-302
@@ -2,364 +2,384 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <fstream>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#elif defined(__APPLE__)
|
||||
#include <mach/mach.h>
|
||||
#endif
|
||||
|
||||
#include "core/ggml_extend_backend.h"
|
||||
#include "core/util.h"
|
||||
#include "ggml-backend.h"
|
||||
|
||||
namespace sd::backend_fit {
|
||||
namespace {
|
||||
|
||||
constexpr int64_t MiB = 1024ll * 1024;
|
||||
static constexpr int64_t MiB = 1024ll * 1024;
|
||||
|
||||
enum class ComponentKind {
|
||||
DIT = 0,
|
||||
VAE = 1,
|
||||
CONDITIONER = 2,
|
||||
};
|
||||
enum class ComponentKind {
|
||||
DIT,
|
||||
CONDITIONER,
|
||||
VAE,
|
||||
};
|
||||
|
||||
struct Component {
|
||||
struct Component {
|
||||
ComponentKind kind;
|
||||
const char* name;
|
||||
int64_t params_bytes = 0;
|
||||
int64_t reserve_bytes = 0;
|
||||
int64_t staging_bytes = 0;
|
||||
};
|
||||
|
||||
struct Device {
|
||||
std::string name;
|
||||
std::string description;
|
||||
int64_t free_bytes = 0;
|
||||
int64_t budget_bytes = 0;
|
||||
};
|
||||
|
||||
enum class ParamsLocation {
|
||||
MAIN_GPU,
|
||||
CPU,
|
||||
OTHER_GPU,
|
||||
DISK,
|
||||
};
|
||||
|
||||
struct Decision {
|
||||
ParamsLocation params_location = ParamsLocation::DISK;
|
||||
size_t params_device = SIZE_MAX;
|
||||
};
|
||||
|
||||
struct Plan {
|
||||
bool valid = false;
|
||||
size_t main_device = SIZE_MAX;
|
||||
std::vector<Decision> decisions;
|
||||
};
|
||||
|
||||
static bool classify_tensor(const std::string& name, ComponentKind& out) {
|
||||
auto contains = [&](const char* s) { return name.find(s) != std::string::npos; };
|
||||
|
||||
if (contains("model.diffusion_model.") || contains("unet.")) {
|
||||
out = ComponentKind::DIT;
|
||||
return true;
|
||||
}
|
||||
if (contains("first_stage_model.") ||
|
||||
name.rfind("vae.", 0) == 0 ||
|
||||
name.rfind("tae.", 0) == 0) {
|
||||
out = ComponentKind::VAE;
|
||||
return true;
|
||||
}
|
||||
if (contains("text_encoders") ||
|
||||
contains("cond_stage_model") ||
|
||||
contains("te.text_model.") ||
|
||||
contains("conditioner") ||
|
||||
name.rfind("text_encoder.", 0) == 0 ||
|
||||
name.rfind("text_embedding_projection.", 0) == 0 ||
|
||||
contains(".aggregate_embed.")) {
|
||||
out = ComponentKind::CONDITIONER;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static std::vector<Component> estimate_components(ModelLoader& loader, ggml_type override_wtype) {
|
||||
int64_t bytes[3] = {0, 0, 0};
|
||||
int64_t largest_tensor[3] = {0, 0, 0};
|
||||
for (const auto& [name, stored_tensor] : loader.get_tensor_storage_map()) {
|
||||
TensorStorage ts = stored_tensor;
|
||||
ComponentKind kind;
|
||||
const char* name;
|
||||
int64_t params_bytes = 0;
|
||||
int64_t reserve_bytes = 0;
|
||||
bool splittable = false;
|
||||
};
|
||||
|
||||
struct Device {
|
||||
ggml_backend_dev_t dev = nullptr;
|
||||
std::string name;
|
||||
std::string description;
|
||||
int64_t free_bytes = 0;
|
||||
int64_t total_bytes = 0;
|
||||
int64_t budget_bytes = 0;
|
||||
};
|
||||
|
||||
struct Decision {
|
||||
ComponentKind kind;
|
||||
bool on_cpu = false;
|
||||
std::vector<size_t> device_idxs;
|
||||
};
|
||||
|
||||
struct Plan {
|
||||
bool valid = false;
|
||||
bool time_share = false;
|
||||
std::vector<Decision> decisions;
|
||||
};
|
||||
|
||||
bool classify_tensor(const std::string& name, ComponentKind& out) {
|
||||
auto contains = [&](const char* s) { return name.find(s) != std::string::npos; };
|
||||
|
||||
if (contains("model.diffusion_model.") || contains("unet.")) {
|
||||
out = ComponentKind::DIT;
|
||||
return true;
|
||||
if (is_unused_tensor(ts.name) || !classify_tensor(ts.name, kind)) {
|
||||
continue;
|
||||
}
|
||||
if (contains("first_stage_model.") ||
|
||||
name.rfind("vae.", 0) == 0 ||
|
||||
name.rfind("tae.", 0) == 0) {
|
||||
out = ComponentKind::VAE;
|
||||
return true;
|
||||
if (ts.expected_type != GGML_TYPE_COUNT) {
|
||||
ts.type = ts.expected_type;
|
||||
} else if (override_wtype != GGML_TYPE_COUNT && loader.tensor_should_be_converted(ts, override_wtype)) {
|
||||
ts.type = override_wtype;
|
||||
}
|
||||
if (contains("text_encoders") ||
|
||||
contains("cond_stage_model") ||
|
||||
contains("te.text_model.") ||
|
||||
contains("conditioner") ||
|
||||
name.rfind("text_encoder.", 0) == 0 ||
|
||||
name.rfind("text_embedding_projection.", 0) == 0 ||
|
||||
contains(".aggregate_embed.")) {
|
||||
out = ComponentKind::CONDITIONER;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
const int64_t tensor_bytes = (int64_t)ts.nbytes() + 64;
|
||||
bytes[int(kind)] += tensor_bytes;
|
||||
largest_tensor[int(kind)] = std::max(largest_tensor[int(kind)], tensor_bytes);
|
||||
}
|
||||
|
||||
std::vector<Component> estimate_components(ModelLoader& loader, ggml_type override_wtype) {
|
||||
const auto& storage = loader.get_tensor_storage_map();
|
||||
return {
|
||||
{ComponentKind::DIT, "DiT", bytes[int(ComponentKind::DIT)], 2048 * MiB, largest_tensor[int(ComponentKind::DIT)]},
|
||||
{ComponentKind::CONDITIONER, "Conditioner", bytes[int(ComponentKind::CONDITIONER)], 2048 * MiB, largest_tensor[int(ComponentKind::CONDITIONER)]},
|
||||
{ComponentKind::VAE, "VAE", bytes[int(ComponentKind::VAE)], 1024 * MiB, largest_tensor[int(ComponentKind::VAE)]},
|
||||
};
|
||||
}
|
||||
|
||||
int64_t bytes[3] = {0, 0, 0};
|
||||
for (const auto& [name, ts_const] : storage) {
|
||||
TensorStorage ts = ts_const;
|
||||
if (is_unused_tensor(ts.name)) {
|
||||
continue;
|
||||
}
|
||||
ComponentKind kind;
|
||||
if (!classify_tensor(ts.name, kind)) {
|
||||
continue;
|
||||
}
|
||||
if (override_wtype != GGML_TYPE_COUNT &&
|
||||
loader.tensor_should_be_converted(ts, override_wtype)) {
|
||||
ts.type = override_wtype;
|
||||
} else if (ts.expected_type != GGML_TYPE_COUNT && ts.expected_type != ts.type) {
|
||||
ts.type = ts.expected_type;
|
||||
}
|
||||
bytes[int(kind)] += (int64_t)ts.nbytes() + 64;
|
||||
static std::string budget_key(std::string name) {
|
||||
std::transform(name.begin(), name.end(), name.begin(), [](unsigned char c) { return (char)std::tolower(c); });
|
||||
return name;
|
||||
}
|
||||
|
||||
static std::vector<Device> enumerate_gpu_devices(const sd::ggml_graph_cut::MaxVramAssignment& budgets) {
|
||||
std::vector<Device> out;
|
||||
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
|
||||
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
|
||||
if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) {
|
||||
continue;
|
||||
}
|
||||
Device device;
|
||||
device.name = ggml_backend_dev_name(dev);
|
||||
device.description = ggml_backend_dev_description(dev);
|
||||
size_t free_bytes = 0, total_bytes = 0;
|
||||
ggml_backend_dev_memory(dev, &free_bytes, &total_bytes);
|
||||
device.free_bytes = (int64_t)free_bytes;
|
||||
|
||||
std::vector<Component> out;
|
||||
out.push_back({ComponentKind::DIT, "DiT", bytes[int(ComponentKind::DIT)], 2048 * MiB, true});
|
||||
out.push_back({ComponentKind::VAE, "VAE", bytes[int(ComponentKind::VAE)], 1024 * MiB, false});
|
||||
out.push_back({ComponentKind::CONDITIONER, "Conditioner", bytes[int(ComponentKind::CONDITIONER)], 2048 * MiB, true});
|
||||
return out;
|
||||
float gib = budgets.default_gib;
|
||||
auto it = budgets.backend_gib.find(budget_key(device.name));
|
||||
if (it != budgets.backend_gib.end()) {
|
||||
gib = it->second;
|
||||
}
|
||||
if (gib > 0.f) {
|
||||
device.budget_bytes = (int64_t)std::min(gib * 1024.0 * MiB, (double)device.free_bytes);
|
||||
} else if (gib < 0.f) {
|
||||
device.budget_bytes = (int64_t)std::max<double>(device.free_bytes + gib * 1024.0 * MiB, 0);
|
||||
} else {
|
||||
device.budget_bytes = std::max<int64_t>(device.free_bytes - 512 * MiB, 0);
|
||||
}
|
||||
out.push_back(std::move(device));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<Device> enumerate_gpu_devices(const sd::ggml_graph_cut::MaxVramAssignment& budgets) {
|
||||
std::vector<Device> out;
|
||||
for (size_t i = 0; i < ggml_backend_dev_count(); i++) {
|
||||
ggml_backend_dev_t dev = ggml_backend_dev_get(i);
|
||||
if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) {
|
||||
continue;
|
||||
}
|
||||
Device d;
|
||||
d.dev = dev;
|
||||
d.name = ggml_backend_dev_name(dev);
|
||||
d.description = ggml_backend_dev_description(dev);
|
||||
size_t free_bytes = 0, total_bytes = 0;
|
||||
ggml_backend_dev_memory(dev, &free_bytes, &total_bytes);
|
||||
d.free_bytes = (int64_t)free_bytes;
|
||||
d.total_bytes = (int64_t)total_bytes;
|
||||
|
||||
std::string budget_key = d.name;
|
||||
std::transform(budget_key.begin(), budget_key.end(), budget_key.begin(),
|
||||
[](unsigned char c) { return (char)std::tolower(c); });
|
||||
float gib = budgets.default_gib;
|
||||
auto it = budgets.backend_gib.find(budget_key);
|
||||
if (it != budgets.backend_gib.end()) {
|
||||
gib = it->second;
|
||||
}
|
||||
if (gib > 0.f) {
|
||||
d.budget_bytes = std::min<int64_t>((int64_t)(gib * 1024.0 * 1024.0 * 1024.0), d.free_bytes);
|
||||
} else if (gib < 0.f) {
|
||||
d.budget_bytes = d.free_bytes + (int64_t)(gib * 1024.0 * 1024.0 * 1024.0);
|
||||
} else {
|
||||
d.budget_bytes = d.free_bytes - 512 * MiB;
|
||||
}
|
||||
d.budget_bytes = std::max<int64_t>(d.budget_bytes, 0);
|
||||
out.push_back(d);
|
||||
}
|
||||
return out;
|
||||
static int64_t available_ram_bytes() {
|
||||
#if defined(_WIN32)
|
||||
MEMORYSTATUSEX status{};
|
||||
status.dwLength = sizeof(status);
|
||||
if (GlobalMemoryStatusEx(&status)) {
|
||||
return (int64_t)status.ullAvailPhys;
|
||||
}
|
||||
|
||||
Plan compute_plan(const std::vector<Component>& components, const std::vector<Device>& devices) {
|
||||
Plan plan;
|
||||
if (devices.empty()) {
|
||||
return plan;
|
||||
#elif defined(__linux__)
|
||||
std::ifstream meminfo("/proc/meminfo");
|
||||
std::string key, unit;
|
||||
int64_t kib = 0;
|
||||
while (meminfo >> key >> kib >> unit) {
|
||||
if (key == "MemAvailable:" && unit == "kB" && kib >= 0) {
|
||||
return kib * 1024;
|
||||
}
|
||||
}
|
||||
#elif defined(__APPLE__)
|
||||
const mach_port_t host = mach_host_self();
|
||||
vm_size_t page_size = 0;
|
||||
vm_statistics64_data_t stats{};
|
||||
mach_msg_type_number_t count = HOST_VM_INFO64_COUNT;
|
||||
const bool ok = host_page_size(host, &page_size) == KERN_SUCCESS &&
|
||||
host_statistics64(host, HOST_VM_INFO64, (host_info64_t)&stats, &count) == KERN_SUCCESS;
|
||||
mach_port_deallocate(mach_task_self(), host);
|
||||
if (ok) {
|
||||
return ((int64_t)stats.free_count + stats.inactive_count) * page_size;
|
||||
}
|
||||
#endif
|
||||
return -1;
|
||||
}
|
||||
|
||||
std::vector<size_t> order(components.size());
|
||||
for (size_t i = 0; i < order.size(); i++) {
|
||||
order[i] = i;
|
||||
static Plan compute_plan(const std::vector<Component>& components,
|
||||
const std::vector<Device>& devices,
|
||||
int64_t ram_budget_bytes) {
|
||||
Plan plan;
|
||||
for (size_t di = 0; di < devices.size(); ++di) {
|
||||
if (devices[di].budget_bytes > 0 &&
|
||||
(plan.main_device == SIZE_MAX || devices[di].budget_bytes > devices[plan.main_device].budget_bytes)) {
|
||||
plan.main_device = di;
|
||||
}
|
||||
std::sort(order.begin(), order.end(), [&](size_t a, size_t b) {
|
||||
return components[a].params_bytes > components[b].params_bytes;
|
||||
});
|
||||
|
||||
{
|
||||
std::vector<int64_t> params_sum(devices.size(), 0);
|
||||
std::vector<int64_t> max_reserve(devices.size(), 0);
|
||||
std::vector<Decision> decisions(components.size());
|
||||
bool ok = true;
|
||||
for (size_t ci : order) {
|
||||
const Component& comp = components[ci];
|
||||
decisions[ci].kind = comp.kind;
|
||||
if (comp.params_bytes == 0) {
|
||||
continue;
|
||||
}
|
||||
int best = -1;
|
||||
for (size_t di = 0; di < devices.size(); di++) {
|
||||
int64_t need = params_sum[di] + comp.params_bytes + std::max(max_reserve[di], comp.reserve_bytes);
|
||||
if (need <= devices[di].budget_bytes &&
|
||||
(best < 0 || devices[di].budget_bytes - params_sum[di] > devices[best].budget_bytes - params_sum[best])) {
|
||||
best = (int)di;
|
||||
}
|
||||
}
|
||||
if (best < 0) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
params_sum[best] += comp.params_bytes;
|
||||
max_reserve[best] = std::max(max_reserve[best], comp.reserve_bytes);
|
||||
decisions[ci].device_idxs.push_back((size_t)best);
|
||||
}
|
||||
if (ok) {
|
||||
plan.valid = true;
|
||||
plan.time_share = false;
|
||||
plan.decisions = std::move(decisions);
|
||||
return plan;
|
||||
}
|
||||
}
|
||||
|
||||
plan.decisions.assign(components.size(), {});
|
||||
for (size_t ci : order) {
|
||||
const Component& comp = components[ci];
|
||||
Decision& decision = plan.decisions[ci];
|
||||
decision.kind = comp.kind;
|
||||
if (comp.params_bytes == 0) {
|
||||
continue;
|
||||
}
|
||||
int best = -1;
|
||||
for (size_t di = 0; di < devices.size(); di++) {
|
||||
if (comp.params_bytes + comp.reserve_bytes <= devices[di].budget_bytes &&
|
||||
(best < 0 || devices[di].budget_bytes > devices[best].budget_bytes)) {
|
||||
best = (int)di;
|
||||
}
|
||||
}
|
||||
if (best >= 0) {
|
||||
decision.device_idxs.push_back((size_t)best);
|
||||
continue;
|
||||
}
|
||||
if (comp.splittable && devices.size() > 1) {
|
||||
int64_t capacity = 0;
|
||||
for (const Device& d : devices) {
|
||||
capacity += std::max<int64_t>(d.budget_bytes - comp.reserve_bytes, 0);
|
||||
}
|
||||
if (comp.params_bytes <= capacity) {
|
||||
std::vector<size_t> idxs(devices.size());
|
||||
for (size_t i = 0; i < idxs.size(); i++) {
|
||||
idxs[i] = i;
|
||||
}
|
||||
std::sort(idxs.begin(), idxs.end(), [&](size_t a, size_t b) {
|
||||
return devices[a].budget_bytes > devices[b].budget_bytes;
|
||||
});
|
||||
decision.device_idxs = std::move(idxs);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
decision.on_cpu = true;
|
||||
}
|
||||
plan.valid = true;
|
||||
plan.time_share = true;
|
||||
}
|
||||
if (plan.main_device == SIZE_MAX) {
|
||||
return plan;
|
||||
}
|
||||
|
||||
void print_plan(const Plan& plan,
|
||||
const std::vector<Component>& components,
|
||||
const std::vector<Device>& devices) {
|
||||
LOG_INFO("auto-fit plan%s:", plan.time_share ? " (time-share: params load per phase and free after)" : "");
|
||||
LOG_INFO(" devices:");
|
||||
for (const Device& d : devices) {
|
||||
LOG_INFO(" %-12s %-32s free %6lld MiB, budget %6lld MiB",
|
||||
d.name.c_str(), d.description.c_str(),
|
||||
(long long)(d.free_bytes / MiB), (long long)(d.budget_bytes / MiB));
|
||||
}
|
||||
LOG_INFO(" components:");
|
||||
for (size_t ci = 0; ci < components.size(); ci++) {
|
||||
const Component& comp = components[ci];
|
||||
const Decision& decision = plan.decisions[ci];
|
||||
std::string target;
|
||||
if (comp.params_bytes == 0) {
|
||||
target = "(not present)";
|
||||
} else if (decision.on_cpu) {
|
||||
target = "CPU";
|
||||
} else {
|
||||
for (size_t k = 0; k < decision.device_idxs.size(); k++) {
|
||||
if (k > 0) {
|
||||
target += " & ";
|
||||
}
|
||||
target += devices[decision.device_idxs[k]].name;
|
||||
}
|
||||
if (decision.device_idxs.size() > 1) {
|
||||
target += " (split)";
|
||||
}
|
||||
}
|
||||
LOG_INFO(" %-12s params %6lld MiB, compute reserve %5lld MiB -> %s",
|
||||
comp.name,
|
||||
(long long)(comp.params_bytes / MiB),
|
||||
(long long)(comp.reserve_bytes / MiB),
|
||||
target.c_str());
|
||||
}
|
||||
std::vector<size_t> order(components.size());
|
||||
for (size_t ci = 0; ci < components.size(); ++ci) {
|
||||
order[ci] = ci;
|
||||
}
|
||||
std::stable_sort(order.begin(), order.end(), [&](size_t a, size_t b) {
|
||||
return components[a].kind < components[b].kind;
|
||||
});
|
||||
|
||||
void append_assignment(std::string& spec, const char* key, const std::string& value) {
|
||||
if (!spec.empty()) {
|
||||
spec += ",";
|
||||
}
|
||||
spec += key;
|
||||
spec += "=";
|
||||
spec += value;
|
||||
std::vector<int64_t> remaining;
|
||||
for (const Device& device : devices) {
|
||||
remaining.push_back(std::max<int64_t>(device.budget_bytes, 0));
|
||||
}
|
||||
ram_budget_bytes = std::max<int64_t>(ram_budget_bytes, 0);
|
||||
plan.decisions.resize(components.size());
|
||||
|
||||
void append_component_decision(const std::vector<Component>& components,
|
||||
const std::vector<Device>& devices,
|
||||
const Plan& plan,
|
||||
ComponentKind kind,
|
||||
const char* module_key,
|
||||
std::string& runtime_spec,
|
||||
std::string& params_spec) {
|
||||
for (size_t ci = 0; ci < components.size(); ci++) {
|
||||
if (components[ci].kind != kind || components[ci].params_bytes == 0) {
|
||||
for (size_t ci : order) {
|
||||
const Component& comp = components[ci];
|
||||
Decision& decision = plan.decisions[ci];
|
||||
if (comp.params_bytes == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Higher-priority offloaded weights need GPU cache space across graph runs.
|
||||
int64_t headroom = 0;
|
||||
for (size_t other = 0; other < components.size(); ++other) {
|
||||
if (components[other].params_bytes == 0) {
|
||||
continue;
|
||||
}
|
||||
const Decision& decision = plan.decisions[ci];
|
||||
if (decision.on_cpu) {
|
||||
append_assignment(runtime_spec, module_key, "cpu");
|
||||
return;
|
||||
const bool resident = other == ci || plan.decisions[other].params_location == ParamsLocation::MAIN_GPU;
|
||||
const int64_t cached_weights = components[other].kind < comp.kind
|
||||
? components[other].params_bytes
|
||||
: components[other].staging_bytes;
|
||||
headroom = std::max(headroom, components[other].reserve_bytes +
|
||||
(resident ? 0 : cached_weights));
|
||||
}
|
||||
int64_t& main_remaining = remaining[plan.main_device];
|
||||
if (headroom <= main_remaining && comp.params_bytes <= main_remaining - headroom) {
|
||||
decision.params_location = ParamsLocation::MAIN_GPU;
|
||||
decision.params_device = plan.main_device;
|
||||
main_remaining -= comp.params_bytes;
|
||||
continue;
|
||||
}
|
||||
if (comp.params_bytes <= ram_budget_bytes) {
|
||||
decision.params_location = ParamsLocation::CPU;
|
||||
ram_budget_bytes -= comp.params_bytes;
|
||||
continue;
|
||||
}
|
||||
|
||||
size_t best = SIZE_MAX;
|
||||
for (size_t di = 0; di < devices.size(); ++di) {
|
||||
if (di != plan.main_device && comp.params_bytes <= remaining[di] &&
|
||||
(best == SIZE_MAX || remaining[di] > remaining[best])) {
|
||||
best = di;
|
||||
}
|
||||
if (decision.device_idxs.empty()) {
|
||||
return;
|
||||
}
|
||||
std::string device_list;
|
||||
for (size_t k = 0; k < decision.device_idxs.size(); k++) {
|
||||
if (k > 0) {
|
||||
device_list += "&";
|
||||
}
|
||||
device_list += devices[decision.device_idxs[k]].name;
|
||||
}
|
||||
append_assignment(runtime_spec, module_key, device_list);
|
||||
if (plan.time_share) {
|
||||
append_assignment(params_spec, module_key, "disk");
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (best != SIZE_MAX) {
|
||||
decision.params_location = ParamsLocation::OTHER_GPU;
|
||||
decision.params_device = best;
|
||||
remaining[best] -= comp.params_bytes;
|
||||
}
|
||||
}
|
||||
plan.valid = true;
|
||||
return plan;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
static std::string params_backend_name(const Decision& decision, const std::vector<Device>& devices) {
|
||||
switch (decision.params_location) {
|
||||
case ParamsLocation::MAIN_GPU:
|
||||
case ParamsLocation::OTHER_GPU:
|
||||
return devices[decision.params_device].name;
|
||||
case ParamsLocation::CPU:
|
||||
return "cpu";
|
||||
case ParamsLocation::DISK:
|
||||
return "disk";
|
||||
}
|
||||
return "disk";
|
||||
}
|
||||
|
||||
static void print_plan(const Plan& plan,
|
||||
const std::vector<Component>& components,
|
||||
const std::vector<Device>& devices,
|
||||
int64_t free_ram,
|
||||
int64_t ram_budget) {
|
||||
LOG_INFO("auto-fit plan (single-GPU compute on %s):", devices[plan.main_device].name.c_str());
|
||||
LOG_INFO(" devices:");
|
||||
for (const Device& device : devices) {
|
||||
LOG_INFO(" %-12s %-32s free %6lld MiB, budget %6lld MiB",
|
||||
device.name.c_str(), device.description.c_str(),
|
||||
(long long)(device.free_bytes / MiB), (long long)(device.budget_bytes / MiB));
|
||||
}
|
||||
if (free_ram < 0) {
|
||||
LOG_WARN("auto-fit: available RAM is unknown; skipping CPU parameter residency");
|
||||
} else {
|
||||
LOG_INFO(" RAM free %6lld MiB, params budget %6lld MiB",
|
||||
(long long)(free_ram / MiB), (long long)(ram_budget / MiB));
|
||||
}
|
||||
LOG_INFO(" main-GPU weight cache priority: diffusion > te > vae");
|
||||
LOG_INFO(" components (params: main GPU -> RAM -> other GPU -> disk):");
|
||||
for (size_t ci = 0; ci < components.size(); ++ci) {
|
||||
const Component& comp = components[ci];
|
||||
if (comp.params_bytes == 0) {
|
||||
continue;
|
||||
}
|
||||
const std::string params = params_backend_name(plan.decisions[ci], devices);
|
||||
LOG_INFO(" %-12s params %6lld MiB, compute reserve %5lld MiB -> compute %s, params %s",
|
||||
comp.name, (long long)(comp.params_bytes / MiB), (long long)(comp.reserve_bytes / MiB),
|
||||
devices[plan.main_device].name.c_str(), params.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
static void append_assignment(std::string& spec, const char* key, const std::string& value) {
|
||||
if (!spec.empty()) {
|
||||
spec += ",";
|
||||
}
|
||||
spec += key;
|
||||
spec += "=";
|
||||
spec += value;
|
||||
}
|
||||
|
||||
static const char* module_key(ComponentKind kind) {
|
||||
switch (kind) {
|
||||
case ComponentKind::DIT:
|
||||
return "diffusion";
|
||||
case ComponentKind::CONDITIONER:
|
||||
return "te";
|
||||
case ComponentKind::VAE:
|
||||
return "vae";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
bool derive_backend_specs(ModelLoader& loader,
|
||||
ggml_type override_wtype,
|
||||
sd::ggml_graph_cut::MaxVramAssignment& budgets,
|
||||
std::string& runtime_spec,
|
||||
std::string& params_spec) {
|
||||
if (!runtime_spec.empty() || !params_spec.empty()) {
|
||||
LOG_WARN("--auto-fit is enabled; ignoring --backend / --params-backend");
|
||||
std::string error;
|
||||
if (!budgets.canonicalize_backend_keys(&error)) {
|
||||
LOG_ERROR("%s", error.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
{
|
||||
std::string error;
|
||||
if (!budgets.canonicalize_backend_keys(&error)) {
|
||||
LOG_ERROR("%s", error.c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
auto components = estimate_components(loader, override_wtype);
|
||||
auto devices = enumerate_gpu_devices(budgets);
|
||||
auto plan = compute_plan(components, devices);
|
||||
const auto components = estimate_components(loader, override_wtype);
|
||||
const auto devices = enumerate_gpu_devices(budgets);
|
||||
const int64_t free_ram = available_ram_bytes();
|
||||
const int64_t ram_budget = std::max<int64_t>(free_ram - std::max<int64_t>(2048 * MiB, free_ram / 10), 0);
|
||||
const auto plan = compute_plan(components, devices, ram_budget);
|
||||
runtime_spec.clear();
|
||||
params_spec.clear();
|
||||
if (!plan.valid) {
|
||||
LOG_WARN("auto-fit: no usable GPU devices; using the default backend");
|
||||
runtime_spec.clear();
|
||||
params_spec.clear();
|
||||
if (devices.empty()) {
|
||||
LOG_WARN("auto-fit: no GPU devices; using the default backend");
|
||||
} else {
|
||||
LOG_WARN("auto-fit: no GPU memory budget available; using CPU");
|
||||
runtime_spec = "cpu";
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
print_plan(plan, components, devices);
|
||||
print_plan(plan, components, devices, free_ram, ram_budget);
|
||||
for (size_t ci = 0; ci < components.size(); ++ci) {
|
||||
if (components[ci].params_bytes == 0) {
|
||||
continue;
|
||||
}
|
||||
const char* key = module_key(components[ci].kind);
|
||||
append_assignment(runtime_spec, key, devices[plan.main_device].name);
|
||||
if (plan.decisions[ci].params_location != ParamsLocation::MAIN_GPU) {
|
||||
append_assignment(params_spec, key, params_backend_name(plan.decisions[ci], devices));
|
||||
}
|
||||
}
|
||||
|
||||
std::string derived_runtime_spec;
|
||||
std::string derived_params_spec;
|
||||
append_component_decision(components, devices, plan, ComponentKind::DIT, "diffusion", derived_runtime_spec, derived_params_spec);
|
||||
append_component_decision(components, devices, plan, ComponentKind::CONDITIONER, "te", derived_runtime_spec, derived_params_spec);
|
||||
append_component_decision(components, devices, plan, ComponentKind::VAE, "vae", derived_runtime_spec, derived_params_spec);
|
||||
|
||||
runtime_spec = std::move(derived_runtime_spec);
|
||||
params_spec = std::move(derived_params_spec);
|
||||
// Keep the planner's safety margin when the runner resolves its device limits.
|
||||
for (const Device& device : devices) {
|
||||
if (device.budget_bytes > 0) {
|
||||
budgets.backend_gib[budget_key(device.name)] = (float)(device.budget_bytes / (1024.0 * MiB));
|
||||
}
|
||||
}
|
||||
budgets.resolved_backend_bytes.clear();
|
||||
|
||||
LOG_INFO("auto-fit: --backend \"%s\"%s%s%s",
|
||||
runtime_spec.empty() ? "(default)" : runtime_spec.c_str(),
|
||||
params_spec.empty() ? "" : " --params-backend \"",
|
||||
params_spec.c_str(),
|
||||
params_spec.empty() ? "" : "\"");
|
||||
params_spec.c_str(), params_spec.empty() ? "" : "\"");
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -874,7 +874,7 @@ public:
|
||||
backend_spec = SAFE_STR(sd_ctx_params->backend);
|
||||
params_backend_spec = SAFE_STR(sd_ctx_params->params_backend);
|
||||
split_mode_spec = SAFE_STR(sd_ctx_params->split_mode);
|
||||
auto_fit_enabled = sd_ctx_params->auto_fit;
|
||||
auto_fit_enabled = sd_ctx_params->auto_fit && backend_spec.empty() && params_backend_spec.empty();
|
||||
max_vram_assignment.reset(0.f);
|
||||
{
|
||||
std::string error;
|
||||
@@ -3578,7 +3578,7 @@ void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params) {
|
||||
sd_ctx_params->backend = nullptr;
|
||||
sd_ctx_params->params_backend = nullptr;
|
||||
sd_ctx_params->split_mode = nullptr;
|
||||
sd_ctx_params->auto_fit = false;
|
||||
sd_ctx_params->auto_fit = true;
|
||||
sd_ctx_params->rpc_servers = nullptr;
|
||||
sd_ctx_params->model_args = nullptr;
|
||||
sd_ctx_params->pulid_weights_path = nullptr;
|
||||
|
||||
Reference in New Issue
Block a user