mirror of
https://github.com/ollama/ollama.git
synced 2026-09-21 05:28:00 -05:00
mlxrunner: bound MLX loads by system free memory while other models are loaded
On Apple silicon the scheduler's free-memory figure for the GPU is the Metal working set minus what Ollama's own runners report. It does not see memory held by other applications, so a second MLX model can pass the fit check on a machine that is already short of memory, and the load pushes the system into swap and compression. While other models are loaded, the MLX fit check now also bounds the available memory by the system's free memory on shared-memory GPUs, the same rule llama-server loads already apply. A miss evicts an idle model and retries instead of starting the load. First loads are unchanged: with nothing else loaded, the model loads against the working-set figure alone, as both engines do today. The check also does not cover memory that grows after load, such as KV caches and prefix-cache snapshots.
This commit is contained in:
@@ -301,11 +301,14 @@ func (c *Client) HasExited() bool {
|
||||
}
|
||||
|
||||
// Load checks whether the model fits in GPU memory and starts the subprocess.
|
||||
func (c *Client) Load(ctx context.Context, _ ml.SystemInfo, gpus []ml.DeviceInfo, requireFull bool) ([]ml.DeviceID, error) {
|
||||
func (c *Client) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, requireFull bool) ([]ml.DeviceID, error) {
|
||||
if len(gpus) > 0 {
|
||||
modelSize := c.memory.Load()
|
||||
// We currently only use the first GPU with MLX
|
||||
available := gpus[0].FreeMemory
|
||||
if requireFull && gpus[0].Integrated && systemInfo.FreeMemory > 0 && systemInfo.FreeMemory < available {
|
||||
available = systemInfo.FreeMemory
|
||||
}
|
||||
overhead := gpus[0].MinimumMemory() + envconfig.GpuOverhead()
|
||||
if available > overhead {
|
||||
available -= overhead
|
||||
|
||||
Reference in New Issue
Block a user