From 1548f78c733bc159de6ca07d705f55c6a65085a5 Mon Sep 17 00:00:00 2001 From: Jesse Gross Date: Thu, 10 Sep 2026 11:49:42 -0700 Subject: [PATCH] mlxrunner: bound MLX loads by system free memory while other models are loaded On Apple silicon the scheduler's free-memory figure for the GPU is the Metal working set minus what Ollama's own runners report. It does not see memory held by other applications, so a second MLX model can pass the fit check on a machine that is already short of memory, and the load pushes the system into swap and compression. While other models are loaded, the MLX fit check now also bounds the available memory by the system's free memory on shared-memory GPUs, the same rule llama-server loads already apply. A miss evicts an idle model and retries instead of starting the load. First loads are unchanged: with nothing else loaded, the model loads against the working-set figure alone, as both engines do today. The check also does not cover memory that grows after load, such as KV caches and prefix-cache snapshots. --- x/mlxrunner/client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x/mlxrunner/client.go b/x/mlxrunner/client.go index 7f242fbd4..728ba654d 100644 --- a/x/mlxrunner/client.go +++ b/x/mlxrunner/client.go @@ -301,11 +301,14 @@ func (c *Client) HasExited() bool { } // Load checks whether the model fits in GPU memory and starts the subprocess. -func (c *Client) Load(ctx context.Context, _ ml.SystemInfo, gpus []ml.DeviceInfo, requireFull bool) ([]ml.DeviceID, error) { +func (c *Client) Load(ctx context.Context, systemInfo ml.SystemInfo, gpus []ml.DeviceInfo, requireFull bool) ([]ml.DeviceID, error) { if len(gpus) > 0 { modelSize := c.memory.Load() // We currently only use the first GPU with MLX available := gpus[0].FreeMemory + if requireFull && gpus[0].Integrated && systemInfo.FreeMemory > 0 && systemInfo.FreeMemory < available { + available = systemInfo.FreeMemory + } overhead := gpus[0].MinimumMemory() + envconfig.GpuOverhead() if available > overhead { available -= overhead