diff --git a/llm/llama_server.go b/llm/llama_server.go index 540aa5644..e529cfc13 100644 --- a/llm/llama_server.go +++ b/llm/llama_server.go @@ -390,6 +390,15 @@ func startLlamaServer(launch llamaServerLaunchConfig, out io.Writer) (cmd *exec. params = append(params, "--no-mmap") } + // Direct I/O skips the page cache on load for integrated CUDA/ROCm GPUs, which + // share system memory with the CPU and would otherwise double-buffer weights. + for _, g := range launch.gpus { + if runtime.GOOS == "linux" && g.Integrated && (strings.EqualFold(g.Library, "CUDA") || strings.EqualFold(g.Library, "ROCm")) { + params = append(params, "--direct-io") + break + } + } + // KV cache type if launch.kvCacheType != "" { params = append(params, "--cache-type-k", launch.kvCacheType, "--cache-type-v", launch.kvCacheType)