upstream-archive/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2026-07-23 09:10:53 -05:00
Code Issues Packages Projects Releases Wiki Activity

llama: enable dio on linux CUDA/ROCm iGPUs (#17286)

Browse Source 
Avoid double memory consumption by enabling direct IO for iGPUs
This commit is contained in:
Daniel Hiltgen
2026-07-21 10:53:08 -07:00
committed by GitHub
parent e2c2edcc27
commit 72116bafb3
1 changed files with 9 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
llm/llama_server.go
View File

@@ -390,6 +390,15 @@ func startLlamaServer(launch llamaServerLaunchConfig, out io.Writer) (cmd *exec.
params = append(params, "--no-mmap") params = append(params, "--no-mmap")
} }
// Direct I/O skips the page cache on load for integrated CUDA/ROCm GPUs, which
// share system memory with the CPU and would otherwise double-buffer weights.
for _, g := range launch.gpus {
if runtime.GOOS == "linux" && g.Integrated && (strings.EqualFold(g.Library, "CUDA") || strings.EqualFold(g.Library, "ROCm")) {
params = append(params, "--direct-io")
break
}
}
// KV cache type // KV cache type
if launch.kvCacheType != "" { if launch.kvCacheType != "" {
params = append(params, "--cache-type-k", launch.kvCacheType, "--cache-type-v", launch.kvCacheType) params = append(params, "--cache-type-k", launch.kvCacheType, "--cache-type-v", launch.kvCacheType)