upstream-archive/ollama
1
0
Fork 0
mirror of https://github.com/ollama/ollama.git synced 2026-07-23 09:10:53 -05:00
Code Issues Packages Projects Releases Wiki Activity

llama-server: allow GPU offload for projectors (#16473)

Browse Source 
Special case Metal iGPUs to enable GPU offload.
This commit is contained in:
Daniel Hiltgen
2026-06-03 13:58:40 -07:00
committed by GitHub
parent e5a38739b4
commit 01557ff313
2 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
llm/llama_server.go
View File

@@ -631,7 +631,7 @@ func shouldDisableMMProjOffload(opts api.Options, gpus []ml.DeviceInfo, modelLay
}
for _, gpu := range gpus {
if gpu.Integrated {
if gpu.Integrated && gpu.Library != "Metal" {
return true, "shared-memory-gpu"
}
memory := gpu.FreeMemory

10
llm/llama_server_test.go
View File

@@ -1839,13 +1839,21 @@ func TestAppendMMProjArgs(t *testing.T) {
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
},
{
name: "integrated gpu disables projector offload",
name: "integrated rocm gpu disables projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "ROCm"}, Integrated: true, FreeMemory: 32 << 30}},
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf", "--no-mmproj-offload"},
},
{
name: "integrated metal gpu keeps projector offload",
projectors: []string{"model.gguf"},
opts: defaultOpts,
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "Metal"}, Integrated: true, FreeMemory: 32 << 30}},
modelLayers: 81,
want: []string{"base", "--mmproj", "model.gguf"},
},
{
name: "cpu only request disables projector offload",
projectors: []string{"model.gguf"},