diff --git a/docs/development.md b/docs/development.md index ef28cee88..dec7b5557 100644 --- a/docs/development.md +++ b/docs/development.md @@ -51,10 +51,10 @@ cmake -B build . -DOLLAMA_LLAMA_BACKENDS=cuda_v13 -DCMAKE_CUDA_ARCHITECTURES=nat cmake -B build . -DOLLAMA_LLAMA_BACKENDS=rocm_v7_2 -DCMAKE_HIP_ARCHITECTURES=gfx1100 ``` -You can tune GGML build options by setting `GGML_*` values during configure. For example, to build CUDA v12 for Pascal without flash attention kernels: +You can tune GGML build options by setting `GGML_*` values during configure. For example, to disable CUDA flash attention kernels for local debugging: ```shell -cmake -B build . -DOLLAMA_LLAMA_BACKENDS=cuda_v12 -DCMAKE_CUDA_ARCHITECTURES=61 -DGGML_CUDA_FA=OFF +cmake -B build . -DOLLAMA_LLAMA_BACKENDS=cuda_v12 -DGGML_CUDA_FA=OFF ``` ## macOS (Apple Silicon) diff --git a/docs/faq.mdx b/docs/faq.mdx index 47f06529d..7cedacece 100644 --- a/docs/faq.mdx +++ b/docs/faq.mdx @@ -343,7 +343,7 @@ When loading a new model, Ollama evaluates the required VRAM for the model again ## How can I enable Flash Attention? -Flash Attention is a feature of most modern models that can significantly reduce memory usage as the context size grows. To enable Flash Attention, set the `OLLAMA_FLASH_ATTENTION` environment variable to `1` when starting the Ollama server. +Flash Attention is a feature of most modern models that can significantly reduce memory usage as the context size grows. Ollama uses Flash Attention automatically when the selected backend and devices support it. To force Flash Attention on, set `OLLAMA_FLASH_ATTENTION=1` when starting the Ollama server. To disable it, set `OLLAMA_FLASH_ATTENTION=0`. ## How can I set the quantization type for the K/V cache? diff --git a/llama/server/CMakePresets.json b/llama/server/CMakePresets.json index ddabece27..9ac2f6aa3 100644 --- a/llama/server/CMakePresets.json +++ b/llama/server/CMakePresets.json @@ -68,7 +68,7 @@ "inherits": ["llama_cuda_v12_base"], "binaryDir": "${sourceDir}/../../build/llama-server-cuda_v12", "cacheVariables": { - "CMAKE_CUDA_ARCHITECTURES": "50-virtual;52-virtual;60-virtual;61-virtual;70;75;80;86;89;90;90a;120" + "CMAKE_CUDA_ARCHITECTURES": "50-virtual;52-virtual;60;61;70;75;80;86;89;90;90a;120" } }, { diff --git a/llm/llama_server_test.go b/llm/llama_server_test.go index 9e52c07cb..70e66302b 100644 --- a/llm/llama_server_test.go +++ b/llm/llama_server_test.go @@ -1955,7 +1955,7 @@ func TestAppendFlashAttentionArgs(t *testing.T) { supportedGPU := []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, DriverMajor: 13, ComputeMajor: 8, ComputeMinor: 9}} oldGPU := []ml.DeviceInfo{ {DeviceID: ml.DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 8, ComputeMinor: 9}, - {DeviceID: ml.DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 6, ComputeMinor: 2}, + {DeviceID: ml.DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 5, ComputeMinor: 0}, } tests := []struct { diff --git a/ml/device.go b/ml/device.go index 5dee4fe69..fbeb07139 100644 --- a/ml/device.go +++ b/ml/device.go @@ -585,7 +585,7 @@ func FlashAttentionSupported(l []DeviceInfo) bool { func cudaFlashAttentionSupported(gpu DeviceInfo) bool { if gpu.Library != "CUDA" || - gpu.ComputeMajor < 7 || + gpu.ComputeMajor < 6 || (gpu.ComputeMajor == 7 && gpu.ComputeMinor == 2) { return false } diff --git a/ml/device_test.go b/ml/device_test.go index 1bb525733..474974f46 100644 --- a/ml/device_test.go +++ b/ml/device_test.go @@ -186,8 +186,19 @@ func TestFlashAttentionSupported(t *testing.T) { gpus: []DeviceInfo{{DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 5, ComputeMinor: 0}}, }, { - name: "cuda compute 6.2 unsupported", + name: "cuda compute 6.0 supported", + gpus: []DeviceInfo{{DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 6, ComputeMinor: 0}}, + want: true, + }, + { + name: "cuda compute 6.1 supported", + gpus: []DeviceInfo{{DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 6, ComputeMinor: 1}}, + want: true, + }, + { + name: "cuda compute 6.2 supported", gpus: []DeviceInfo{{DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 6, ComputeMinor: 2}}, + want: true, }, { name: "cuda compute 7.2 unsupported", @@ -216,7 +227,7 @@ func TestFlashAttentionSupported(t *testing.T) { name: "mixed cuda unsupported", gpus: []DeviceInfo{ {DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 8, ComputeMinor: 9}, - {DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 6, ComputeMinor: 2}, + {DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 5, ComputeMinor: 0}, }, }, {