diff --git a/api/types_test.go b/api/types_test.go index d7be59dfa..d15726202 100644 --- a/api/types_test.go +++ b/api/types_test.go @@ -4,9 +4,12 @@ import ( "encoding/json" "errors" "math" + "reflect" + "strings" "testing" "time" + "github.com/ollama/ollama/types/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -213,6 +216,22 @@ func TestMainGPUParsingFromJSON(t *testing.T) { } } +func TestGenerationDefaultMappingsAreOptions(t *testing.T) { + jsonOpts := make(map[string]struct{}) + for _, field := range reflect.VisibleFields(reflect.TypeOf(Options{})) { + jsonTag := strings.Split(field.Tag.Get("json"), ",")[0] + if jsonTag != "" { + jsonOpts[jsonTag] = struct{}{} + } + } + + for _, option := range model.GenerationDefaultOptions() { + if _, ok := jsonOpts[option]; !ok { + t.Fatalf("%s should be defined on api.Options", option) + } + } +} + func TestUseMmapFormatParams(t *testing.T) { tr := true fa := false diff --git a/fs/gguf/keyvalue.go b/fs/gguf/keyvalue.go index 5843326c1..474201165 100644 --- a/fs/gguf/keyvalue.go +++ b/fs/gguf/keyvalue.go @@ -26,6 +26,15 @@ func value[T any](v Value, kinds ...reflect.Kind) (t T) { return } +func valueOK[T any](v Value, kinds ...reflect.Kind) (t T, ok bool) { + vv := reflect.ValueOf(v.value) + if !vv.IsValid() || !slices.Contains(kinds, vv.Kind()) { + return t, false + } + + return vv.Convert(reflect.TypeOf(t)).Interface().(T), true +} + func values[T any](v Value, kinds ...reflect.Kind) (ts []T) { switch vv := reflect.ValueOf(v.value); vv.Kind() { case reflect.Slice: @@ -44,6 +53,12 @@ func (v Value) Int() int64 { return value[int64](v, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64) } +// IntOK converts a signed integer value to int64 and reports whether the +// underlying type was signed. +func (v Value) IntOK() (int64, bool) { + return valueOK[int64](v, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64) +} + // Ints returns Value as a signed integer slice. If it is not a signed integer slice, it returns nil. func (v Value) Ints() (i64s []int64) { return values[int64](v, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64) @@ -54,6 +69,12 @@ func (v Value) Uint() uint64 { return value[uint64](v, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64) } +// UintOK converts an unsigned integer value to uint64 and reports whether the +// underlying type was unsigned. +func (v Value) UintOK() (uint64, bool) { + return valueOK[uint64](v, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64) +} + // Uints returns Value as a unsigned integer slice. If it is not a unsigned integer slice, it returns nil. func (v Value) Uints() (u64s []uint64) { return values[uint64](v, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64) @@ -64,6 +85,12 @@ func (v Value) Float() float64 { return value[float64](v, reflect.Float32, reflect.Float64) } +// FloatOK converts a float value to float64 and reports whether the underlying +// type was a float. +func (v Value) FloatOK() (float64, bool) { + return valueOK[float64](v, reflect.Float32, reflect.Float64) +} + // Floats returns Value as a float slice. If it is not a float slice, it returns nil. func (v Value) Floats() (f64s []float64) { return values[float64](v, reflect.Float32, reflect.Float64) diff --git a/server/images.go b/server/images.go index 9515e3558..1993649e5 100644 --- a/server/images.go +++ b/server/images.go @@ -10,6 +10,7 @@ import ( "io" "log" "log/slog" + "math" "net" "net/http" "net/url" @@ -77,6 +78,7 @@ type Model struct { License []string Digest string Options map[string]any + GenerationDefaults model.GenerationDefaults Messages []api.Message Template *template.Template @@ -93,6 +95,47 @@ func (m *Model) isGGUF() bool { return m.Config.ModelFormat == "" || m.Config.ModelFormat == "gguf" } +func generationDefaultsFromGGUF(f *gguf.File) model.GenerationDefaults { + return model.ParseGGUFGenerationDefaults( + func(key string) (int64, bool) { + return ggufIntGenerationDefault(f.KeyValue(key)) + }, + func(key string) (float64, bool) { + return ggufFloatGenerationDefault(f.KeyValue(key)) + }, + ) +} + +func ggufIntGenerationDefault(kv gguf.KeyValue) (int64, bool) { + if value, ok := kv.IntOK(); ok { + return value, true + } + if value, ok := kv.UintOK(); ok { + if value > math.MaxInt64 { + return 0, false + } + return int64(value), true + } + if value, ok := kv.FloatOK(); ok { + // Match api.Options.FromMap; rounding may be better for near-integers. + return int64(value), true + } + return 0, false +} + +func ggufFloatGenerationDefault(kv gguf.KeyValue) (float64, bool) { + if value, ok := kv.FloatOK(); ok { + return value, true + } + if value, ok := kv.IntOK(); ok { + return float64(value), true + } + if value, ok := kv.UintOK(); ok { + return float64(value), true + } + return 0, false +} + func appendCapability(capabilities []model.Capability, capability model.Capability) []model.Capability { if slices.Contains(capabilities, capability) { return capabilities @@ -703,6 +746,7 @@ func GetModel(name string) (*Model, error) { if err := json.NewDecoder(configFile).Decode(&m.Config); err != nil { return nil, err } + m.GenerationDefaults = m.Config.GenerationDefaults } modelHasPooling := false @@ -726,6 +770,7 @@ func GetModel(name string) (*Model, error) { ggufChatTemplate = f.KeyValue("tokenizer.chat_template").String() m.HasChatTemplate = ggufChatTemplate != "" modelHasPooling = f.KeyValue("pooling_type").Valid() + m.GenerationDefaults = generationDefaultsFromGGUF(f) f.Close() } case manifest.MediaTypeImageDraft: diff --git a/server/images_test.go b/server/images_test.go index a3de7fc0f..4f2a59b52 100644 --- a/server/images_test.go +++ b/server/images_test.go @@ -14,6 +14,7 @@ import ( "github.com/ollama/ollama/api" "github.com/ollama/ollama/fs/ggml" + fsgguf "github.com/ollama/ollama/fs/gguf" "github.com/ollama/ollama/manifest" "github.com/ollama/ollama/template" "github.com/ollama/ollama/types/model" @@ -60,6 +61,63 @@ func TestPruneLayersSkipsRecentOrphans(t *testing.T) { } } +func TestGenerationDefaultsFromGGUF(t *testing.T) { + file, err := os.CreateTemp(t.TempDir(), "model-*.gguf") + if err != nil { + t.Fatal(err) + } + + if err := ggml.WriteGGUF(file, ggml.KV{ + "general.architecture": "llama", + "general.sampling.top_k": uint32(40), + "general.sampling.top_p": int32(1), + "general.sampling.min_p": float32(0), + "general.sampling.typ_p": float32(0.95), + "general.sampling.temp": uint32(1), + "general.sampling.penalty_last_n": float32(64), + "general.sampling.penalty_repeat": float32(1.05), + "general.sampling.penalty_freq": uint32(0), + "general.sampling.penalty_present": int32(0), + "general.sampling.xtc_threshold": float32(0.5), + "general.sampling.mirostat_tau": float32(5), + }, nil); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + + f, err := fsgguf.Open(file.Name()) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + defaults := generationDefaultsFromGGUF(f) + check := func(key string, want any) { + t.Helper() + if got := defaults[key]; got != want { + t.Fatalf("%s = %#v, want %#v", key, got, want) + } + } + + check("top_k", int64(40)) + check("top_p", float64(1)) + check("min_p", float64(0)) + check("typical_p", float64(float32(0.95))) + check("temperature", float64(1)) + check("repeat_last_n", int64(64)) + check("repeat_penalty", float64(float32(1.05))) + check("frequency_penalty", float64(0)) + check("presence_penalty", float64(0)) + if _, ok := defaults["mirostat_tau"]; ok { + t.Fatal("mirostat_tau should not be mapped to an Ollama option") + } + if _, ok := defaults["xtc_threshold"]; ok { + t.Fatal("xtc_threshold should not be mapped to an Ollama option") + } +} + func TestGetModelTemplateMetadata(t *testing.T) { customTemplate := "CUSTOM {{ .Prompt }}" diff --git a/server/routes.go b/server/routes.go index cddd82785..53e392978 100644 --- a/server/routes.go +++ b/server/routes.go @@ -138,6 +138,9 @@ func (s *Server) modelOptionsWithEmbeddingBatchDefault(model *Model, requestOpts draftNumPredictSet := hasOption(requestOpts, "draft_num_predict") if model != nil { draftNumPredictSet = draftNumPredictSet || hasOption(model.Options, "draft_num_predict") + if err := opts.FromMap(model.GenerationDefaults); err != nil { + return api.Options{}, err + } if err := opts.FromMap(model.Options); err != nil { return api.Options{}, err } diff --git a/server/routes_options_test.go b/server/routes_options_test.go index 0e0da0669..49e2a3d11 100644 --- a/server/routes_options_test.go +++ b/server/routes_options_test.go @@ -129,6 +129,52 @@ func TestModelOptionsNumCtxPriority(t *testing.T) { } } +func TestModelOptionsGenerationDefaultsPriority(t *testing.T) { + m := &Model{ + GenerationDefaults: model.GenerationDefaults{ + "top_k": int64(12), + "top_p": float64(0.7), + "min_p": float64(0.05), + "temperature": float64(0.4), + "repeat_last_n": int64(128), + "repeat_penalty": float64(1.2), + }, + Options: map[string]any{ + "top_p": float64(0.5), + "min_p": float64(0), + "repeat_last_n": float64(0), + }, + } + requestOpts := map[string]any{ + "temperature": float64(0), + "repeat_penalty": float64(1.5), + } + + opts, err := (&Server{}).modelOptions(m, requestOpts) + if err != nil { + t.Fatal(err) + } + + if opts.TopK != 12 { + t.Fatalf("TopK = %d, want 12", opts.TopK) + } + if opts.TopP != 0.5 { + t.Fatalf("TopP = %v, want 0.5", opts.TopP) + } + if opts.MinP != 0 { + t.Fatalf("MinP = %v, want 0", opts.MinP) + } + if opts.Temperature != 0 { + t.Fatalf("Temperature = %v, want 0", opts.Temperature) + } + if opts.RepeatLastN != 0 { + t.Fatalf("RepeatLastN = %d, want 0", opts.RepeatLastN) + } + if opts.RepeatPenalty != 1.5 { + t.Fatalf("RepeatPenalty = %v, want 1.5", opts.RepeatPenalty) + } +} + func TestModelOptionsEmbeddingNumBatchDefault(t *testing.T) { tests := []struct { name string diff --git a/types/model/config.go b/types/model/config.go index 235199010..3fd903d30 100644 --- a/types/model/config.go +++ b/types/model/config.go @@ -7,9 +7,12 @@ type ConfigV2 struct { ModelFamilies []string `json:"model_families"` ModelType string `json:"model_type"` // shown as Parameter Size FileType string `json:"file_type"` // shown as Quantization Level - Renderer string `json:"renderer,omitempty"` - Parser string `json:"parser,omitempty"` - Requires string `json:"requires,omitempty"` + // GenerationDefaults stores model-authored sampler defaults. These are + // lower precedence than Modelfile PARAMETERs and request options. + GenerationDefaults GenerationDefaults `json:"generation_defaults,omitempty"` + Renderer string `json:"renderer,omitempty"` + Parser string `json:"parser,omitempty"` + Requires string `json:"requires,omitempty"` RemoteHost string `json:"remote_host,omitempty"` RemoteModel string `json:"remote_model,omitempty"` diff --git a/types/model/generation.go b/types/model/generation.go new file mode 100644 index 000000000..ec79bb467 --- /dev/null +++ b/types/model/generation.go @@ -0,0 +1,146 @@ +package model + +import "encoding/json" + +// GenerationDefaults contains model-authored sampler defaults keyed by Ollama +// option names. +type GenerationDefaults map[string]any + +type generationDefaultKind int + +const ( + generationDefaultInt generationDefaultKind = iota + generationDefaultFloat +) + +type generationDefaultMapping struct { + option string + hfKeys []string + ggufKeys []string + kind generationDefaultKind +} + +func generationDefault(option string, kind generationDefaultKind, ggufKey string, hfKeys ...string) generationDefaultMapping { + return generationDefaultMapping{ + option: option, + hfKeys: hfKeys, + ggufKeys: []string{ggufKey}, + kind: kind, + } +} + +var generationDefaultMappings = []generationDefaultMapping{ + generationDefault("top_k", generationDefaultInt, "general.sampling.top_k", "top_k"), + generationDefault("top_p", generationDefaultFloat, "general.sampling.top_p", "top_p"), + generationDefault("min_p", generationDefaultFloat, "general.sampling.min_p", "min_p"), + generationDefault("typical_p", generationDefaultFloat, "general.sampling.typ_p", "typical_p"), + generationDefault("temperature", generationDefaultFloat, "general.sampling.temp", "temperature"), + generationDefault("repeat_last_n", generationDefaultInt, "general.sampling.penalty_last_n", "repeat_last_n", "penalty_last_n"), + generationDefault("repeat_penalty", generationDefaultFloat, "general.sampling.penalty_repeat", "repetition_penalty", "repeat_penalty", "penalty_repeat"), + generationDefault("presence_penalty", generationDefaultFloat, "general.sampling.penalty_present", "presence_penalty"), + generationDefault("frequency_penalty", generationDefaultFloat, "general.sampling.penalty_freq", "frequency_penalty"), +} + +// GenerationDefaultOptions returns the Ollama option names that can be populated +// from model-authored generation defaults. +func GenerationDefaultOptions() []string { + options := make([]string, 0, len(generationDefaultMappings)) + for _, mapping := range generationDefaultMappings { + options = append(options, mapping.option) + } + + return options +} + +// ParseHFGenerationDefaults extracts sampler defaults from Hugging Face +// generation_config.json data. +func ParseHFGenerationDefaults(data []byte) (GenerationDefaults, error) { + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return nil, err + } + + defaults := GenerationDefaults{} + for _, mapping := range generationDefaultMappings { + for _, key := range mapping.hfKeys { + b, ok := raw[key] + if !ok { + continue + } + + switch mapping.kind { + case generationDefaultInt: + if value, ok := intGenerationDefault(b); ok { + defaults[mapping.option] = value + } + case generationDefaultFloat: + if value, ok := floatGenerationDefault(b); ok { + defaults[mapping.option] = value + } + } + + if _, ok := defaults[mapping.option]; ok { + break + } + } + } + + if len(defaults) == 0 { + return nil, nil + } + + return defaults, nil +} + +// ParseGGUFGenerationDefaults extracts sampler defaults from GGUF metadata. +func ParseGGUFGenerationDefaults(intValue func(string) (int64, bool), floatValue func(string) (float64, bool)) GenerationDefaults { + defaults := GenerationDefaults{} + for _, mapping := range generationDefaultMappings { + for _, key := range mapping.ggufKeys { + switch mapping.kind { + case generationDefaultInt: + if value, ok := intValue(key); ok { + defaults[mapping.option] = value + } + case generationDefaultFloat: + if value, ok := floatValue(key); ok { + defaults[mapping.option] = value + } + } + + if _, ok := defaults[mapping.option]; ok { + break + } + } + } + + if len(defaults) == 0 { + return nil + } + + return defaults +} + +func intGenerationDefault(data json.RawMessage) (int64, bool) { + var value int64 + if err := json.Unmarshal(data, &value); err == nil { + return value, true + } + + var f float64 + if err := json.Unmarshal(data, &f); err != nil { + return 0, false + } + + // Match api.Options.FromMap; rounding may be better for near-integers. + return int64(f), true +} + +func floatGenerationDefault(data json.RawMessage) (float64, bool) { + var value float64 + if err := json.Unmarshal(data, &value); err != nil { + return 0, false + } + + return value, true +} diff --git a/types/model/generation_test.go b/types/model/generation_test.go new file mode 100644 index 000000000..670024200 --- /dev/null +++ b/types/model/generation_test.go @@ -0,0 +1,79 @@ +package model + +import "testing" + +func TestParseHFGenerationDefaults(t *testing.T) { + defaults, err := ParseHFGenerationDefaults([]byte(`{ + "top_k": 40.0, + "top_p": 0.7, + "min_p": 0, + "typical_p": 0.95, + "temperature": 0.6, + "repetition_penalty": 1.05, + "penalty_repeat": 1.4, + "presence_penalty": 0.1, + "frequency_penalty": 0.2, + "penalty_last_n": 64.0 + }`)) + if err != nil { + t.Fatal(err) + } + + check := func(key string, want any) { + t.Helper() + if got := defaults[key]; got != want { + t.Fatalf("%s = %#v, want %#v", key, got, want) + } + } + + check("top_k", int64(40)) + check("top_p", float64(0.7)) + check("min_p", float64(0)) + check("typical_p", float64(0.95)) + check("temperature", float64(0.6)) + check("repeat_penalty", float64(1.05)) + check("presence_penalty", float64(0.1)) + check("frequency_penalty", float64(0.2)) + check("repeat_last_n", int64(64)) +} + +func TestParseHFGenerationDefaultsIgnoresUnsupportedValues(t *testing.T) { + defaults, err := ParseHFGenerationDefaults([]byte(`{ + "top_p": 0.8, + "do_sample": true, + "eos_token_id": 128001, + "pad_token_id": 128002, + "max_new_tokens": 2048, + "mirostat_tau": 5.0 + }`)) + if err != nil { + t.Fatal(err) + } + + if got := defaults["top_p"]; got != float64(0.8) { + t.Fatalf("top_p = %#v, want %#v", got, float64(0.8)) + } + + for _, key := range []string{"do_sample", "eos_token_id", "pad_token_id", "max_new_tokens", "mirostat_tau"} { + if _, ok := defaults[key]; ok { + t.Fatalf("%s should be ignored", key) + } + } +} + +func TestParseHFGenerationDefaultsSkipsInvalidValues(t *testing.T) { + defaults, err := ParseHFGenerationDefaults([]byte(`{ + "top_k": "40", + "top_p": 0.8 + }`)) + if err != nil { + t.Fatal(err) + } + + if _, ok := defaults["top_k"]; ok { + t.Fatal("top_k should be skipped when it is not numeric") + } + if got := defaults["top_p"]; got != float64(0.8) { + t.Fatalf("top_p = %#v, want %#v", got, float64(0.8)) + } +} diff --git a/x/create/client/create.go b/x/create/client/create.go index ae35caa20..3f101c29b 100644 --- a/x/create/client/create.go +++ b/x/create/client/create.go @@ -9,6 +9,7 @@ package client import ( "bytes" "encoding/json" + "errors" "fmt" "io" "os" @@ -331,6 +332,17 @@ func readConfigV2(m *imagemanifest.ModelManifest) (*model.ConfigV2, error) { return &cfg, nil } +func readHFGenerationDefaults(modelDir string) (model.GenerationDefaults, error) { + data, err := os.ReadFile(filepath.Join(modelDir, "generation_config.json")) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } else if err != nil { + return nil, err + } + + return model.ParseHFGenerationDefaults(data) +} + func inferSafetensorsCapabilities(modelDir, parserName string) []string { capabilities := []string{"completion"} @@ -400,6 +412,15 @@ func newManifestWriter(opts CreateOptions, capabilities []string, parserName, re } configData.Parser = resolveParserName(opts.Modelfile, parserName) configData.Renderer = resolveRendererName(opts.Modelfile, rendererName) + if slices.Contains(capabilities, "completion") { + defaults, err := readHFGenerationDefaults(opts.ModelDir) + if err != nil { + return fmt.Errorf("failed to read generation_config.json: %w", err) + } + if len(defaults) > 0 { + configData.GenerationDefaults = defaults + } + } if opts.Modelfile != nil && opts.Modelfile.Draft != "" { draft, err := draftMetadata(opts.Modelfile.Draft) if err != nil { diff --git a/x/create/client/create_test.go b/x/create/client/create_test.go index 607f6b7d4..1d30696fb 100644 --- a/x/create/client/create_test.go +++ b/x/create/client/create_test.go @@ -554,6 +554,67 @@ func TestNewManifestWriter_PopulatesFileTypeFromEffectiveQuantize(t *testing.T) } } +func TestNewManifestWriter_PopulatesGenerationDefaults(t *testing.T) { + t.Setenv("OLLAMA_MODELS", t.TempDir()) + + modelDir := t.TempDir() + if err := os.WriteFile(filepath.Join(modelDir, "generation_config.json"), []byte(`{ + "temperature": 0, + "top_k": 12, + "top_p": 0.7, + "min_p": 0.05, + "repetition_penalty": 1.2, + "penalty_last_n": -1 + }`), 0o644); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + + opts := CreateOptions{ + ModelName: "test-generation-defaults", + ModelDir: modelDir, + } + + writer := newManifestWriter(opts, []string{"completion"}, "qwen3", "qwen3") + if err := writer(opts.ModelName, create.LayerInfo{}, nil, create.Classification{}); err != nil { + t.Fatalf("newManifestWriter() error = %v", err) + } + + name := model.ParseName(opts.ModelName) + mf, err := manifest.ParseNamedManifest(name) + if err != nil { + t.Fatalf("ParseNamedManifest() error = %v", err) + } + + configPath, err := manifest.BlobsPath(mf.Config.Digest) + if err != nil { + t.Fatalf("BlobsPath() error = %v", err) + } + + data, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + + var cfg model.ConfigV2 + if err := json.Unmarshal(data, &cfg); err != nil { + t.Fatalf("Unmarshal() error = %v", err) + } + + check := func(key string, want any) { + t.Helper() + if got := cfg.GenerationDefaults[key]; got != want { + t.Fatalf("%s = %#v, want %#v", key, got, want) + } + } + + check("temperature", float64(0)) + check("top_k", float64(12)) + check("top_p", float64(0.7)) + check("min_p", float64(0.05)) + check("repeat_penalty", float64(1.2)) + check("repeat_last_n", float64(-1)) +} + func TestNewManifestWriter_PopulatesDraftMetadata(t *testing.T) { t.Setenv("OLLAMA_MODELS", t.TempDir())