From 4b2d529966f504db4b6fb76b25d030194a8a1efe Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Tue, 19 May 2026 15:53:08 -0700 Subject: [PATCH] Reduce startup model hydration (#16215) * Reduce startup model hydration Add a lightweight model list cache for tags and launch inventory, while keeping show cache population lazy. This avoids loading every local model at startup on large model stores. * harden flaky scheduler unit test * remove extra launch model metadata text * review comments * review comments --- api/types.go | 19 +- cmd/launch/integrations_test.go | 10 +- cmd/launch/launch.go | 27 +- cmd/launch/launch_test.go | 48 ++ cmd/launch/models.go | 27 +- cmd/launch/models_test.go | 83 ++++ server/create.go | 2 + server/model_caches.go | 5 + server/model_list_cache.go | 824 ++++++++++++++++++++++++++++++++ server/model_list_cache_test.go | 239 +++++++++ server/model_show_cache.go | 65 +-- server/model_show_cache_test.go | 19 + server/routes.go | 56 +-- server/routes_list_test.go | 14 +- server/routes_test.go | 14 +- server/sched_test.go | 30 +- 16 files changed, 1361 insertions(+), 121 deletions(-) create mode 100644 cmd/launch/models_test.go create mode 100644 server/model_list_cache.go create mode 100644 server/model_list_cache_test.go diff --git a/api/types.go b/api/types.go index 51ddafb86..a46d3edb1 100644 --- a/api/types.go +++ b/api/types.go @@ -824,14 +824,15 @@ type ProcessResponse struct { // ListModelResponse is a single model description in [ListResponse]. type ListModelResponse struct { - Name string `json:"name"` - Model string `json:"model"` - RemoteModel string `json:"remote_model,omitempty"` - RemoteHost string `json:"remote_host,omitempty"` - ModifiedAt time.Time `json:"modified_at"` - Size int64 `json:"size"` - Digest string `json:"digest"` - Details ModelDetails `json:"details,omitempty"` + Name string `json:"name"` + Model string `json:"model"` + RemoteModel string `json:"remote_model,omitempty"` + RemoteHost string `json:"remote_host,omitempty"` + ModifiedAt time.Time `json:"modified_at"` + Size int64 `json:"size"` + Digest string `json:"digest"` + Details ModelDetails `json:"details,omitempty"` + Capabilities []model.Capability `json:"capabilities,omitempty"` } // ProcessModelResponse is a single model description in [ProcessResponse]. @@ -924,6 +925,8 @@ type ModelDetails struct { Families []string `json:"families"` ParameterSize string `json:"parameter_size"` QuantizationLevel string `json:"quantization_level"` + ContextLength int `json:"context_length,omitempty"` + EmbeddingLength int `json:"embedding_length,omitempty"` } // UserResponse provides information about a user. diff --git a/cmd/launch/integrations_test.go b/cmd/launch/integrations_test.go index 7e7879bba..f1ef59ee5 100644 --- a/cmd/launch/integrations_test.go +++ b/cmd/launch/integrations_test.go @@ -481,11 +481,11 @@ func TestBuildModelList_ExistingRecommendedMarked(t *testing.T) { func TestBuildModelList_PreservesRecommendationRequiredPlanForExistingCloudModel(t *testing.T) { recommendations := []ModelItem{ { - Name: "glm-5:cloud", - Description: "Reasoning and code generation", - Recommended: true, - RequiredPlan: "pro", - ContextLength: 202_752, + Name: "glm-5:cloud", + Description: "Reasoning and code generation", + Recommended: true, + RequiredPlan: "pro", + Details: api.ModelDetails{ContextLength: 202_752}, }, } existing := []modelInfo{{Name: "glm-5:cloud", Remote: true}} diff --git a/cmd/launch/launch.go b/cmd/launch/launch.go index 0cb81c14e..9311ca393 100644 --- a/cmd/launch/launch.go +++ b/cmd/launch/launch.go @@ -12,6 +12,7 @@ import ( "github.com/ollama/ollama/api" "github.com/ollama/ollama/cmd/config" + modelpkg "github.com/ollama/ollama/types/model" "github.com/spf13/cobra" "golang.org/x/term" ) @@ -240,9 +241,12 @@ type SupportedIntegration interface { } type modelInfo struct { - Name string - Remote bool - ToolCapable bool + Name string + Remote bool + ToolCapable bool + Capabilities []modelpkg.Capability + Size int64 + Details api.ModelDetails } // ModelInfo re-exports launcher model inventory details for callers. @@ -254,9 +258,12 @@ type ModelItem struct { Description string Recommended bool VRAMBytes int64 - ContextLength int MaxOutputTokens int RequiredPlan string + ToolCapable bool + Capabilities []modelpkg.Capability + Size int64 + Details api.ModelDetails } // SelectionItem represents a model row after launch has derived selector-only UI state. @@ -1179,9 +1186,11 @@ func (c *launcherClient) requestRecommendations(ctx context.Context) ([]ModelIte Description: description, Recommended: true, VRAMBytes: rec.VRAMBytes, - ContextLength: rec.ContextLength, MaxOutputTokens: rec.MaxOutputTokens, RequiredPlan: strings.TrimSpace(rec.RequiredPlan), + Details: api.ModelDetails{ + ContextLength: rec.ContextLength, + }, }) } @@ -1367,8 +1376,12 @@ func (c *launcherClient) loadModelInventoryOnce(ctx context.Context) error { c.modelInventory = c.modelInventory[:0] for _, model := range resp.Models { c.modelInventory = append(c.modelInventory, ModelInfo{ - Name: model.Name, - Remote: model.RemoteModel != "", + Name: model.Name, + Remote: model.RemoteModel != "", + ToolCapable: slices.Contains(model.Capabilities, modelpkg.CapabilityTools), + Capabilities: slices.Clone(model.Capabilities), + Size: model.Size, + Details: model.Details, }) } diff --git a/cmd/launch/launch_test.go b/cmd/launch/launch_test.go index c0903aaf9..c813ce2c9 100644 --- a/cmd/launch/launch_test.go +++ b/cmd/launch/launch_test.go @@ -11,6 +11,7 @@ import ( "runtime" "slices" "strings" + "sync/atomic" "testing" "time" @@ -1462,6 +1463,53 @@ func TestBuildLauncherState_ToleratesInventoryFailure(t *testing.T) { } } +func TestBuildLauncherState_UsesTagsInventoryWithoutShow(t *testing.T) { + tmpDir := t.TempDir() + setLaunchTestHome(t, tmpDir) + + if err := config.SetLastModel("llama3.2"); err != nil { + t.Fatalf("failed to seed last model: %v", err) + } + if err := config.SaveIntegration("codex", []string{"qwen3:8b"}); err != nil { + t.Fatalf("failed to seed codex config: %v", err) + } + + var showCalls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/tags": + fmt.Fprint(w, `{"models":[`+ + `{"name":"llama3.2","capabilities":["completion","tools"],"context_length":131072,"size":3200000000},`+ + `{"name":"qwen3:8b","capabilities":["completion","tools"],"context_length":65536,"size":4500000000}`+ + `]}`) + case "/api/show": + showCalls.Add(1) + fmt.Fprint(w, `{"model_info":{"general.context_length":131072}}`) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + t.Setenv("OLLAMA_HOST", srv.URL) + + state, err := BuildLauncherState(context.Background()) + if err != nil { + t.Fatalf("BuildLauncherState returned error: %v", err) + } + if !state.RunModelUsable { + t.Fatal("expected saved run model to be usable from tags inventory") + } + if state.Integrations["codex"].CurrentModel != "qwen3:8b" { + t.Fatalf("expected codex current model from saved config, got %q", state.Integrations["codex"].CurrentModel) + } + if !state.Integrations["codex"].ModelUsable { + t.Fatal("expected saved codex model to be usable from tags inventory") + } + if got := showCalls.Load(); got != 0 { + t.Fatalf("show calls = %d, want 0 for broad launcher state", got) + } +} + func TestResolveRunModel_UsesSavedModelWithoutSelector(t *testing.T) { tmpDir := t.TempDir() setLaunchTestHome(t, tmpDir) diff --git a/cmd/launch/models.go b/cmd/launch/models.go index 2283af541..2114dfb8e 100644 --- a/cmd/launch/models.go +++ b/cmd/launch/models.go @@ -23,10 +23,10 @@ import ( ) var recommendedModels = []ModelItem{ - {Name: "kimi-k2.6:cloud", Description: "State-of-the-art coding, long-horizon execution, and multimodal agent swarm capability", Recommended: true, ContextLength: 262_144, MaxOutputTokens: 262_144}, - {Name: "qwen3.5:cloud", Description: "Reasoning, coding, and agentic tool use with vision", Recommended: true, ContextLength: 262_144, MaxOutputTokens: 32_768}, - {Name: "glm-5.1:cloud", Description: "Reasoning and code generation", Recommended: true, ContextLength: 202_752, MaxOutputTokens: 131_072}, - {Name: "minimax-m2.7:cloud", Description: "Fast, efficient coding and real-world productivity", Recommended: true, ContextLength: 204_800, MaxOutputTokens: 128_000}, + {Name: "kimi-k2.6:cloud", Description: "State-of-the-art coding, long-horizon execution, and multimodal agent swarm capability", Recommended: true, Details: api.ModelDetails{ContextLength: 262_144}, MaxOutputTokens: 262_144}, + {Name: "qwen3.5:cloud", Description: "Reasoning, coding, and agentic tool use with vision", Recommended: true, Details: api.ModelDetails{ContextLength: 262_144}, MaxOutputTokens: 32_768}, + {Name: "glm-5.1:cloud", Description: "Reasoning and code generation", Recommended: true, Details: api.ModelDetails{ContextLength: 202_752}, MaxOutputTokens: 131_072}, + {Name: "minimax-m2.7:cloud", Description: "Fast, efficient coding and real-world productivity", Recommended: true, Details: api.ModelDetails{ContextLength: 204_800}, MaxOutputTokens: 128_000}, {Name: "gemma4", Description: "Reasoning and code generation locally", Recommended: true, VRAMBytes: 12 * format.GigaByte}, {Name: "qwen3.5", Description: "Reasoning, coding, and visual understanding locally", Recommended: true, VRAMBytes: 14 * format.GigaByte}, } @@ -115,7 +115,7 @@ func setDynamicCloudModelLimits(limits map[string]cloudModelLimit) { func cloudModelLimitsFromRecommendations(recommendations []ModelItem) map[string]cloudModelLimit { limits := make(map[string]cloudModelLimit, len(recommendations)) for _, rec := range recommendations { - if !isCloudModelName(rec.Name) || rec.ContextLength <= 0 || rec.MaxOutputTokens <= 0 { + if !isCloudModelName(rec.Name) || rec.Details.ContextLength <= 0 || rec.MaxOutputTokens <= 0 { continue } base, stripped := modelref.StripCloudSourceTag(rec.Name) @@ -123,7 +123,7 @@ func cloudModelLimitsFromRecommendations(recommendations []ModelItem) map[string continue } limits[base] = cloudModelLimit{ - Context: rec.ContextLength, + Context: rec.Details.ContextLength, Output: rec.MaxOutputTokens, } } @@ -365,11 +365,11 @@ func buildModelListWithRecommendations(existing []modelInfo, recommendations []M } displayName := strings.TrimSuffix(m.Name, ":latest") existingModels[displayName] = true - item := ModelItem{Name: displayName, Recommended: recommended[displayName], Description: recDesc[displayName]} if rec, ok := recByName[displayName]; ok { - item = copyModelRecommendationFields(displayName, rec) + items = append(items, modelItemFromInventory(displayName, m, copyModelRecommendationFields(displayName, rec))) + } else { + items = append(items, modelItemFromInventory(displayName, m, ModelItem{Name: displayName, Recommended: recommended[displayName], Description: recDesc[displayName]})) } - items = append(items, item) } for _, rec := range recommendations { @@ -483,6 +483,15 @@ func copyModelRecommendationFields(name string, rec ModelItem) ModelItem { return rec } +func modelItemFromInventory(name string, info modelInfo, item ModelItem) ModelItem { + item.Name = name + item.ToolCapable = info.ToolCapable + item.Capabilities = slices.Clone(info.Capabilities) + item.Size = info.Size + item.Details = info.Details + return item +} + // isCloudModelName reports whether the model name has an explicit cloud source. func isCloudModelName(name string) bool { return modelref.HasExplicitCloudSource(name) diff --git a/cmd/launch/models_test.go b/cmd/launch/models_test.go new file mode 100644 index 000000000..30abf4091 --- /dev/null +++ b/cmd/launch/models_test.go @@ -0,0 +1,83 @@ +package launch + +import ( + "testing" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/format" + modelpkg "github.com/ollama/ollama/types/model" +) + +func TestBuildModelList_UsesInventoryMetadataForInstalledModels(t *testing.T) { + existing := []modelInfo{ + { + Name: "custom-tools:latest", + ToolCapable: true, + Capabilities: []modelpkg.Capability{modelpkg.CapabilityCompletion, modelpkg.CapabilityTools, modelpkg.CapabilityThinking}, + Size: 7500 * format.MegaByte, + Details: api.ModelDetails{ + ParameterSize: "8B", + QuantizationLevel: "Q4_K_M", + ContextLength: 131_072, + EmbeddingLength: 4096, + }, + }, + } + + items, _, _, _ := buildModelList(existing, nil, "") + var got ModelItem + for _, item := range items { + if item.Name == "custom-tools" { + got = item + break + } + } + if got.Name == "" { + t.Fatal("custom-tools not found in items") + } + if !got.ToolCapable { + t.Fatal("expected installed model to preserve tool capability from tags metadata") + } + if got.Details.ContextLength != 131_072 { + t.Fatalf("Details.ContextLength = %d, want 131072", got.Details.ContextLength) + } + if got.Size != 7500*format.MegaByte { + t.Fatalf("Size = %d, want %d", got.Size, 7500*format.MegaByte) + } + if got.Description != "" { + t.Fatalf("Description = %q, want empty for installed model without recommendation copy", got.Description) + } +} + +func TestBuildModelList_InstalledRecommendedPreservesRecommendationAndMetadata(t *testing.T) { + existing := []modelInfo{ + { + Name: "qwen3.5", + ToolCapable: true, + Capabilities: []modelpkg.Capability{modelpkg.CapabilityCompletion, modelpkg.CapabilityTools, modelpkg.CapabilityVision}, + Size: 14 * format.GigaByte, + Details: api.ModelDetails{ContextLength: 262_144}, + }, + } + + items, _, _, _ := buildModelList(existing, nil, "") + var got ModelItem + for _, item := range items { + if item.Name == "qwen3.5" { + got = item + break + } + } + if got.Name == "" { + t.Fatal("qwen3.5 not found in items") + } + if !got.Recommended || !got.ToolCapable { + t.Fatalf("recommended/tool metadata = %v/%v, want true/true", got.Recommended, got.ToolCapable) + } + if got.Details.ContextLength != 262_144 { + t.Fatalf("Details.ContextLength = %d, want 262144", got.Details.ContextLength) + } + if got.Description != "Reasoning, coding, and visual understanding locally" { + t.Fatalf("Description = %q, want recommendation description", got.Description) + } +} diff --git a/server/create.go b/server/create.go index 49ca901e6..a6600a42f 100644 --- a/server/create.go +++ b/server/create.go @@ -271,6 +271,8 @@ func (s *Server) CreateHandler(c *gin.Context) { } } + s.refreshModelListCache(name) + ch <- api.ProgressResponse{Status: "success"} }() diff --git a/server/model_caches.go b/server/model_caches.go index 720fa703d..6c09b477c 100644 --- a/server/model_caches.go +++ b/server/model_caches.go @@ -5,12 +5,14 @@ import "context" type modelCaches struct { recommendations *modelRecommendationsCache show *modelShowCache + modelList *modelListCache } func newModelCaches() *modelCaches { return &modelCaches{ recommendations: newModelRecommendationsCache(), show: newModelShowCache(), + modelList: newModelListCache(), } } @@ -24,4 +26,7 @@ func (c *modelCaches) Start(ctx context.Context) { if c.show != nil { c.show.Start(ctx) } + if c.modelList != nil { + c.modelList.Start(ctx) + } } diff --git a/server/model_list_cache.go b/server/model_list_cache.go new file mode 100644 index 000000000..8889ed63e --- /dev/null +++ b/server/model_list_cache.go @@ -0,0 +1,824 @@ +package server + +import ( + "bufio" + "cmp" + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "log/slog" + "os" + "slices" + "strings" + "sync" + "time" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/manifest" + "github.com/ollama/ollama/model/parsers" + ollamatemplate "github.com/ollama/ollama/template" + "github.com/ollama/ollama/thinking" + "github.com/ollama/ollama/types/model" +) + +type modelListSummary struct { + Model string + Name string + RemoteModel string + RemoteHost string + Size int64 + Digest string + ModifiedAt time.Time + Details api.ModelDetails + Capabilities []model.Capability +} + +type modelListCacheEntry struct { + Digest string + Summary modelListSummary +} + +type modelListCache struct { + mu sync.RWMutex + + entries map[string]modelListCacheEntry + + once sync.Once + readyOnce sync.Once + ready chan struct{} + hydrateErr error + build func(model.Name, *manifest.Manifest) (modelListSummary, error) +} + +func newModelListCache() *modelListCache { + return &modelListCache{ + entries: make(map[string]modelListCacheEntry), + ready: make(chan struct{}), + build: buildModelListSummary, + } +} + +func (c *modelListCache) Start(ctx context.Context) { + if c == nil { + return + } + + c.once.Do(func() { + slog.Debug("starting model list cache") + go func() { + err := c.hydrate(ctx) + c.markReady(err) + if err != nil { + if ctx != nil && ctx.Err() != nil { + return + } + slog.Warn("model list cache hydration failed", "error", err) + } + }() + }) +} + +func (c *modelListCache) hydrate(ctx context.Context) error { + start := time.Now() + + manifests, err := manifest.Manifests(true) + if err != nil { + return err + } + + var hydrated, failed int + for name, mf := range manifests { + if ctx != nil { + if err := ctx.Err(); err != nil { + return err + } + } + + summary, err := c.build(name, mf) + if err != nil { + failed++ + slog.Warn("failed to hydrate model list cache", "model", name.String(), "error", err) + continue + } + + c.set(name, mf.Digest(), summary) + hydrated++ + } + + slog.Info("model list cache hydration complete", "models", hydrated, "failures", failed, "elapsed", time.Since(start)) + return nil +} + +func (c *modelListCache) markReady(err error) { + c.mu.Lock() + c.hydrateErr = err + c.mu.Unlock() + + c.readyOnce.Do(func() { + close(c.ready) + }) +} + +func (c *modelListCache) Wait(ctx context.Context) error { + if c == nil { + return nil + } + if ctx == nil { + ctx = context.Background() + } + + select { + case <-c.ready: + c.mu.RLock() + err := c.hydrateErr + c.mu.RUnlock() + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (c *modelListCache) List(ctx context.Context) ([]api.ListModelResponse, error) { + if err := c.Wait(ctx); err != nil { + return nil, err + } + if err := c.syncManifests(ctx); err != nil { + return nil, err + } + + c.mu.RLock() + models := make([]api.ListModelResponse, 0, len(c.entries)) + for _, entry := range c.entries { + models = append(models, entry.Summary.ListModelResponse()) + } + c.mu.RUnlock() + + sortListModelResponses(models) + return models, nil +} + +func (c *modelListCache) syncManifests(ctx context.Context) error { + manifests, err := manifest.Manifests(true) + if err != nil { + return err + } + + c.mu.RLock() + current := make(map[string]string, len(c.entries)) + for name, entry := range c.entries { + current[name] = entry.Digest + } + c.mu.RUnlock() + + type update struct { + name model.Name + digest string + summary modelListSummary + } + + seen := make(map[string]struct{}, len(manifests)) + stale := make(map[string]struct{}) + var updates []update + for name, mf := range manifests { + if ctx != nil { + if err := ctx.Err(); err != nil { + return err + } + } + + key := name.String() + digest := mf.Digest() + seen[key] = struct{}{} + if current[key] == digest { + continue + } + + summary, err := c.build(name, mf) + if err != nil { + slog.Warn("failed to refresh model list cache", "model", key, "error", err) + if _, ok := current[key]; ok { + stale[key] = struct{}{} + } + continue + } + updates = append(updates, update{name: name, digest: digest, summary: summary}) + } + + c.mu.Lock() + for name := range c.entries { + if _, ok := seen[name]; !ok { + delete(c.entries, name) + continue + } + if _, ok := stale[name]; ok { + delete(c.entries, name) + } + } + for _, update := range updates { + c.entries[update.name.String()] = modelListCacheEntry{ + Digest: update.digest, + Summary: cloneModelListSummary(update.summary), + } + } + c.mu.Unlock() + + return nil +} + +func (c *modelListCache) RefreshModel(name model.Name) error { + if c == nil { + return nil + } + + if !name.IsFullyQualified() { + var err error + name, err = getExistingName(name) + if err != nil { + return err + } + } + + mf, err := manifest.ParseNamedManifest(name) + if err != nil { + c.DeleteModel(name) + return err + } + + summary, err := c.build(name, mf) + if err != nil { + c.DeleteModel(name) + return err + } + + c.set(name, mf.Digest(), summary) + return nil +} + +func (c *modelListCache) DeleteModel(name model.Name) { + if c == nil { + return + } + + c.mu.Lock() + delete(c.entries, name.String()) + c.mu.Unlock() +} + +func (c *modelListCache) Get(name model.Name) (modelListSummary, bool) { + if c == nil { + return modelListSummary{}, false + } + + if !name.IsFullyQualified() { + if existing, err := getExistingName(name); err == nil { + name = existing + } + } + + c.mu.RLock() + entry, ok := c.entries[name.String()] + c.mu.RUnlock() + if !ok { + return modelListSummary{}, false + } + + return cloneModelListSummary(entry.Summary), true +} + +func (c *modelListCache) Len() int { + if c == nil { + return 0 + } + + c.mu.RLock() + defer c.mu.RUnlock() + return len(c.entries) +} + +func (c *modelListCache) set(name model.Name, digest string, summary modelListSummary) { + c.mu.Lock() + c.entries[name.String()] = modelListCacheEntry{ + Digest: digest, + Summary: cloneModelListSummary(summary), + } + c.mu.Unlock() +} + +func buildModelListSummary(name model.Name, mf *manifest.Manifest) (modelListSummary, error) { + cfg, err := readModelListConfig(mf) + if err != nil { + return modelListSummary{}, err + } + + var modified time.Time + if fi := mf.FileInfo(); fi != nil { + modified = fi.ModTime() + } + + summary := modelListSummary{ + Model: name.DisplayShortest(), + Name: name.DisplayShortest(), + RemoteModel: cfg.RemoteModel, + RemoteHost: cfg.RemoteHost, + Size: mf.Size(), + Digest: mf.Digest(), + ModifiedAt: modified, + Details: api.ModelDetails{ + Format: cfg.ModelFormat, + Family: cfg.ModelFamily, + Families: append([]string(nil), cfg.ModelFamilies...), + ParameterSize: cfg.ModelType, + QuantizationLevel: cfg.FileType, + ContextLength: cfg.ContextLen, + EmbeddingLength: cfg.EmbedLen, + }, + } + + modelPath, projectorCount, tmpl, err := readModelListLayers(mf, &summary) + if err != nil { + return modelListSummary{}, err + } + + if cfg.RemoteHost == "" && cfg.RemoteModel == "" && modelPath != "" { + info, err := readModelListGGUF(modelPath) + if err != nil { + slog.Debug("failed to read gguf model metadata", "model", name.String(), "error", err) + } else { + summary.Capabilities = appendModelListCapabilities(summary.Capabilities, info.Capabilities...) + if summary.Details.ContextLength == 0 { + summary.Details.ContextLength = info.ContextLength + } + if summary.Details.EmbeddingLength == 0 { + summary.Details.EmbeddingLength = info.EmbeddingLength + } + } + } + + for _, c := range cfg.Capabilities { + summary.Capabilities = appendModelListCapability(summary.Capabilities, model.Capability(c)) + } + + builtinParser := parsers.ParserForName(cfg.Parser) + if tmpl != nil { + vars, err := tmpl.Vars() + if err != nil { + slog.Warn("model template contains errors", "model", name.String(), "error", err) + } + if slices.Contains(vars, "tools") || (builtinParser != nil && builtinParser.HasToolSupport()) { + summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityTools) + } + if slices.Contains(vars, "suffix") { + summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityInsert) + } + + openingTag, closingTag := thinking.InferTags(tmpl.Template) + hasTags := openingTag != "" && closingTag != "" + isGptoss := slices.Contains([]string{"gptoss", "gpt-oss"}, cfg.ModelFamily) + if !slices.Contains(summary.Capabilities, model.CapabilityThinking) && + (hasTags || isGptoss || (builtinParser != nil && builtinParser.HasThinkingSupport())) { + summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityThinking) + } + } + + if projectorCount > 0 { + summary.Capabilities = appendModelListCapability(summary.Capabilities, model.CapabilityVision) + } + + if cfg.ModelFormat == "safetensors" && isGemma4Renderer(cfg.Renderer) { + summary.Capabilities = slices.DeleteFunc(summary.Capabilities, func(c model.Capability) bool { + return c == model.CapabilityVision || c == model.CapabilityAudio + }) + } + + return summary, nil +} + +func readModelListConfig(mf *manifest.Manifest) (model.ConfigV2, error) { + var cfg model.ConfigV2 + if mf == nil || mf.Config.Digest == "" { + return cfg, nil + } + + f, err := mf.Config.Open() + if err != nil { + return cfg, err + } + defer f.Close() + + if err := json.NewDecoder(f).Decode(&cfg); err != nil { + return cfg, err + } + + return cfg, nil +} + +func readModelListLayers(mf *manifest.Manifest, summary *modelListSummary) (string, int, *ollamatemplate.Template, error) { + var modelPath string + var projectorCount int + tmpl := ollamatemplate.DefaultTemplate + + for _, layer := range mf.Layers { + switch layer.MediaType { + case "application/vnd.ollama.image.model": + filename, err := manifest.BlobsPath(layer.Digest) + if err != nil { + return "", 0, nil, err + } + modelPath = filename + summary.Details.ParentModel = layer.From + case "application/vnd.ollama.image.projector": + projectorCount++ + case "application/vnd.ollama.image.prompt", + "application/vnd.ollama.image.template": + filename, err := manifest.BlobsPath(layer.Digest) + if err != nil { + return "", 0, nil, err + } + bts, err := os.ReadFile(filename) + if err != nil { + return "", 0, nil, err + } + + tmpl, err = ollamatemplate.Parse(string(bts)) + if err != nil { + return "", 0, nil, err + } + } + } + + return modelPath, projectorCount, tmpl, nil +} + +type modelListGGUF struct { + Capabilities []model.Capability + ContextLength int + EmbeddingLength int +} + +const ( + modelListGGUFMagicLE = 0x46554747 + modelListGGUFMagicBE = 0x47475546 +) + +const ( + modelListGGUFTypeUint8 uint32 = iota + modelListGGUFTypeInt8 + modelListGGUFTypeUint16 + modelListGGUFTypeInt16 + modelListGGUFTypeUint32 + modelListGGUFTypeInt32 + modelListGGUFTypeFloat32 + modelListGGUFTypeBool + modelListGGUFTypeString + modelListGGUFTypeArray + modelListGGUFTypeUint64 + modelListGGUFTypeInt64 + modelListGGUFTypeFloat64 +) + +// readModelListGGUF scans only the small GGUF header values launch needs +// and stops before tokenizer arrays. Using gguf.File.KeyValue for missing keys +// can otherwise advance through large arrays just to discover absence. +func readModelListGGUF(path string) (modelListGGUF, error) { + f, err := os.Open(path) + if err != nil { + return modelListGGUF{}, err + } + defer f.Close() + + r := bufio.NewReaderSize(f, 32<<10) + var magic uint32 + if err := binary.Read(r, binary.LittleEndian, &magic); err != nil { + return modelListGGUF{}, err + } + + var byteOrder binary.ByteOrder = binary.LittleEndian + switch magic { + case modelListGGUFMagicLE: + case modelListGGUFMagicBE: + byteOrder = binary.BigEndian + default: + return modelListGGUF{}, fmt.Errorf("invalid file magic") + } + + var version uint32 + if err := binary.Read(r, byteOrder, &version); err != nil { + return modelListGGUF{}, err + } + + var numKV uint64 + switch version { + case 1: + var header struct { + NumTensor uint32 + NumKV uint32 + } + if err := binary.Read(r, byteOrder, &header); err != nil { + return modelListGGUF{}, err + } + numKV = uint64(header.NumKV) + default: + var header struct { + NumTensor uint64 + NumKV uint64 + } + if err := binary.Read(r, byteOrder, &header); err != nil { + return modelListGGUF{}, err + } + numKV = header.NumKV + } + + info := modelListGGUF{} + var architecture string + var hasPoolingType bool + + for range numKV { + key, err := readModelListGGUFString(r, byteOrder, version) + if err != nil { + return modelListGGUF{}, err + } + + var valueType uint32 + if err := binary.Read(r, byteOrder, &valueType); err != nil { + return modelListGGUF{}, err + } + + if key == "general.architecture" { + value, err := readModelListGGUFStringValue(r, byteOrder, version, valueType) + if err != nil { + return modelListGGUF{}, err + } + architecture = value + continue + } + + if architecture != "" && strings.HasPrefix(key, "tokenizer.") { + break + } + + if architecture != "" && strings.HasPrefix(key, architecture+".") { + switch strings.TrimPrefix(key, architecture+".") { + case "pooling_type": + hasPoolingType = true + case "vision.block_count": + info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityVision) + case "audio.block_count": + info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityAudio) + case "context_length": + value, err := readModelListGGUFIntValue(r, byteOrder, version, valueType) + if err != nil { + return modelListGGUF{}, err + } + info.ContextLength = value + continue + case "embedding_length": + value, err := readModelListGGUFIntValue(r, byteOrder, version, valueType) + if err != nil { + return modelListGGUF{}, err + } + info.EmbeddingLength = value + continue + } + } + + if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil { + return modelListGGUF{}, err + } + } + + if hasPoolingType { + info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityEmbedding) + } else { + info.Capabilities = appendModelListCapability(info.Capabilities, model.CapabilityCompletion) + } + + return info, nil +} + +func readModelListGGUFStringValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) (string, error) { + if valueType != modelListGGUFTypeString { + if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil { + return "", err + } + return "", fmt.Errorf("unexpected gguf string type %d", valueType) + } + return readModelListGGUFString(r, byteOrder, version) +} + +func readModelListGGUFIntValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) (int, error) { + switch valueType { + case modelListGGUFTypeUint8: + var value uint8 + if err := binary.Read(r, byteOrder, &value); err != nil { + return 0, err + } + return int(value), nil + case modelListGGUFTypeInt8: + var value int8 + if err := binary.Read(r, byteOrder, &value); err != nil { + return 0, err + } + return int(value), nil + case modelListGGUFTypeUint16: + var value uint16 + if err := binary.Read(r, byteOrder, &value); err != nil { + return 0, err + } + return int(value), nil + case modelListGGUFTypeInt16: + var value int16 + if err := binary.Read(r, byteOrder, &value); err != nil { + return 0, err + } + return int(value), nil + case modelListGGUFTypeUint32: + var value uint32 + if err := binary.Read(r, byteOrder, &value); err != nil { + return 0, err + } + return int(value), nil + case modelListGGUFTypeInt32: + var value int32 + if err := binary.Read(r, byteOrder, &value); err != nil { + return 0, err + } + return int(value), nil + case modelListGGUFTypeUint64: + var value uint64 + if err := binary.Read(r, byteOrder, &value); err != nil { + return 0, err + } + return int(value), nil + case modelListGGUFTypeInt64: + var value int64 + if err := binary.Read(r, byteOrder, &value); err != nil { + return 0, err + } + return int(value), nil + default: + if err := skipModelListGGUFValue(r, byteOrder, version, valueType); err != nil { + return 0, err + } + return 0, fmt.Errorf("unexpected gguf integer type %d", valueType) + } +} + +func skipModelListGGUFValue(r io.Reader, byteOrder binary.ByteOrder, version uint32, valueType uint32) error { + switch valueType { + case modelListGGUFTypeUint8, modelListGGUFTypeInt8, modelListGGUFTypeBool: + return discardModelListGGUFBytes(r, 1) + case modelListGGUFTypeUint16, modelListGGUFTypeInt16: + return discardModelListGGUFBytes(r, 2) + case modelListGGUFTypeUint32, modelListGGUFTypeInt32, modelListGGUFTypeFloat32: + return discardModelListGGUFBytes(r, 4) + case modelListGGUFTypeUint64, modelListGGUFTypeInt64, modelListGGUFTypeFloat64: + return discardModelListGGUFBytes(r, 8) + case modelListGGUFTypeString: + return skipModelListGGUFString(r, byteOrder, version) + case modelListGGUFTypeArray: + var arrayType uint32 + if err := binary.Read(r, byteOrder, &arrayType); err != nil { + return err + } + var count uint64 + if err := binary.Read(r, byteOrder, &count); err != nil { + return err + } + return skipModelListGGUFArray(r, byteOrder, version, arrayType, count) + default: + return fmt.Errorf("unsupported gguf value type %d", valueType) + } +} + +func skipModelListGGUFArray(r io.Reader, byteOrder binary.ByteOrder, version uint32, arrayType uint32, count uint64) error { + var size uint64 + switch arrayType { + case modelListGGUFTypeUint8, modelListGGUFTypeInt8, modelListGGUFTypeBool: + size = 1 + case modelListGGUFTypeUint16, modelListGGUFTypeInt16: + size = 2 + case modelListGGUFTypeUint32, modelListGGUFTypeInt32, modelListGGUFTypeFloat32: + size = 4 + case modelListGGUFTypeUint64, modelListGGUFTypeInt64, modelListGGUFTypeFloat64: + size = 8 + case modelListGGUFTypeString: + for range count { + if err := skipModelListGGUFString(r, byteOrder, version); err != nil { + return err + } + } + return nil + default: + return fmt.Errorf("unsupported gguf array type %d", arrayType) + } + return discardModelListGGUFBytes(r, int64(count*size)) +} + +func readModelListGGUFString(r io.Reader, byteOrder binary.ByteOrder, version uint32) (string, error) { + var length uint64 + if err := binary.Read(r, byteOrder, &length); err != nil { + return "", err + } + + if length == 0 { + return "", nil + } + + bts := make([]byte, length) + if _, err := io.ReadFull(r, bts); err != nil { + return "", err + } + if version == 1 && bts[len(bts)-1] == 0 { + bts = bts[:len(bts)-1] + } + return string(bts), nil +} + +func skipModelListGGUFString(r io.Reader, byteOrder binary.ByteOrder, version uint32) error { + var length uint64 + if err := binary.Read(r, byteOrder, &length); err != nil { + return err + } + return discardModelListGGUFBytes(r, int64(length)) +} + +func discardModelListGGUFBytes(r io.Reader, n int64) error { + if n <= 0 { + return nil + } + _, err := io.CopyN(io.Discard, r, n) + return err +} + +func appendModelListCapabilities(capabilities []model.Capability, values ...model.Capability) []model.Capability { + for _, capability := range values { + capabilities = appendModelListCapability(capabilities, capability) + } + return capabilities +} + +func appendModelListCapability(capabilities []model.Capability, capability model.Capability) []model.Capability { + if capability == "" || slices.Contains(capabilities, capability) { + return capabilities + } + return append(capabilities, capability) +} + +func cloneModelListSummary(summary modelListSummary) modelListSummary { + summary.Details.Families = append([]string(nil), summary.Details.Families...) + summary.Capabilities = append([]model.Capability(nil), summary.Capabilities...) + return summary +} + +func (s modelListSummary) ListModelResponse() api.ListModelResponse { + resp := api.ListModelResponse{ + Model: s.Model, + Name: s.Name, + RemoteModel: s.RemoteModel, + RemoteHost: s.RemoteHost, + Size: s.Size, + Digest: s.Digest, + ModifiedAt: s.ModifiedAt, + Details: api.ModelDetails{ + ParentModel: s.Details.ParentModel, + Format: s.Details.Format, + Family: s.Details.Family, + Families: append([]string(nil), s.Details.Families...), + ParameterSize: s.Details.ParameterSize, + QuantizationLevel: s.Details.QuantizationLevel, + ContextLength: s.Details.ContextLength, + EmbeddingLength: s.Details.EmbeddingLength, + }, + } + + resp.Capabilities = append([]model.Capability(nil), s.Capabilities...) + + return resp +} + +func sortListModelResponses(models []api.ListModelResponse) { + slices.SortStableFunc(models, func(i, j api.ListModelResponse) int { + // Preserve the existing /api/tags order: most recently modified first. + return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix()) + }) +} + +func (s *Server) refreshModelListCache(name model.Name) { + if s == nil || s.modelCaches == nil || s.modelCaches.modelList == nil { + return + } + + if err := s.modelCaches.modelList.RefreshModel(name); err != nil { + slog.Warn("failed to refresh model list cache", "model", name.String(), "error", err) + } +} + +func (s *Server) deleteModelListCache(name model.Name) { + if s == nil || s.modelCaches == nil || s.modelCaches.modelList == nil { + return + } + + s.modelCaches.modelList.DeleteModel(name) +} diff --git a/server/model_list_cache_test.go b/server/model_list_cache_test.go new file mode 100644 index 000000000..1f40c7b3e --- /dev/null +++ b/server/model_list_cache_test.go @@ -0,0 +1,239 @@ +package server + +import ( + "context" + "errors" + "net/http" + "slices" + "testing" + + "github.com/gin-gonic/gin" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/manifest" + "github.com/ollama/ollama/types/model" +) + +func TestModelListCacheHydratesSummary(t *testing.T) { + gin.SetMode(gin.TestMode) + setTestHome(t, t.TempDir()) + createListCacheModel(t, "list-cache", map[string]any{ + "test.context_length": uint32(4096), + "test.embedding_length": uint32(384), + }, "{{ .prompt }}{{ if .tools }}{{ .tools }}{{ end }}{{ if .suffix }}{{ .suffix }}{{ end }}") + + cache := newModelListCache() + if err := cache.hydrate(context.Background()); err != nil { + t.Fatalf("hydrate failed: %v", err) + } + + summary, ok := cache.Get(model.ParseName("list-cache")) + if !ok { + t.Fatal("list summary missing") + } + + if summary.Model != "list-cache:latest" || summary.Name != "list-cache:latest" { + t.Fatalf("summary model/name = %q/%q, want list-cache:latest", summary.Model, summary.Name) + } + if summary.Digest == "" { + t.Fatal("summary digest is empty") + } + if summary.Size == 0 { + t.Fatal("summary size is zero") + } + if summary.Details.Family != "test" || summary.Details.Format != "gguf" { + t.Fatalf("summary details = %+v, want gguf/test", summary.Details) + } + if summary.Details.ContextLength != 4096 { + t.Fatalf("context length = %d, want 4096", summary.Details.ContextLength) + } + if summary.Details.EmbeddingLength != 384 { + t.Fatalf("embedding length = %d, want 384", summary.Details.EmbeddingLength) + } + + for _, capability := range []model.Capability{model.CapabilityCompletion, model.CapabilityTools, model.CapabilityInsert} { + if !slices.Contains(summary.Capabilities, capability) { + t.Fatalf("capabilities = %v, want %s", summary.Capabilities, capability) + } + } + + listModel := summary.ListModelResponse() + if !slices.Contains(listModel.Capabilities, model.CapabilityTools) || + listModel.Details.ContextLength != 4096 || + listModel.Details.EmbeddingLength != 384 { + t.Fatalf("list response = %+v, want capabilities/context/embedding", listModel) + } +} + +func TestModelListCacheRefreshUpdatesEntry(t *testing.T) { + gin.SetMode(gin.TestMode) + setTestHome(t, t.TempDir()) + createListCacheModel(t, "list-refresh", map[string]any{"test.context_length": uint32(1024)}, "") + + cache := newModelListCache() + if err := cache.hydrate(context.Background()); err != nil { + t.Fatalf("hydrate failed: %v", err) + } + + name := model.ParseName("list-refresh") + first, ok := cache.Get(name) + if !ok { + t.Fatal("list summary missing") + } + + changeShowCacheManifest(t, "list-refresh") + if err := cache.RefreshModel(name); err != nil { + t.Fatalf("refresh failed: %v", err) + } + + refreshed, ok := cache.Get(name) + if !ok { + t.Fatal("refreshed list summary missing") + } + if refreshed.Digest == first.Digest { + t.Fatalf("digest did not change after refresh: %s", refreshed.Digest) + } + if cache.Len() != 1 { + t.Fatalf("cache entries = %d, want 1", cache.Len()) + } +} + +func TestModelListCacheMutationHooks(t *testing.T) { + gin.SetMode(gin.TestMode) + setTestHome(t, t.TempDir()) + + cache := newModelListCache() + s := Server{modelCaches: &modelCaches{modelList: cache}} + + _, digest := createBinFile(t, map[string]any{"test.context_length": uint32(2048)}, nil) + w := createRequest(t, s.CreateHandler, api.CreateRequest{ + Model: "list-hooks", + Files: map[string]string{"model.gguf": digest}, + Stream: &stream, + }) + if w.Code != http.StatusOK { + t.Fatalf("create model status = %d, want 200: %s", w.Code, w.Body.String()) + } + + if _, ok := cache.Get(model.ParseName("list-hooks")); !ok { + t.Fatal("create did not refresh model list cache") + } + + w = createRequest(t, s.CopyHandler, api.CopyRequest{ + Source: "list-hooks", + Destination: "list-hooks-copy", + }) + if w.Code != http.StatusOK { + t.Fatalf("copy model status = %d, want 200: %s", w.Code, w.Body.String()) + } + if _, ok := cache.Get(model.ParseName("list-hooks-copy")); !ok { + t.Fatal("copy did not refresh model list cache") + } + + w = createRequest(t, s.DeleteHandler, api.DeleteRequest{Model: "list-hooks-copy"}) + if w.Code != http.StatusOK { + t.Fatalf("delete model status = %d, want 200: %s", w.Code, w.Body.String()) + } + if _, ok := cache.Get(model.ParseName("list-hooks-copy")); ok { + t.Fatal("delete did not remove model list cache entry") + } +} + +func TestModelListCacheSyncsManifestChanges(t *testing.T) { + gin.SetMode(gin.TestMode) + setTestHome(t, t.TempDir()) + createListCacheModel(t, "list-sync-a", map[string]any{"test.context_length": uint32(1024)}, "") + + cache := newModelListCache() + cache.Start(context.Background()) + if err := cache.Wait(context.Background()); err != nil { + t.Fatalf("wait failed: %v", err) + } + + createListCacheModel(t, "list-sync-b", map[string]any{"test.context_length": uint32(2048)}, "") + models, err := cache.List(context.Background()) + if err != nil { + t.Fatalf("list failed: %v", err) + } + + names := make([]string, 0, len(models)) + for _, m := range models { + names = append(names, m.Name) + } + for _, want := range []string{"list-sync-a:latest", "list-sync-b:latest"} { + if !slices.Contains(names, want) { + t.Fatalf("names = %v, want %s", names, want) + } + } + + var other Server + w := createRequest(t, other.DeleteHandler, api.DeleteRequest{Model: "list-sync-a"}) + if w.Code != http.StatusOK { + t.Fatalf("delete model status = %d, want 200: %s", w.Code, w.Body.String()) + } + + models, err = cache.List(context.Background()) + if err != nil { + t.Fatalf("list after delete failed: %v", err) + } + names = names[:0] + for _, m := range models { + names = append(names, m.Name) + } + if slices.Contains(names, "list-sync-a:latest") || !slices.Contains(names, "list-sync-b:latest") { + t.Fatalf("names after delete = %v, want only list-sync-b", names) + } +} + +func TestModelListCacheSyncDropsStaleEntryOnRefreshFailure(t *testing.T) { + gin.SetMode(gin.TestMode) + setTestHome(t, t.TempDir()) + createListCacheModel(t, "list-stale", map[string]any{"test.context_length": uint32(1024)}, "") + + cache := newModelListCache() + cache.Start(context.Background()) + if err := cache.Wait(context.Background()); err != nil { + t.Fatalf("wait failed: %v", err) + } + + name := model.ParseName("list-stale") + if _, ok := cache.Get(name); !ok { + t.Fatal("list summary missing") + } + + changeShowCacheManifest(t, "list-stale") + cache.build = func(model.Name, *manifest.Manifest) (modelListSummary, error) { + return modelListSummary{}, errors.New("refresh failed") + } + + models, err := cache.List(context.Background()) + if err != nil { + t.Fatalf("list failed: %v", err) + } + if len(models) != 0 { + t.Fatalf("models = %+v, want stale entry removed", models) + } + if _, ok := cache.Get(name); ok { + t.Fatal("stale entry remained in cache after refresh failure") + } +} + +func createListCacheModel(t *testing.T, name string, kv map[string]any, tmpl string) { + t.Helper() + _, digest := createBinFile(t, kv, nil) + + req := api.CreateRequest{ + Model: name, + Files: map[string]string{"model.gguf": digest}, + Stream: &stream, + } + if tmpl != "" { + req.Template = tmpl + } + + var s Server + w := createRequest(t, s.CreateHandler, req) + if w.Code != http.StatusOK { + t.Fatalf("create model status = %d, want 200: %s", w.Code, w.Body.String()) + } +} diff --git a/server/model_show_cache.go b/server/model_show_cache.go index b32e025e8..bab5f241f 100644 --- a/server/model_show_cache.go +++ b/server/model_show_cache.go @@ -26,13 +26,15 @@ import ( The /api/show cache stores full api.ShowResponse values because callers use more than capabilities: launch flows also need context length, embeddings metadata, quantization details, remote metadata, and model-specific fields. +TODO(parthsareen): Consider removing show cache if /api/tags grows to cover +the remaining callers. -Local model entries are stored by canonical model name and verbose flag, with -the manifest digest recorded in the entry. The manifest digest is the freshness -boundary: if the model content changes, the digest changes, so the previous -response is replaced instead of accumulating under an old digest key. Requests -with System or Options overlays bypass the cache because those overlays mutate -the effective show response. +Local model entries are stored lazily by canonical model name and verbose flag, +with the manifest digest recorded in the entry. The manifest digest is the +freshness boundary: if the model content changes, the digest changes, so the +previous response is replaced instead of accumulating under an old digest key. +Requests with System or Options overlays bypass the cache because those overlays +mutate the effective show response. Cloud model entries are keyed by normalized cloud base model name and verbose. They use stale-while-revalidate behavior: a warm read returns the cached @@ -42,10 +44,11 @@ in separate maps, so a local "qwen3.5" and an explicit "qwen3.5:cloud" cannot collide. The cloud suffix is request routing intent; api.ShowResponse does not carry a model-name field to reconstruct on the way out. -The cache is process-local. Startup hydration runs asynchronously from the -current local manifests and cloud tags; no show responses are written to or read -from ~/.ollama/cache/show. That keeps cache lifetime tied to the server process -and avoids snapshot freshness and invalidation cases for this iteration. +The cache is process-local. Cloud startup hydration runs asynchronously from +cloud tags, while local show responses are populated on demand. No show +responses are written to or read from ~/.ollama/cache/show. That keeps cache +lifetime tied to the server process and avoids snapshot freshness and +invalidation cases for this iteration. */ const ( @@ -117,9 +120,9 @@ func modelShowCacheable(req api.ShowRequest) bool { return req.System == "" && len(req.Options) == 0 } -// Start kicks off non-blocking startup hydration. The cache remains -// process-local; warm entries appear as the background local and cloud scans -// populate the maps. +// Start kicks off non-blocking startup hydration for cloud entries. Local show +// responses stay lazy because even non-verbose show must load GGUF metadata that +// is expensive for large model stores. func (c *modelShowCache) Start(ctx context.Context) { c.once.Do(func() { slog.Debug("starting model show cache") @@ -127,34 +130,18 @@ func (c *modelShowCache) Start(ctx context.Context) { }) } -// runStartup hydrates local and cloud caches concurrently. It is only called in -// a goroutine from Start, so manifest scans and cloud requests cannot delay the -// listener from accepting traffic. +// runStartup hydrates the cloud cache. It is only called in a goroutine from +// Start, so cloud requests cannot delay the listener from accepting traffic. func (c *modelShowCache) runStartup(ctx context.Context) { - var wg sync.WaitGroup - wg.Add(2) - - go func() { - defer wg.Done() - if err := c.hydrateLocal(ctx); err != nil && !errors.Is(err, context.Canceled) { - slog.Warn("model show local cache hydration failed", "error", err) + if err := c.hydrateCloud(ctx); err != nil { + switch { + case errors.Is(err, context.Canceled): + case errors.Is(err, errModelShowNoCloud): + slog.Debug("skipping model show cloud cache hydration because cloud is disabled") + default: + slog.Warn("model show cloud cache hydration failed", "error", err) } - }() - - go func() { - defer wg.Done() - if err := c.hydrateCloud(ctx); err != nil { - switch { - case errors.Is(err, context.Canceled): - case errors.Is(err, errModelShowNoCloud): - slog.Debug("skipping model show cloud cache hydration because cloud is disabled") - default: - slog.Warn("model show cloud cache hydration failed", "error", err) - } - } - }() - - wg.Wait() + } } // GetLocal returns a cached local show response when the current manifest diff --git a/server/model_show_cache_test.go b/server/model_show_cache_test.go index 8b8175c66..04bab53ba 100644 --- a/server/model_show_cache_test.go +++ b/server/model_show_cache_test.go @@ -148,6 +148,25 @@ func TestModelShowCacheLocalHydrationSkipsUnchangedInMemory(t *testing.T) { } } +func TestModelShowCacheStartupSkipsLocalHydration(t *testing.T) { + gin.SetMode(gin.TestMode) + setTestHome(t, t.TempDir()) + t.Setenv("OLLAMA_NO_CLOUD", "1") + createShowCacheModel(t, "show-cache-startup", map[string]any{"test.context_length": uint32(1024)}) + + cache := newModelShowCache() + cache.getModelInfo = func(req api.ShowRequest) (*api.ShowResponse, error) { + t.Fatalf("startup should not hydrate local show cache, got request: %+v", req) + return nil, nil + } + + cache.runStartup(context.Background()) + + if len(cache.local) != 0 { + t.Fatalf("local cache entries = %d, want 0", len(cache.local)) + } +} + func TestModelShowCacheBypassesSystemAndOptionsOverlays(t *testing.T) { gin.SetMode(gin.TestMode) setTestHome(t, t.TempDir()) diff --git a/server/routes.go b/server/routes.go index 042ac4337..1acd61619 100644 --- a/server/routes.go +++ b/server/routes.go @@ -969,7 +969,10 @@ func (s *Server) PullHandler(c *gin.Context) { if err := PullModel(ctx, name.DisplayShortest(), regOpts, fn); err != nil { ch <- gin.H{"error": err.Error()} + return } + + s.refreshModelListCache(name) }() if req.Stream != nil && !*req.Stream { @@ -1108,6 +1111,8 @@ func (s *Server) DeleteHandler(c *gin.Context) { return } + s.deleteModelListCache(n) + if err := m.RemoveLayers(); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return @@ -1431,54 +1436,17 @@ func getModelData(digest string, verbose bool) (ggml.KV, ggml.Tensors, error) { } func (s *Server) ListHandler(c *gin.Context) { - ms, err := manifest.Manifests(true) + if s.modelCaches == nil || s.modelCaches.modelList == nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "model list cache unavailable"}) + return + } + + models, err := s.modelCaches.modelList.List(c.Request.Context()) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } - models := []api.ListModelResponse{} - for n, m := range ms { - var cf model.ConfigV2 - - if m.Config.Digest != "" { - f, err := m.Config.Open() - if err != nil { - slog.Warn("bad manifest filepath", "name", n, "error", err) - continue - } - defer f.Close() - - if err := json.NewDecoder(f).Decode(&cf); err != nil { - slog.Warn("bad manifest config", "name", n, "error", err) - continue - } - } - - // tag should never be masked - models = append(models, api.ListModelResponse{ - Model: n.DisplayShortest(), - Name: n.DisplayShortest(), - RemoteModel: cf.RemoteModel, - RemoteHost: cf.RemoteHost, - Size: m.Size(), - Digest: m.Digest(), - ModifiedAt: m.FileInfo().ModTime(), - Details: api.ModelDetails{ - Format: cf.ModelFormat, - Family: cf.ModelFamily, - Families: cf.ModelFamilies, - ParameterSize: cf.ModelType, - QuantizationLevel: cf.FileType, - }, - }) - } - - slices.SortStableFunc(models, func(i, j api.ListModelResponse) int { - // most recently modified first - return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix()) - }) - c.JSON(http.StatusOK, api.ListResponse{Models: models}) } @@ -1518,6 +1486,8 @@ func (s *Server) CopyHandler(c *gin.Context) { c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found", r.Source)}) } else if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + } else { + s.refreshModelListCache(dst) } } diff --git a/server/routes_list_test.go b/server/routes_list_test.go index f6e899ad7..e18f27e61 100644 --- a/server/routes_list_test.go +++ b/server/routes_list_test.go @@ -1,6 +1,7 @@ package server import ( + "context" "encoding/json" "net/http" "slices" @@ -28,7 +29,12 @@ func TestList(t *testing.T) { "myhost/mynamespace/lips:code", } - var s Server + s := Server{modelCaches: &modelCaches{modelList: newModelListCache()}} + s.modelCaches.modelList.Start(context.Background()) + if err := s.modelCaches.modelList.Wait(context.Background()); err != nil { + t.Fatal(err) + } + for _, n := range expectNames { _, digest := createBinFile(t, nil, nil) @@ -63,4 +69,10 @@ func TestList(t *testing.T) { if !slices.Equal(actualNames, expectNames) { t.Fatalf("expected slices to be equal %v", actualNames) } + + for _, m := range resp.Models { + if !slices.Contains(m.Capabilities, "completion") { + t.Fatalf("capabilities for %q = %v, want completion", m.Name, m.Capabilities) + } + } } diff --git a/server/routes_test.go b/server/routes_test.go index 8ffe8a510..036e61b3d 100644 --- a/server/routes_test.go +++ b/server/routes_test.go @@ -93,6 +93,9 @@ func (t *panicTransport) RoundTrip(r *http.Request) (*http.Response, error) { var panicOnRoundTrip = &http.Client{Transport: &panicTransport{}} func TestRoutes(t *testing.T) { + modelsDir := t.TempDir() + t.Setenv("OLLAMA_MODELS", modelsDir) + type testCase struct { Name string Method string @@ -101,6 +104,12 @@ func TestRoutes(t *testing.T) { Expected func(t *testing.T, resp *http.Response) } + s := &Server{modelCaches: &modelCaches{modelList: newModelListCache()}} + s.modelCaches.modelList.Start(context.Background()) + if err := s.modelCaches.modelList.Wait(context.Background()); err != nil { + t.Fatal(err) + } + createTestModel := func(t *testing.T, name string) { t.Helper() @@ -138,6 +147,7 @@ func TestRoutes(t *testing.T) { if err := createModel(r, modelName, baseLayers, config, fn); err != nil { t.Fatal(err) } + s.refreshModelListCache(modelName) } testCases := []testCase{ @@ -496,9 +506,6 @@ func TestRoutes(t *testing.T) { }, } - modelsDir := t.TempDir() - t.Setenv("OLLAMA_MODELS", modelsDir) - rc := &ollama.Registry{ // This is a temporary measure to allow us to move forward, // surfacing any code contacting ollama.com we do not intended @@ -514,7 +521,6 @@ func TestRoutes(t *testing.T) { HTTPClient: panicOnRoundTrip, } - s := &Server{} router, err := s.GenerateRoutes(rc) if err != nil { t.Fatalf("failed to generate routes: %v", err) diff --git a/server/sched_test.go b/server/sched_test.go index f40dc117f..450372ec6 100644 --- a/server/sched_test.go +++ b/server/sched_test.go @@ -536,7 +536,7 @@ func TestSchedGetRunnerReusesSameDigestWhenModelPathEmpty(t *testing.T) { } func TestSchedExpireRunner(t *testing.T) { - ctx, done := context.WithTimeout(t.Context(), 20*time.Millisecond) + ctx, done := context.WithCancel(t.Context()) defer done() s := InitScheduler(ctx) s.waitForRecovery = 10 * time.Millisecond @@ -556,8 +556,11 @@ func TestSchedExpireRunner(t *testing.T) { {Name: "output.weight", Kind: uint32(0), Offset: uint64(0), Shape: []uint64{1, 1, 1, 1}, WriterTo: bytes.NewReader(make([]byte, 32))}, }) + reqCtx, cancelReq := context.WithCancel(ctx) + defer cancelReq() + req := &LlmRequest{ - ctx: ctx, + ctx: reqCtx, model: &Model{ModelPath: modelPath}, opts: api.DefaultOptions(), successCh: make(chan *runnerRef, 1), @@ -587,16 +590,33 @@ func TestSchedExpireRunner(t *testing.T) { s.loadedMu.Unlock() } - s.expireRunner(&Model{ModelPath: modelPath}) + completedDone := make(chan struct{}) + go func() { + defer close(completedDone) + s.processCompleted(ctx) + }() - s.finishedReqCh <- req - s.processCompleted(ctx) + s.expireRunner(&Model{ModelPath: modelPath}) + cancelReq() + + select { + case <-s.unloadedCh: + case <-time.After(5 * time.Second): + t.Fatal("expected model to be unloaded") + } s.loadedMu.Lock() if len(s.loaded) != 0 { t.Fatalf("expected model to be unloaded") } s.loadedMu.Unlock() + + done() + select { + case <-completedDone: + case <-time.After(5 * time.Second): + t.Fatal("expected completed loop to stop") + } } // TODO - add one scenario that triggers the bogus finished event with positive ref count