diff --git a/tools/ui/src/lib/components/app/models/ModelAvatar.svelte b/tools/ui/src/lib/components/app/models/ModelAvatar.svelte index 6838f43070..dd0ba778ab 100644 --- a/tools/ui/src/lib/components/app/models/ModelAvatar.svelte +++ b/tools/ui/src/lib/components/app/models/ModelAvatar.svelte @@ -1,9 +1,13 @@ -{#if orgName} +{#if useProviderIcon} + + + {#snippet fallback()} + {#if isLocal} + + {:else} + + {/if} + {/snippet} + + +{:else if orgName} import { CircleAlert, Eject, Loader2, Power, RotateCw } from '@lucide/svelte'; import { ActionIcon } from '$lib/components/app'; + import { BackendIcon } from '$lib/components/app/backends'; import { ICON_CLASS_DEFAULT } from '$lib/constants'; - import { modelsStore } from '$lib/stores'; + import { backendsModelsStore, modelsStore } from '$lib/stores'; import type { ModelOption } from '$lib/types/models'; + import { getBackend } from '$lib/utils/api-base'; interface Props { + /** Backend can load and unload models, llama-compat servers only. */ + canLoad: boolean; class?: string; isFailed?: boolean; isLoaded: boolean; @@ -16,9 +20,14 @@ revealOnHover?: boolean; /** Renders the state alone, for a caller that moves load and unload elsewhere. */ showAction?: boolean; + /** Non-loadable rows show the provider mark, which identifies them in a flat list. */ + showBackendMark?: boolean; + /** Table rows mark a remote provider, which this UI cannot load or unload. */ + showRemoteMark?: boolean; } let { + canLoad, class: className = '', isFailed = false, isLoaded, @@ -26,12 +35,35 @@ isSleeping = false, option, revealOnHover = true, - showAction = true + showAction = true, + showBackendMark = false, + showRemoteMark = false }: Props = $props(); + + let backendName = $derived(getBackend(option.backendId)?.name ?? 'Remote provider'); + /** The provider's listing failed, so nothing it serves is selectable right now. */ + let isBackendFailed = $derived( + Boolean(option.backendId && backendsModelsStore.get(option.backendId).error) + );
- {#if isLoading} + {#if !canLoad} + {#if showBackendMark} + + {:else if showRemoteMark} + + + + {#if isBackendFailed} + + {/if} + + {/if} + {:else if isLoading} {:else} diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte index ed66116998..845f723f08 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManager.svelte @@ -2,18 +2,25 @@ import ModelsManagerModelsTable from './ModelsManagerModelsTable.svelte'; import { groupModelQuants, + loadOverrides, type ModalityKey, modelContextLength, + modelDraftsFor, + type ModelOverride, type ModelQuantGroup, type ModelsTableGroup, - modelSupports + modelSupports, + saveOverrides } from './utils'; import { LOCAL_BACKEND_ID } from '$lib/constants'; import { ModelCapability } from '$lib/enums'; - import { modelsStore, uiStore } from '$lib/stores'; + import { backendsStore, modelsStore, uiStore } from '$lib/stores'; import type { ModelOption } from '$lib/types/models'; + import { getBackend } from '$lib/utils/api-base'; + import { getBackendCapabilities } from '$lib/utils/backend'; import { type Snippet } from 'svelte'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; + import { toast } from 'svelte-sonner'; interface Props { class?: string; @@ -24,15 +31,20 @@ let { class: className, toolbarEnd }: Props = $props(); let filter = $state(''); + let providerFilter = $state([]); let contextLimit = $state(0); let modalityFilter = $state([]); let capabilityFilter = $state([]); + let draftFilter = $state(false); let selectedId = $state(null); + let overrides = $state>(loadOverrides()); let allModels = $derived(modelsStore.models); let isFavorite = $derived((option: ModelOption) => modelsStore.favoriteModelIds.has(option.model) ); + // every filter but the provider one, so a provider count does not fall to zero + // the moment that provider is the one being looked at let matching = $derived.by(() => { const term = filter.trim().toLowerCase(); @@ -48,6 +60,13 @@ return false; } + if ( + draftFilter && + modelDraftsFor(option, overrides[option.id]?.load?.speculativeDecoding).length === 0 + ) { + return false; + } + // a model whose modalities are unknown cannot be shown to match if (modalityFilter.length > 0 && !modalityFilter.some((key) => option.modalities?.[key])) { return false; @@ -56,6 +75,24 @@ return contextLimit === 0 || modelContextLength(option) >= contextLimit; }); }); + let visible = $derived.by(() => + providerFilter.length === 0 + ? matching + : matching.filter((option) => providerFilter.includes(option.backendId ?? LOCAL_BACKEND_ID)) + ); + // the rail counts follow the active view and filter, so it always says how many + // repos each provider contributes to what the table is showing + let providerCounts = $derived.by(() => { + const counts: Record = {}; + + for (const entry of groupModelQuants(matching)) { + const backendId = entry.base.backendId ?? LOCAL_BACKEND_ID; + + counts[backendId] = (counts[backendId] ?? 0) + 1; + } + + return counts; + }); // recently used models lead their section, the rest keep the server's order const rank = new SvelteMap(); @@ -68,12 +105,16 @@ rank.size === 0 ? list : [...list].sort((a, b) => rankOf(a) - rankOf(b)); // one entry per repo, so a model with several quants is a single table row; - // loaded models lead the table, then favorites, then the local block - let entries = $derived(byRecency(groupModelQuants(matching))); + // loaded models lead the table, then favorites, then one block per provider, + // and an entry lands in the first group that claims it + let entries = $derived(byRecency(groupModelQuants(visible))); let groups = $derived.by(() => { // A loaded quant is a model of its own: it moves to the loaded section, and - // the quants of its repo that are not loaded stay behind as that repo. - const isLoaded = (option: ModelOption) => modelsStore.isModelLoaded(option.model); + // the quants of its repo that are not loaded stay behind as that repo. Only + // llama-compat servers report a load state. + const isLoaded = (option: ModelOption) => + getBackendCapabilities(getBackend(option.backendId)).loadUnload && + modelsStore.isModelLoaded(option.model); const loaded: ModelQuantGroup[] = []; const rest: ModelQuantGroup[] = []; @@ -100,7 +141,18 @@ (entry) => !claimed.has(entry.key) && entry.quants.some((q) => modelsStore.isHidden(q.id)) ); const hiddenKeys = new SvelteSet(hidden.map((entry) => entry.key)); - const local = rest.filter((entry) => !claimed.has(entry.key) && !hiddenKeys.has(entry.key)); + const byBackend = new SvelteMap(); + + for (const entry of rest) { + if (claimed.has(entry.key) || hiddenKeys.has(entry.key)) continue; + + const backendId = entry.base.backendId ?? LOCAL_BACKEND_ID; + + if (!byBackend.has(backendId)) byBackend.set(backendId, []); + + byBackend.get(backendId)!.push(entry); + } + const ordered: ModelsTableGroup[] = []; if (loaded.length) { @@ -125,17 +177,36 @@ }); } - if (local.length) { + const localItems = byBackend.get(LOCAL_BACKEND_ID); + + if (localItems?.length) { ordered.push({ backendId: LOCAL_BACKEND_ID, isLocal: true, - items: local, + items: localItems, key: LOCAL_BACKEND_ID, kind: 'local', label: 'Local models' }); } + for (const backend of backendsStore.enabled) { + if (backend.id === LOCAL_BACKEND_ID) continue; + + const items = byBackend.get(backend.id); + + if (items?.length) { + ordered.push({ + backendId: backend.id, + isLocal: false, + items, + key: backend.id, + kind: 'provider', + label: backend.name + }); + } + } + if (hidden.length) { ordered.push({ backendId: null, @@ -162,6 +233,24 @@ uiStore.manageModelFocus = null; }); + + /** Point the selected model's load settings at another model as its draft. */ + function useAsDraft(draft: ModelOption, targetId: string): void { + const target = modelsStore.models.find((option) => option.id === targetId); + + if (!target) return; + + saveOverride(target, { + ...overrides[target.id], + load: { ...overrides[target.id]?.load, speculativeDecoding: draft.id } + }); + } + + function saveOverride(option: ModelOption, override: ModelOverride): void { + overrides = { ...overrides, [option.id]: override }; + saveOverrides(overrides); + toast.success(`Saved settings for ${option.name}`); + }
@@ -169,11 +258,16 @@ (selectedId = option.id)} + onUseAsDraft={useAsDraft} + {overrides} + {providerCounts} {selectedId} {toolbarEnd} /> diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte index 2de4ebc693..bd8c1a6d55 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerFilters.svelte @@ -1,25 +1,40 @@ +{#snippet providerMark(backend: Backend)} + {#if backend.id === LOCAL_BACKEND_ID} + + {#snippet fallback()} + + {/snippet} + + {:else} + + {/if} +{/snippet} + + {#if backends.length > 1} + + + {#snippet child({ props })} + + {/snippet} + + + + + Providers + + {#each backends as backend (backend.id)} + toggleProvider(backend.id, checked)} + > + {@render providerMark(backend)} + + {backend.name} + + {providerCounts[backend.id] ?? 0} + + {/each} + + + {#if providers.length > 0} + + + (providers = [])} + >Show every provider + {/if} + + + {/if} + (contextLimit = Number(value))} type="single" @@ -82,6 +168,26 @@ + + + + + Has draft sidecar + + import ModelsManagerFilters from './ModelsManagerFilters.svelte'; import { + isLocalOption, type ModalityKey, modelContextLength, + type ModelDraft, + modelDraftsFor, + type ModelOverride, type ModelQuantGroup, type ModelsTableGroup } from './utils'; @@ -19,7 +23,8 @@ MoreHorizontal, Power, Trash2, - X + X, + Zap } from '@lucide/svelte'; import { DropdownMenuActions, @@ -40,8 +45,16 @@ import { Button } from '$lib/components/ui/button'; import { FAMILY_ROW_WINDOW, MODEL_ICON, MODEL_ROW_WINDOW } from '$lib/constants'; import { ModelCapability, ModelDownloadConfirmAction, ServerModelStatus } from '$lib/enums'; - import { modelsStore, settingsStore, uiStore } from '$lib/stores'; + import { + backendsModelsStore, + backendsStore, + modelsStore, + settingsStore, + uiStore + } from '$lib/stores'; import type { ModelOption } from '$lib/types/models'; + import { getBackend } from '$lib/utils/api-base'; + import { getBackendCapabilities } from '$lib/utils/backend'; import { groupModelFamilies, type ModelFamilyGroup } from '$lib/utils/model-families'; import type { Snippet } from 'svelte'; import { SvelteSet } from 'svelte/reactivity'; @@ -51,12 +64,22 @@ capabilities?: ModelCapability[]; /** Smallest context a model must support; 0 keeps every model. */ contextLimit?: number; + /** Keep only models that have a draft sidecar to speculate with. */ + draft?: boolean; filter?: string; groups: ModelsTableGroup[]; isFavorite: (option: ModelOption) => boolean; onSelect: (option: ModelOption) => void; + /** Per-model load and inference overrides, keyed by backend-qualified id. */ /** Modalities a model must support at least one of. */ modalities?: ModalityKey[]; + overrides: Record; + /** Backend ids to keep; empty keeps every provider. */ + providers?: string[]; + /** Repos each provider contributes to the current search, for the filter menu. */ + providerCounts?: Record; + /** Called when a row is set as the draft of the selected model. */ + onUseAsDraft?: (draft: ModelOption, targetId: string) => void; selectedId: string | null; /** Rendered at the toolbar's right end, past the filters. */ toolbarEnd?: Snippet; @@ -65,17 +88,28 @@ let { capabilities = $bindable([]), contextLimit = $bindable(0), + draft = $bindable(false), filter = $bindable(''), groups, isFavorite, modalities = $bindable([]), onSelect, + onUseAsDraft, + overrides, + providerCounts = {}, + providers = $bindable([]), selectedId, toolbarEnd }: Props = $props(); let isEmpty = $derived(groups.every((group) => group.items.length === 0)); - let hasFilters = $derived(contextLimit > 0 || modalities.length > 0 || capabilities.length > 0); + let hasFilters = $derived( + providers.length > 0 || + contextLimit > 0 || + modalities.length > 0 || + capabilities.length > 0 || + draft + ); let filterInput = $state(null); // the dialog hands focus to its first control, so the filter takes it instead @@ -153,20 +187,38 @@ } /** Row actions follow the app's dropdown pattern: icon, label, separators, variants. */ - function rowActions(option: ModelOption, favorite: boolean, isHidden: boolean) { + function rowActions(option: ModelOption, canLoad: boolean, favorite: boolean, isHidden: boolean) { + // a draft only makes sense for the model the configuration pane has open, and + // only when both ends can load and unload at all + const canBeDraft = canLoad && selectedId !== null && selectedId !== option.id && selectedName; + return [ + ...(canBeDraft + ? [ + { + icon: Zap, + label: `Use as draft for ${selectedName}`, + onclick: () => onUseAsDraft?.(option, selectedId as string), + separator: true + } + ] + : []), { icon: favorite ? HeartOff : Heart, label: favorite ? 'Remove from favorites' : 'Add to favorites', onclick: () => modelsStore.toggleFavorite(option.model) }, - { - icon: Trash2, - label: 'Delete from disk', - onclick: () => requestDelete(option), - separator: true, - variant: 'destructive' as const - }, + ...(canLoad + ? [ + { + icon: Trash2, + label: 'Delete from disk', + onclick: () => requestDelete(option), + separator: true, + variant: 'destructive' as const + } + ] + : []), { icon: isHidden ? Eye : EyeOff, label: isHidden ? 'Unhide model' : 'Hide model', @@ -181,9 +233,25 @@ return modelsStore.getModelStatus(option.model); } - /** Context the model runs with: what a loaded model reports. */ + /** Drafts to show on a row: the configured one first, then this repo's own sidecars. */ + function draftsFor(option: ModelOption): ModelDraft[] { + return modelDraftsFor(option, overrides[option.id]?.load?.speculativeDecoding); + } + + /** Name of the model the configuration pane has open, for the draft action label. */ + let selectedName = $derived( + modelsStore.models.find((option) => option.id === selectedId)?.name ?? null + ); + + /** Context the model runs with: the stored override, else what a loaded model reports. */ function configuredContext(option: ModelOption): number | null { - return isLoadedOption(option) ? modelsStore.props.getModelContextSize(option.model) : null; + const override = overrides[option.id]?.load?.contextLength; + + if (override) return override; + + return isLocalOption(option) && isLoadedOption(option) + ? modelsStore.props.getModelContextSize(option.model) + : null; } type SortKey = 'context' | 'name' | 'status'; @@ -260,12 +328,14 @@ {@const isSleeping = status === ServerModelStatus.SLEEPING} {/snippet} @@ -290,6 +360,7 @@ {#snippet row(option: ModelOption, indent = 0)} {@const favorite = isFavorite(option)} + {@const canLoad = getBackendCapabilities(getBackend(option.backendId)).loadUnload} {@const isHidden = modelsStore.isHidden(option.id)} @@ -317,6 +388,7 @@ option.backendId ?? '')).size} {@const groupLabel = - entry.kind === 'variants' - ? `${entry.quants.length} variants` - : `${entry.quants.length} quants available`} + entry.kind === 'providers' + ? `${providerCount} provider${providerCount === 1 ? '' : 's'}` + : entry.kind === 'variants' + ? `${entry.quants.length} variants` + : `${entry.quants.length} quants available`} {@const anyLoaded = entry.quants.some(isLoadedOption)} @@ -377,6 +452,7 @@ {/snippet} -{#snippet quantRow(option: ModelOption, indent = 0)} +{#snippet quantRow(option: ModelOption, indent = 0, showProvider = false)} {@const favorite = isFavorite(option)} + {@const canLoad = getBackendCapabilities(getBackend(option.backendId)).loadUnload} {@const isHidden = modelsStore.isHidden(option.id)} {@const quant = option.parsedId?.quantization ?? option.model} @@ -435,7 +512,9 @@ tabindex="0" > - {quant} + + {showProvider ? (getBackend(option.backendId)?.name ?? quant) : quant} + {option.model} @@ -446,7 +525,7 @@
- + {#if hasFilters}