ui : put the draft sidecar on the model id and its details in the pane

Assisted-by: pi:llama.cpp/DeepSeek-V4.1-Flash
This commit is contained in:
Aleksander Grygier
2026-09-28 12:10:13 +02:00
parent 2d59a0d668
commit f2432ce485
11 changed files with 190 additions and 127 deletions
@@ -1,41 +0,0 @@
<script lang="ts">
import ModelAvatar from './ModelAvatar.svelte';
import type { ModelDraft } from './ModelsManager/utils';
import { Zap } from '@lucide/svelte';
import { Badge } from '$lib/components/ui/badge';
import type { ModelOption } from '$lib/types/models';
import { rawModelId } from '$lib/utils/model-option-id';
interface Props {
/** The draft model's option, when the manager lists it, so its avatar can be shown. */
option?: ModelOption | null;
draft: ModelDraft;
}
let { draft, option = null }: Props = $props();
</script>
{#if draft.active}
<Badge
class="h-5 max-w-40 gap-1 px-1.5 text-[10px]"
title="Speculative draft in use"
variant="secondary"
>
<Zap class="h-3 w-3 shrink-0" />
{#if option}
<ModelAvatar {option} showRepoOrgAvatar size="size-3.5" />
{/if}
<span class="truncate">{rawModelId(option?.model ?? draft.model ?? draft.kind ?? 'draft')}</span
>
</Badge>
{:else}
<Badge
class="h-5 gap-1 px-1.5 text-[10px] text-muted-foreground/70"
title="Draft sidecar in this repo, not in use"
variant="outline"
>
{draft.kind}
</Badge>
{/if}
@@ -1,9 +1,11 @@
<script lang="ts">
import ModelCapabilityIcons from './ModelCapabilityIcons.svelte';
import type { ModelDraft } from './ModelsManager/utils';
import { Database, ScrollText } from '@lucide/svelte';
import { TruncatedText } from '$lib/components/app';
import * as Tooltip from '$lib/components/ui/tooltip';
import { type ModelSidecar } from '$lib/constants';
import { MODEL_BADGE_CLASS, MODEL_VARIANT_BADGE_CLASS } from '$lib/constants';
import { HuggingFaceService } from '$lib/services';
import { ModelsService } from '$lib/services/models.service';
import { settingsStore } from '$lib/stores';
@@ -36,6 +38,8 @@
/** Min/max GGUF file size (main + draft) across quants; renders a range when set. */
sizeRange?: { min: number; max: number } | null;
draftSidecars?: ModelSidecar[];
/** Drafts a load would use, rendered as `+ [KIND] [QUANT]` next to the id. */
drafts?: ModelDraft[];
/** Allow badges to wrap onto new lines instead of truncating. */
wrap?: boolean;
class?: string;
@@ -45,6 +49,7 @@
aliases,
class: className = '',
contextLength,
drafts = [],
draftSidecars = [],
hideCapabilities = false,
hideModalities = false,
@@ -67,12 +72,10 @@
...rest
}: Props = $props();
const badgeClass =
'inline-flex w-fit shrink-0 items-center justify-center whitespace-nowrap rounded-md border border-border/50 px-1 py-0 text-[10px] font-mono bg-foreground/15 dark:bg-foreground/10 text-foreground [a&]:hover:bg-foreground/25';
const badgeClass = MODEL_BADGE_CLASS;
const tagBadgeClass =
'inline-flex w-fit shrink-0 items-center justify-center whitespace-nowrap rounded-md border border-border/50 px-1 py-0 text-[10px] font-mono text-foreground [a&]:hover:bg-accent [a&]:hover:text-accent-foreground';
const variantBadgeClass =
'inline-flex w-fit shrink-0 items-center justify-center whitespace-nowrap rounded-md bg-primary px-1.5 py-0 text-[10px] font-mono font-semibold uppercase tracking-wide text-primary-foreground';
const variantBadgeClass = MODEL_VARIANT_BADGE_CLASS;
/** Alias badges beyond this many collapse into a single `+x more` badge. */
const MAX_ALIAS_BADGES = 2;
@@ -89,6 +92,7 @@
let uniqueAliases = $derived([...new Set(aliases ?? [])]);
let uniqueTags = $derived([...new Set([...(parsed.tags ?? []), ...(tags ?? [])])]);
let uniqueDraftSidecars = $derived([...new Set(draftSidecars)].filter((s) => !isAuxSidecar(s)));
let activeDrafts = $derived(drafts.filter((draft) => draft.active && draft.kind));
let primaryAlias = $derived(uniqueAliases.length === 1 ? uniqueAliases[0] : null);
let displayName = $derived(primaryAlias ?? parsed.modelName ?? modelId);
@@ -131,6 +135,18 @@
</span>
{/if}
{#each activeDrafts as draft (draft.kind ?? 'draft')}
<span class="flex shrink-0 items-center gap-1">
<span class="text-[10px] text-muted-foreground">+</span>
<span class={variantBadgeClass} title="Speculative draft in use">{draft.kind}</span>
{#if draft.quant}
<span class={badgeClass}>{draft.quant}</span>
{/if}
</span>
{/each}
{#each uniqueDraftSidecars as sidecar (sidecar)}
<span class={variantBadgeClass} title={`${sidecar.toUpperCase()} draft model available`}>
{sidecar}
@@ -108,7 +108,12 @@
<div class="min-h-0 flex-1 overflow-y-auto px-4 py-4">
<Tabs.Content value="information">
<ModelsManagerModelConfigurationInformation hub={hubDetails} {option} {serverProps} />
<ModelsManagerModelConfigurationInformation
draftSetting={override?.load?.speculativeDecoding ?? null}
hub={hubDetails}
{option}
{serverProps}
/>
</Tabs.Content>
<Tabs.Content value="load">
@@ -1,5 +1,11 @@
<script lang="ts">
import { isLocalOption, modelQuantLabel, modelSizeLabel, resolveModelSize } from '../utils';
import {
isLocalOption,
modelDraftsFor,
modelQuantLabel,
modelSizeLabel,
resolveModelSize
} from '../utils';
import {
ActionIconCopyToClipboard,
BadgesModality,
@@ -14,13 +20,15 @@
import { formatFileSize, formatNumber, formatParameters } from '$lib/utils/formatters';
interface Props {
/** Draft the load settings name, when one is set. */
draftSetting?: string | null;
/** Hugging Face details, filled in when discovery is on and the server has none. */
hub?: HfModelDetailInfo | null;
option: ModelOption;
serverProps: ApiLlamaCppServerProps | null | undefined;
}
let { hub = null, option, serverProps }: Props = $props();
let { draftSetting = null, hub = null, option, serverProps }: Props = $props();
let quant = $derived(modelQuantLabel(option));
let size = $derived(modelSizeLabel(option));
@@ -55,6 +63,10 @@
? `${formatNumber(hub.gguf.context_length)} tokens`
: null
);
// the draft a load would speculate with, and any other sidecar on disk
let drafts = $derived(modelDraftsFor(option, draftSetting));
let activeDraft = $derived(drafts.find((draft) => draft.active) ?? null);
let idleDrafts = $derived(drafts.filter((draft) => !draft.active));
let meta = $derived(option.meta);
// the server reports these once the model is loaded; the Hub knows them anyway
let gguf = $derived(hub?.gguf ?? null);
@@ -93,6 +105,22 @@
value: (option.meta?.architecture as string) ?? gguf?.architecture ?? null
},
{ label: 'Format', value: isLocalOption(option) ? 'GGUF' : null },
...(activeDraft
? [
{ isBadge: true, label: 'Draft sidecar', value: activeDraft.kind },
{
isCopyable: true,
label: 'Draft model',
value: activeDraft.model ?? option.model.split(':')[0]
},
{ isBadge: true, label: 'Draft quant', value: activeDraft.quant }
]
: []),
{
isBadge: true,
label: 'Other sidecars',
value: idleDrafts.length > 0 ? idleDrafts.map((draft) => draft.kind).join(', ') : null
},
{ label: 'Parallel Slots', value: serverProps ? String(serverProps.total_slots) : null },
{ isMono: true, label: 'Build Info', value: serverProps?.build_info ?? null }
] satisfies Array<{
@@ -1,8 +1,8 @@
<script lang="ts">
import ModelDraftChip from '../ModelDraftChip.svelte';
import ModelsManagerFilters from './ModelsManagerFilters.svelte';
import {
draftFromArgs,
draftFromSetting,
isLocalOption,
type ModalityKey,
modelContextLength,
@@ -10,7 +10,8 @@
modelDrafts,
type ModelOverride,
type ModelQuantGroup,
type ModelsTableGroup
type ModelsTableGroup,
sidecarFilesFor
} from './utils';
import {
ArrowDown,
@@ -46,20 +47,13 @@
import ModelsSelectorDownloadItem from '$lib/components/app/models/ModelsSelector/ModelsSelectorDownloadItem.svelte';
import { Badge } from '$lib/components/ui/badge';
import { Button } from '$lib/components/ui/button';
import { FAMILY_ROW_WINDOW, LOCAL_BACKEND_ID, MODEL_ROW_WINDOW } from '$lib/constants';
import { FAMILY_ROW_WINDOW, MODEL_ROW_WINDOW } from '$lib/constants';
import { ModelDownloadConfirmAction, ServerModelStatus } from '$lib/enums';
import {
backendsModelsStore,
backendsStore,
modelsStore,
settingsStore,
uiStore
} from '$lib/stores';
import { backendsStore, modelsStore, settingsStore, uiStore } from '$lib/stores';
import type { ModelOption } from '$lib/types/models';
import { getBackend } from '$lib/utils/api-base';
import { getBackendCapabilities } from '$lib/utils/backend';
import { groupModelFamilies, type ModelFamilyGroup } from '$lib/utils/model-families';
import { rawModelId } from '$lib/utils/model-option-id';
import { SvelteSet } from 'svelte/reactivity';
interface Props {
@@ -227,32 +221,13 @@
// speculative decoding is a llama.cpp feature
if (!getBackendCapabilities(getBackend(option.backendId)).loadUnload) return [];
const repo = option.model.split(':')[0] ?? '';
const state = backendsModelsStore.get(option.backendId ?? LOCAL_BACKEND_ID);
// the router reports the arguments a model loads with, draft included
const serverDraft = draftFromArgs(
modelsStore.routerModels.find((model) => model.id === option.model)?.status?.args
);
const args = modelsStore.routerModels.find((model) => model.id === option.model)?.status?.args;
const configured =
serverDraft?.model ?? overrides[option.id]?.load?.speculativeDecoding ?? null;
draftFromArgs(args, option) ??
draftFromSetting(option, overrides[option.id]?.load?.speculativeDecoding);
return modelDrafts(option, state.drafts?.[repo] ?? [], configured, serverDraft?.kind ?? null);
}
/** The draft model's own option, when the manager lists it, for its avatar. */
function draftOptionFor(draft: ModelDraft): ModelOption | null {
const id = draft.model;
if (!id) return null;
const raw = rawModelId(id);
// a draft may be named as a repo, in which case any of its quants will do
return (
modelsStore.models.find((option) => option.id === id) ??
modelsStore.models.find((option) => rawModelId(option.id).startsWith(`${raw}:`)) ??
null
);
return modelDrafts(option, sidecarFilesFor(option), configured);
}
/** Name of the model the configuration pane has open, for the draft action label. */
@@ -405,6 +380,7 @@
<ModelId
aliases={option.aliases}
class="min-w-0 flex-1"
drafts={draftsFor(option)}
hideCapabilities
modalities={option.modalities}
modelId={option.model}
@@ -413,10 +389,6 @@
/>
<ModelCapabilities {option} />
{#each draftsFor(option) as draft (draft.model ?? draft.kind)}
<ModelDraftChip {draft} option={draftOptionFor(draft)} />
{/each}
</span>
</span>
@@ -471,6 +443,7 @@
<ModelId
aliases={entry.base.aliases}
class="min-w-0"
drafts={draftsFor(entry.base)}
hideCapabilities
hideQuantization
modalities={mediaSource.modalities}
@@ -480,10 +453,6 @@
/>
<ModelCapabilities option={entry.base} />
{#each draftsFor(entry.base) as draft (draft.model ?? draft.kind)}
<ModelDraftChip {draft} option={draftOptionFor(draft)} />
{/each}
</span>
<span class="block text-xs text-muted-foreground">{groupLabel}</span>
@@ -7,8 +7,13 @@ import {
SPEC_TYPE
} from '$lib/constants';
import { HuggingFaceService, ModelsService } from '$lib/services';
import { settingsStore } from '$lib/stores';
import type { ModelLoadProgress, ModelModalities, ModelOption } from '$lib/types/models';
import { backendsModelsStore, modelsStore, settingsStore } from '$lib/stores';
import type {
ModelLoadProgress,
ModelModalities,
ModelOption,
ModelSidecarFile
} from '$lib/types/models';
import { getBackend } from '$lib/utils/api-base';
import { formatFileSize, formatParameters } from '$lib/utils/formatters';
import { rawModelId } from '$lib/utils/model-option-id';
@@ -121,13 +126,15 @@ export function saveOverrides(overrides: ModelOverrideMap): void {
}
/** Backend a model is served by, the local server reads as "This server". */
/** A draft a model can speculate with, and whether the load settings point at it. */
/** A draft a model can speculate with, and whether a load would use it. */
export interface ModelDraft {
/** The draft the settings name, which is the one a load would use. */
/** The draft a load would use, from the server's own arguments or from the settings. */
active: boolean;
kind: ModelSidecar | null;
/** Qualified id of a draft from another repo; null when the file sits in this repo. */
/** Repo the draft comes from; null when the file sits in the model's own repo. */
model: string | null;
params: string | null;
quant: string | null;
}
/** Repo an id belongs to: the id without its quant tag. */
@@ -166,19 +173,64 @@ function repoFromDraftPath(path: string): string {
* Draft the server's own launch arguments point at. The router reports the arguments a
* model loads with, so this is what a load would really speculate with.
*/
export function draftFromArgs(args: string[] | undefined): ModelDraft | null {
export function draftFromArgs(args: string[] | undefined, option: ModelOption): ModelDraft | null {
const flag = args?.indexOf('--model-draft') ?? -1;
const path = flag === -1 ? null : (args?.[flag + 1] ?? null);
if (!path) return null;
const parsed = ModelsService.parseModelId(path.split(/[/\\]/).pop() ?? path);
const repo = repoFromDraftPath(path);
return {
active: true,
kind: sidecarFromSpecType(args?.[(args?.indexOf('--spec-type') ?? -1) + 1]),
model: repoFromDraftPath(path)
kind: sidecarFromSpecType(args?.[(args?.indexOf('--spec-type') ?? -1) + 1]) ?? parsed.sidecar,
model: repo === repoOf(option.model) ? null : repo,
params: parsed.params
? `${parsed.params}${parsed.activatedParams ? `-${parsed.activatedParams}` : ''}`
: null,
quant: parsed.quantization
};
}
/** Draft the load settings name, resolved against the model's own repo. */
export function draftFromSetting(option: ModelOption, value?: string | null): ModelDraft | null {
const id = value?.trim();
if (!id || id === 'off') return null;
const parsed = ModelsService.parseModelId(id);
return {
active: true,
kind: parsed.sidecar,
model: repoOf(id) === repoOf(option.model) ? null : id,
params: parsed.params
? `${parsed.params}${parsed.activatedParams ? `-${parsed.activatedParams}` : ''}`
: null,
quant: parsed.quantization
};
}
/**
* Drafts of a model, in the order they matter: what the server loads with, else what the
* settings name, then any other sidecar the model's own repo ships.
*/
export function modelDraftsFor(option: ModelOption, settingValue?: string | null): ModelDraft[] {
const args = modelsStore.routerModels.find((model) => model.id === option.model)?.status?.args;
const configured = draftFromArgs(args, option) ?? draftFromSetting(option, settingValue);
return modelDrafts(option, sidecarFilesFor(option), configured);
}
/** Draft sidecars a listing reported for the model's repo. */
export function sidecarFilesFor(option: ModelOption): ModelSidecarFile[] {
const repo = option.model.split(':')[0] ?? '';
const state = backendsModelsStore.get(option.backendId ?? LOCAL_BACKEND_ID);
return state.drafts?.[repo] ?? [];
}
/**
* Drafts to show for a model: the one the load settings name, plus any draft sidecar
* the model's own repo ships. The configured one is the active draft; a sidecar that
@@ -186,32 +238,26 @@ export function draftFromArgs(args: string[] | undefined): ModelDraft | null {
*/
export function modelDrafts(
option: ModelOption,
available: ModelSidecar[] = [],
configured?: string | null,
configuredKind?: ModelSidecar | null
available: ModelSidecarFile[] = [],
configured?: ModelDraft | null
): ModelDraft[] {
const value = configured?.trim() ?? '';
const drafts: ModelDraft[] = [];
if (value && value !== 'off') {
const parsed = ModelsService.parseModelId(value);
const sameRepo = repoOf(value) === repoOf(option.model);
if (configured) drafts.push(configured);
for (const file of available) {
// a sidecar a load already points at is the active draft, not a second entry
if (drafts.some((draft) => draft.kind === file.kind)) continue;
drafts.push({
active: true,
kind: parsed.sidecar ?? configuredKind ?? null,
model: sameRepo ? null : value
active: false,
kind: file.kind,
model: null,
params: file.params,
quant: file.quant
});
}
for (const kind of available) {
const alreadyActive = drafts.some((draft) => draft.active && draft.kind === kind);
if (alreadyActive) continue;
drafts.push({ active: false, kind, model: null });
}
return drafts;
}
@@ -1,5 +1,6 @@
<script lang="ts">
import ModelLoadHighlight from '../ModelLoadHighlight.svelte';
import { modelDraftsFor } from '../ModelsManager/utils';
import { ModelAvatar, ModelId, ModelLoadControl, ModelRowActions } from '$lib/components/app';
import { ModelCapability, ServerModelStatus } from '$lib/enums';
import { modelsStore } from '$lib/stores';
@@ -94,6 +95,7 @@
<ModelId
aliases={option.aliases}
class="min-w-0 flex-1"
drafts={modelDraftsFor(option)}
{hideOrgName}
{modalities}
modelId={option.model}
@@ -29,6 +29,14 @@ export const CHAT_INPUT_FOCUS_SELECTOR =
*/
export const DROPDOWN_MENU_CONTENT_SEARCH_SELECTOR = '[data-slot="dropdown-menu-content"] input';
/** Neutral model badge: params, quantization, tags. */
export const MODEL_BADGE_CLASS =
'inline-flex w-fit shrink-0 items-center justify-center whitespace-nowrap rounded-md border border-border/50 px-1 py-0 text-[10px] font-mono bg-foreground/15 dark:bg-foreground/10 text-foreground [a&]:hover:bg-foreground/25';
/** Emphasis model badge: draft sidecars and other markers that must stand out. */
export const MODEL_VARIANT_BADGE_CLASS =
'inline-flex w-fit shrink-0 items-center justify-center whitespace-nowrap rounded-md bg-primary px-1.5 py-0 text-[10px] font-mono font-semibold uppercase tracking-wide text-primary-foreground';
/** Default Tailwind size class for inline icon components (lucide, etc.). */
export const ICON_CLASS_DEFAULT = 'h-4 w-4';
+24 -9
View File
@@ -14,7 +14,7 @@ import {
SIDECAR_TOKENS
} from '$lib/constants';
import { ServerModelStatus } from '$lib/enums';
import type { ParsedModelId } from '$lib/types/models';
import type { ModelSidecarFile, ParsedModelId } from '$lib/types/models';
import {
apiDelete,
apiFetch,
@@ -40,6 +40,13 @@ function sidecarTokenInFilename(modelId: string): ModelSidecar | null {
return (match as ModelSidecar | undefined) ?? null;
}
/** Parameter label a parsed id reports, e.g. `35B-A3B`. */
function paramsLabel(parsed: ParsedModelId): string | null {
if (!parsed.params) return null;
return `${parsed.params}${parsed.activatedParams ? `-${parsed.activatedParams}` : ''}`;
}
export class ModelsService {
private static readonly SSE_RECONNECT_MS = 1000;
@@ -117,27 +124,35 @@ export class ModelsService {
* belong to. The router lists a downloaded sidecar as a model of its own, so this
* keeps the pairing that the model list itself is filtered to drop.
*/
static draftSidecarsByRepo(response: ApiModelsListResponse): Record<string, ModelSidecar[]> {
const byRepo: Record<string, ModelSidecar[]> = {};
static draftSidecarsByRepo(response: ApiModelsListResponse): Record<string, ModelSidecarFile[]> {
const byRepo: Record<string, ModelSidecarFile[]> = {};
for (const entry of response.data ?? []) {
const sidecar =
ModelsService.parseModelId(entry.id).sidecar ?? sidecarTokenInFilename(entry.id);
const parsed = ModelsService.parseModelId(entry.id);
const sidecar = parsed.sidecar ?? sidecarTokenInFilename(entry.id);
if (!sidecar || isAuxSidecar(sidecar)) continue;
// the repo is the id with its quant tag and sidecar token taken off; naming
// parts of the id are not touched, a model name may carry `-4b` for instance
const repo = entry.id
const model = entry.id
.split(MODEL_ID.QUANTIZATION_SEPARATOR)[0]
.replace(MODEL_ID.WEIGHT_EXTENSION_REGEX, '')
.replace(new RegExp(`[-_ ]?${sidecar}([-_ ]?draft)?$`, 'i'), '');
if (!repo) continue;
if (!model) continue;
const kinds = (byRepo[repo] ??= []);
const files = (byRepo[model] ??= []);
if (!kinds.includes(sidecar)) kinds.push(sidecar);
if (files.some((file) => file.kind === sidecar)) continue;
files.push({
id: entry.id,
kind: sidecar,
model,
params: paramsLabel(parsed),
quant: parsed.quantization
});
}
return byRepo;
@@ -7,16 +7,16 @@
* cache is the prefetch layer the switches start from.
*/
import type { ModelSidecar } from '$lib/constants';
import { BackendsService } from '$lib/services/backends.service';
import { ModelsService } from '$lib/services/models.service';
import { backendsStore } from '$lib/stores/backends.svelte';
import type { ApiModelsListResponse } from '$lib/types';
import type { ModelSidecarFile } from '$lib/types/models';
import type { ModelOption } from '$lib/types/models';
export interface BackendModelsState {
/** Draft sidecars the listing carries, keyed by the repo they belong to. */
drafts?: Record<string, ModelSidecar[]>;
drafts?: Record<string, ModelSidecarFile[]>;
error: string | null;
loaded: boolean;
loading: boolean;
+15
View File
@@ -51,6 +51,21 @@ export interface ModelDownloadProgress {
files: Record<string, ModelDownloadFileProgress>;
}
/**
* A draft sidecar file a model listing reports as its own entry. The router lists a
* downloaded sidecar as a model, so this is what pairs it back with its model.
*/
export interface ModelSidecarFile {
id: string;
kind: ModelSidecar;
/** Repo the sidecar belongs to, e.g. `ggml-org/Qwen3.6-35B-A3B-GGUF`. */
model: string;
/** Parameter count the sidecar reports, e.g. `35B-A3B`. */
params: string | null;
/** Quantization of the sidecar file, e.g. `Q4_0`. */
quant: string | null;
}
export interface ParsedModelId {
raw: string;
orgName: string | null;