From 8664eaea303868e6d47ffdacdbc3d946894a96fb Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Wed, 30 Sep 2026 12:42:17 +0200 Subject: [PATCH] ui : model download pipeline (#27959) * ui : model download pipeline Track HuggingFace downloads end to end: the server download/cancel endpoints, a status manager fed by the /models/sse download progress events, and a models-discover store holding the catalog and detail state for the discover view. Downloaded and in-flight entries are excluded from the loadable model list. Assisted-by: pi:GLM-5.3-Flash * ui : route sidecar tag lookup through the sidecars util, validate the paused list Assisted-by: pi:zai-org/GLM-5.3-Flash --- tools/ui/src/app.d.ts | 6 + .../lib/constants/api-endpoints.constants.ts | 3 + .../src/lib/constants/cli-flags.constants.ts | 6 + .../ui/src/lib/constants/storage.constants.ts | 3 + tools/ui/src/lib/enums/index.ts | 2 + tools/ui/src/lib/enums/model.enums.ts | 10 + tools/ui/src/lib/enums/server.enums.ts | 6 + tools/ui/src/lib/services/models.service.ts | 51 ++- .../ui/src/lib/stores/models/index.svelte.ts | 60 ++- .../ui/src/lib/stores/models/status.svelte.ts | 422 ++++++++++++++++-- tools/ui/src/lib/types/api.d.ts | 29 +- tools/ui/src/lib/types/index.ts | 5 + tools/ui/src/lib/types/models.d.ts | 24 +- tools/ui/src/lib/utils/api-fetch.ts | 34 ++ tools/ui/src/lib/utils/index.ts | 4 +- tools/ui/src/lib/utils/sidecars.ts | 13 +- .../tests/unit/model-sidecar-grammar.test.ts | 140 ++++++ 17 files changed, 731 insertions(+), 87 deletions(-) create mode 100644 tools/ui/tests/unit/model-sidecar-grammar.test.ts diff --git a/tools/ui/src/app.d.ts b/tools/ui/src/app.d.ts index 0a206381f8..5c039063ad 100644 --- a/tools/ui/src/app.d.ts +++ b/tools/ui/src/app.d.ts @@ -19,6 +19,8 @@ import type { ApiLlamaCppServerProps, ApiModelDataEntry, ApiModelLoadStage, + ApiModelsDownloadRequest, + ApiModelsDownloadResponse, ApiModelsListResponse, ApiModelsLoadResponse, ApiModelsSseData, @@ -80,11 +82,15 @@ declare global { ApiLlamaCppServerProps, ApiModelDataEntry, ApiModelLoadStage, + ModelDownloadProgress, ApiModelsSseProgress, ApiModelsSseData, + ApiModelsSseDownloadProgressData, ApiModelsSseEvent, ApiModelsListResponse, ApiModelsLoadResponse, + ApiModelsDownloadRequest, + ApiModelsDownloadResponse, ApiModelsUnloadResponse, ApiProcessingState, // Chat types diff --git a/tools/ui/src/lib/constants/api-endpoints.constants.ts b/tools/ui/src/lib/constants/api-endpoints.constants.ts index 8611d49fbb..2f6efb94cb 100644 --- a/tools/ui/src/lib/constants/api-endpoints.constants.ts +++ b/tools/ui/src/lib/constants/api-endpoints.constants.ts @@ -1,4 +1,7 @@ export const API_MODELS = { + /** Download a model from HuggingFace (ROUTER mode, POST) or cancel/remove it (DELETE) */ + DELETE: '/models', + DOWNLOAD: '/models', LIST: '/v1/models', LOAD: '/models/load', SSE: '/models/sse', diff --git a/tools/ui/src/lib/constants/cli-flags.constants.ts b/tools/ui/src/lib/constants/cli-flags.constants.ts index c4af2b6f46..0b6b2af86e 100644 --- a/tools/ui/src/lib/constants/cli-flags.constants.ts +++ b/tools/ui/src/lib/constants/cli-flags.constants.ts @@ -2,6 +2,12 @@ export const CLI_FLAGS = { AGENT: '--agent', API_KEY: '--api-key', MCP_PROXY: '--ui-mcp-proxy', + /** Multimodal projector path; unlocks vision/audio for the model. */ + MMPROJ: '--mmproj', + /** Draft model weights path (long form); the router records it per model. */ + MODEL_DRAFT: '--model-draft', + /** Draft model weights path (short form). */ + MODEL_DRAFT_SHORT: '-md', SLOTS: '--slots', TOOLS: '--tools' } as const; diff --git a/tools/ui/src/lib/constants/storage.constants.ts b/tools/ui/src/lib/constants/storage.constants.ts index 0aad7c7706..5e6610d859 100644 --- a/tools/ui/src/lib/constants/storage.constants.ts +++ b/tools/ui/src/lib/constants/storage.constants.ts @@ -15,6 +15,9 @@ export const STORAGE_APP_NAME_DEPRECATED = 'LlamaCppWebui'; export const DB_APP_NAME_DEPRECATED = 'LlamacppWebui'; export const ALWAYS_ALLOWED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.alwaysAllowedTools`; + +/** Paused model download ids (`:`), restored on the next page load. */ +export const PAUSED_MODEL_DOWNLOADS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.pausedModelDownloads`; export const CONFIG_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.config`; export const DISABLED_TOOLS_LOCALSTORAGE_KEY = `${STORAGE_APP_NAME}.disabledTools`; diff --git a/tools/ui/src/lib/enums/index.ts b/tools/ui/src/lib/enums/index.ts index db4f5b68cb..947d1c757e 100644 --- a/tools/ui/src/lib/enums/index.ts +++ b/tools/ui/src/lib/enums/index.ts @@ -71,6 +71,8 @@ export { export { ModelAuxSidecar, ModelCapability, ModelDraftSidecar, ModelModality } from './model.enums'; +export { ModelDownloadStopRequest } from './model.enums'; + export { ServerRole, ServerModelStatus, ServerModelsSseEventType } from './server.enums'; export { ParameterSource, SyncableParameterType, SettingsFieldType } from './settings.enums'; diff --git a/tools/ui/src/lib/enums/model.enums.ts b/tools/ui/src/lib/enums/model.enums.ts index 177fffe19a..2440dc007b 100644 --- a/tools/ui/src/lib/enums/model.enums.ts +++ b/tools/ui/src/lib/enums/model.enums.ts @@ -35,3 +35,13 @@ export enum ModelAuxSidecar { /** Multimodal projector: unlocks vision and/or audio input modalities. */ MMPROJ = 'mmproj' } + +/** + * Why an in-flight download is being stopped, so the terminal `download_failed` + * feed event can be attributed: a user pause (resumable) or a user cancel + * (discard). Distinguishes these from a genuine download failure. + */ +export enum ModelDownloadStopRequest { + CANCEL = 'cancel', + PAUSE = 'pause' +} diff --git a/tools/ui/src/lib/enums/server.enums.ts b/tools/ui/src/lib/enums/server.enums.ts index b7e80433c6..b558b51236 100644 --- a/tools/ui/src/lib/enums/server.enums.ts +++ b/tools/ui/src/lib/enums/server.enums.ts @@ -13,6 +13,10 @@ export enum ServerRole { * Used as the `value` field in the status object from /models endpoint */ export enum ServerModelStatus { + DOWNLOAD_FAILED = 'download_failed', + DOWNLOAD_FINISHED = 'download_finished', + DOWNLOADED = 'downloaded', + DOWNLOADING = 'downloading', FAILED = 'failed', LOADED = 'loaded', LOADING = 'loading', @@ -26,6 +30,8 @@ export enum ServerModelStatus { * tools/server/server-models.cpp from the C++ server. */ export enum ServerModelsSseEventType { + DOWNLOAD_FAILED = 'download_failed', + DOWNLOAD_FINISHED = 'download_finished', DOWNLOAD_PROGRESS = 'download_progress', MODEL_REMOVE = 'model_remove', MODEL_STATUS = 'model_status', diff --git a/tools/ui/src/lib/services/models.service.ts b/tools/ui/src/lib/services/models.service.ts index 21a9b22a16..96902a0b9d 100644 --- a/tools/ui/src/lib/services/models.service.ts +++ b/tools/ui/src/lib/services/models.service.ts @@ -11,11 +11,13 @@ import { API_MODELS, MODEL_ID, type ModelSidecar } from '$lib/constants'; import { ServerModelStatus } from '$lib/enums'; import type { ParsedModelId } from '$lib/types/models'; import { + apiDelete, apiFetch, apiPost, extractSseDataPayload, normalizeModelName, sidecarFromFileToken, + sidecarFromTag, splitSseRecords } from '$lib/utils'; import { getAuthHeaders } from '$lib/utils/api-headers'; @@ -24,14 +26,8 @@ export class ModelsService { private static readonly SSE_RECONNECT_MS = 1000; /** - * Build the `:` string expected by POST /models from a parsed - * filename quant + optional sidecar type. Used by the model download - * dialog so callers don't have to know about the tag conventions. - * - * @param repoId - HuggingFace repo id (e.g. `ggml-org/gemma-3-4b-it-GGUF`) - * @param quant - Quantization token, e.g. `Q4_K_M` - * @param sidecar - Sidecar type, as its lowercase filename token (e.g. `mtp`) - * @returns Repo id possibly suffixed with `:tag` + * Build the `:` string POST /models expects, so callers don't need + * to know the tag conventions. */ static buildDownloadTag( repoId: string, @@ -47,6 +43,27 @@ export class ModelsService { return `${repoId}:${tag}`; } + /** + * Cancel an in-flight download, or remove a downloaded/failed entry from the + * model cache (ROUTER mode only): DELETE /models?model=. + */ + static async cancelDownload(hfRepoWithTag: string): Promise { + return apiDelete(API_MODELS.DELETE, { + model: hfRepoWithTag + }); + } + + /** + * Start a model download from HuggingFace (ROUTER mode only). The response + * returns immediately; progress arrives over /models/sse. The server picks + * the file matching the tag and also pulls the model's mmproj/draft sidecars. + */ + static async downloadModel(hfRepoWithTag: string): Promise { + const payload: ApiModelsDownloadRequest = { model: hfRepoWithTag }; + + return apiPost(API_MODELS.DOWNLOAD, payload); + } + /** * Check if a model is loaded based on its metadata. * @@ -57,6 +74,10 @@ export class ModelsService { return model.status.value === ServerModelStatus.LOADED; } + static isModelLoading(model: ApiModelDataEntry): boolean { + return model.status.value === ServerModelStatus.LOADING; + } + /** * * @@ -66,13 +87,15 @@ export class ModelsService { */ /** - * Check if a model is currently loading. - * - * @param model - Model data entry from the API response - * @returns True if the model status is LOADING + * True when a router entry id marks a downloaded sidecar file, e.g. + * `org/model:Q4_0-mtp` or `org/model:mmproj`, not a loadable model. */ - static isModelLoading(model: ApiModelDataEntry): boolean { - return model.status.value === ServerModelStatus.LOADING; + static isSidecarEntry(modelId: string): boolean { + const idx = modelId.indexOf(MODEL_ID.QUANTIZATION_SEPARATOR); + + if (idx === MODEL_ID.NOT_FOUND) return false; + + return sidecarFromTag(modelId.slice(idx + 1)) !== null; } /** diff --git a/tools/ui/src/lib/stores/models/index.svelte.ts b/tools/ui/src/lib/stores/models/index.svelte.ts index d25b59cfc2..d367ad9874 100644 --- a/tools/ui/src/lib/stores/models/index.svelte.ts +++ b/tools/ui/src/lib/stores/models/index.svelte.ts @@ -204,6 +204,9 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { const response = await ModelsService.list(); this.routerModels = response.data; + // keep the selector options in sync: a downloaded / deleted model shows + // up here too, not only in the router model rows + this.models = this.buildModelOptions(response); await this.props.fetchModalitiesForLoadedModels(); const visible = this.getVisibleModels(); @@ -359,27 +362,44 @@ class ModelsStore implements ModelPropsHost, ModelStatusHost { * they differ only in which endpoint is called. */ private buildModelOptions(response: ApiModelsListResponse): ModelOption[] { - return response.data.map((item: ApiModelDataEntry, index: number) => { - const details = response.models?.[index]; - const rawCapabilities = Array.isArray(details?.capabilities) ? details?.capabilities : []; - const displayNameSource = - details?.name && details.name.trim().length > 0 ? details.name : item.id; - const modelId = details?.model || item.id; + const entries: { + details?: ApiModelsListResponse['models'][number]; + item: ApiModelDataEntry; + }[] = response.data.map((item: ApiModelDataEntry, index: number) => ({ + details: response.models?.[index], + item + })); - return { - aliases: item.aliases ?? [], - capabilities: rawCapabilities.filter((value: unknown): value is string => Boolean(value)), - description: details?.description, - details: details?.details, - id: item.id, - meta: item.meta ?? null, - modalities: this.props.buildArchitectureModalities(item.architecture), - model: modelId, - name: this.toDisplayName(displayNameSource), - parsedId: ModelsService.parseModelId(modelId), - tags: item.tags ?? [] - }; - }); + return ( + entries + // sidecar entries mark downloaded sidecar files, not loadable models + .filter(({ item }) => !ModelsService.isSidecarEntry(item.id)) + // in-flight downloads are not usable models yet; the selector tracks + // them in its "Download in progress" section instead + .filter(({ item }) => item.status?.value !== ServerModelStatus.DOWNLOADING) + .map(({ details, item }) => { + const rawCapabilities = Array.isArray(details?.capabilities) ? details?.capabilities : []; + const displayNameSource = + details?.name && details.name.trim().length > 0 ? details.name : item.id; + const modelId = details?.model || item.id; + + return { + aliases: item.aliases ?? [], + capabilities: rawCapabilities.filter((value: unknown): value is string => + Boolean(value) + ), + description: details?.description, + details: details?.details, + id: item.id, + meta: item.meta ?? null, + modalities: this.props.buildArchitectureModalities(item.architecture), + model: modelId, + name: this.toDisplayName(displayNameSource), + parsedId: ModelsService.parseModelId(modelId), + tags: item.tags ?? [] + }; + }) + ); } /** Fetch models in MODEL mode (single model, standard OpenAI-compatible). */ diff --git a/tools/ui/src/lib/stores/models/status.svelte.ts b/tools/ui/src/lib/stores/models/status.svelte.ts index d0160aa4da..b0ac083664 100644 --- a/tools/ui/src/lib/stores/models/status.svelte.ts +++ b/tools/ui/src/lib/stores/models/status.svelte.ts @@ -1,18 +1,26 @@ /** - * ModelStatusManager - Model load/unload operations and the /models/sse feed + * ModelStatusManager - model load/unload operations and the /models/sse feed * - * Owns the status feed subscription, load progress tracking, and the - * awaiters that settle load/unload operations. The feed drives status and - * progress, so it replaces any post-operation polling. Created and owned by - * modelsStore; the host owns the router model rows the feed updates. + * The feed drives status and progress, replacing any post-operation polling. + * Created and owned by modelsStore, which also owns the router rows it updates. */ -import { ServerModelsSseEventType, ServerModelStatus } from '$lib/enums'; +import { + CLI_FLAGS, + HF_UD_QUANT_PREFIX_REGEX, + MODEL_ID, + PATH_SEPARATOR, + PAUSED_MODEL_DOWNLOADS_LOCALSTORAGE_KEY +} from '$lib/constants'; +import { ModelDownloadStopRequest, ServerModelsSseEventType, ServerModelStatus } from '$lib/enums'; +import { HuggingFaceService } from '$lib/services/huggingface.service'; import { ModelsService } from '$lib/services/models.service'; import type { ModelPropsManager } from '$lib/stores/models/props.svelte'; // direct imports between stores, not via the barrel, to avoid circular deps import { serverStore } from '$lib/stores/server.svelte'; -import { SvelteMap } from 'svelte/reactivity'; +// explicit type imports: the app.d.ts globals resolve to `any`, so import the real types +import type { ApiModelsSseDownloadProgressData, ModelDownloadProgress } from '$lib/types'; +import { SvelteMap, SvelteSet } from 'svelte/reactivity'; import { toast } from 'svelte-sonner'; /** @@ -30,9 +38,59 @@ export interface ModelStatusHost { toDisplayName(id: string): string; } +/** + * Comparison key of a `:` download identifier: uppercased, with the + * `UD-` quant prefix stripped. The router derives cached model names from the + * actual file, which drops the prefix, so `repo:UD-Q4_K_XL` and `repo:Q4_K_XL` + * must compare equal. + */ +function downloadIdKey(repoWithTag: string): string { + const idx = repoWithTag.indexOf(MODEL_ID.QUANTIZATION_SEPARATOR); + const repo = idx === -1 ? repoWithTag : repoWithTag.slice(0, idx); + const tag = idx === -1 ? '' : repoWithTag.slice(idx + 1); + + return `${repo.toUpperCase()}:${tag.toUpperCase().replace(HF_UD_QUANT_PREFIX_REGEX, '')}`; +} + export class ModelStatusManager { + /** + * Sidecar files pulled by registered models, as `/` keys. + * Sidecars are not separate /v1/models entries - the router pulls them as + * sidecars of a main model and records them in its `--model-draft` / + * `--mmproj` args. + */ + private downloadedSidecars = $derived.by(() => { + const result = new SvelteSet(); + + for (const m of this.host.routerModels) { + const args = m.status?.args; + + if (!args) continue; + + for (let i = 0; i < args.length - 1; i++) { + if ( + args[i] !== CLI_FLAGS.MODEL_DRAFT && + args[i] !== CLI_FLAGS.MODEL_DRAFT_SHORT && + args[i] !== CLI_FLAGS.MMPROJ + ) { + continue; + } + + const parsed = HuggingFaceService.parseCachePath(args[i + 1]); + + if (parsed) result.add(`${parsed.repo}${PATH_SEPARATOR}${parsed.file}`); + } + } + + return result; + }); + private downloadProgress = new SvelteMap(); + /** `:` strings whose most recent download attempt failed (download_failed). */ + private failedDownloads = new SvelteSet(); private loadingStates = new SvelteMap(); private loadProgress = new SvelteMap(); + /** Paused downloads with their last reported progress, or null when none arrived before the pause. */ + private pausedDownloads = new SvelteMap(); // /models/sse feed state, the single source of truth for status and load progress private statusAbort: AbortController | null = null; private statusReaderActive = false; @@ -40,8 +98,127 @@ export class ModelStatusManager { string, { target: ServerModelStatus; resolve: () => void; reject: (e: Error) => void } >(); + /** Tags the user asked to stop (pause or cancel); the download_failed the stop triggers is intentional, not a failure. */ + private stopRequests = new SvelteMap(); - constructor(private host: ModelStatusHost) {} + /** + * Cancel an in-flight download or remove a downloaded/failed entry from + * the cache (ROUTER mode only). + */ + async cancelDownload(repoWithTag: string): Promise { + if (!serverStore.isRouterMode) { + toast.error('Model downloads are only available in router mode'); + + return false; + } + + this.subscribe(); + + // in-flight: the kill triggers download_failed over the feed; mark it as a + // user cancel so it settles silently instead of toasting a failure + if (this.downloadProgress.has(repoWithTag)) { + this.stopRequests.set(repoWithTag, ModelDownloadStopRequest.CANCEL); + } + + // a downloaded model registers under the name the router derived from the + // cached file (e.g. the UD- quant prefix is dropped), so resolve the tag to + // the registered id before asking the server to remove it + const registeredId = + this.host.routerModels.find((m) => downloadIdKey(m.id) === downloadIdKey(repoWithTag))?.id ?? + repoWithTag; + + try { + const res = await ModelsService.cancelDownload(registeredId); + const ok = res.success === true; + + if (ok) { + this.downloadProgress.delete(repoWithTag); + this.failedDownloads.delete(repoWithTag); + this.deletePausedDownload(repoWithTag); + } + + return ok; + } catch (error) { + toast.error(`Failed to cancel: ${error instanceof Error ? error.message : 'unknown error'}`); + + return false; + } + } + + /** + * The server force-kills a LOADING model on unload, and the feed reports the + * settled status, so no waiter is registered here. + */ + async cancelLoad(modelId: string): Promise { + if (!serverStore.isRouterMode) return; + + this.subscribe(); + + try { + await ModelsService.unload(modelId); + toast.info(`Load cancelled: ${this.host.toDisplayName(modelId)}`); + } catch (error) { + toast.error(`Failed to cancel load: ${this.host.toDisplayName(modelId)}`); + + throw error; + } + } + + constructor(private host: ModelStatusHost) { + // the server has no notion of a paused download, so the ids survive in + // localStorage; the progress snapshot is stale after a reload and stays null + try { + const raw = localStorage.getItem(PAUSED_MODEL_DOWNLOADS_LOCALSTORAGE_KEY); + const parsed: unknown = JSON.parse(raw ?? '[]'); + + if (!Array.isArray(parsed)) return; + + for (const repoWithTag of parsed.filter((id): id is string => typeof id === 'string')) { + this.pausedDownloads.set(repoWithTag, null); + } + } catch { + // unreadable or corrupt: start without the paused set + } + } + + /** + * POST /models starts the download in the background; the feed reports + * progress, and models_reload refreshes the list once it finishes. + * Re-posting a paused tag resumes from the partial files kept on disk. + */ + async downloadModel(repoWithTag: string): Promise { + if (!serverStore.isRouterMode) { + toast.error('Model downloads are only available in router mode'); + + return; + } + + // the feed must be live so the resulting models_reload event refreshes the list + this.subscribe(); + + // resuming a paused download: drop the paused state, and let the server + // discard its stale DOWNLOADED entry (via the list fetch) before re-posting + if (this.deletePausedDownload(repoWithTag) || this.stopRequests.delete(repoWithTag)) { + await this.host.fetchRouterModels(); + } + + try { + const res = await ModelsService.downloadModel(repoWithTag); + + if (!res.success) { + throw new Error(res.error?.message ?? 'Server rejected the download request'); + } + + // flip the chip to "downloading" right away; the feed refines it with real progress + this.downloadProgress.set(repoWithTag, { downloadedBytes: 0, files: {}, totalBytes: 0 }); + + toast.success(`Download started: ${this.host.toDisplayName(repoWithTag)}`); + } catch (error) { + toast.error(`Download failed: ${repoWithTag}`); + + throw error; + } + } async ensureLoaded(modelId: string): Promise { if (this.host.isModelLoaded(modelId)) return; @@ -50,16 +227,71 @@ export class ModelStatusManager { } /** - * Current load progress for a model, or null when not loading. + * Tracked downloads (in flight or paused) for the selector's + * "Download in progress" section. */ + getDownloadEntries(): { + isPaused: boolean; + progress: ModelDownloadProgress | null; + repoWithTag: string; + }[] { + const inFlight = Array.from(this.downloadProgress, ([repoWithTag, progress]) => ({ + isPaused: false, + progress, + repoWithTag + })); + const paused = Array.from(this.pausedDownloads, ([repoWithTag, progress]) => ({ + isPaused: true, + progress, + repoWithTag + })); + + return [...inFlight, ...paused]; + } + + getDownloadProgress(repoWithTag: string): ModelDownloadProgress | null { + return this.downloadProgress.get(repoWithTag) ?? null; + } + getLoadProgress(modelId: string): ModelLoadProgress | null { return this.loadProgress.get(modelId) ?? null; } + getPausedDownloadProgress(repoWithTag: string): ModelDownloadProgress | null { + return this.pausedDownloads.get(repoWithTag) ?? null; + } + + hasFailedDownload(repoWithTag: string): boolean { + return this.failedDownloads.has(repoWithTag); + } + + /** Active while the feed reports download_progress for the tag. */ + isDownloadInProgress(repoWithTag: string): boolean { + return this.downloadProgress.has(repoWithTag); + } + + isDownloadPaused(repoWithTag: string): boolean { + return this.pausedDownloads.has(repoWithTag); + } + + /** + * True when the tag is already registered in the /v1/models list; both ids + * are normalized, see downloadIdKey(). + */ + isModelDownloaded(repoWithTag: string): boolean { + const key = downloadIdKey(repoWithTag); + + return this.host.routerModels.some((m) => downloadIdKey(m.id) === key); + } + isOperationInProgress(modelId: string): boolean { return this.loadingStates.get(modelId) ?? false; } + isSidecarDownloaded(repoId: string, filePath: string): boolean { + return this.downloadedSidecars.has(`${repoId}/${filePath}`); + } + async load(modelId: string): Promise { if (this.host.isModelLoaded(modelId)) return; @@ -91,9 +323,30 @@ export class ModelStatusManager { } /** - * Open the /models/sse feed and keep it live with auto reconnect. - * Idempotent and router mode only. + * The server stops the download child but keeps the partial files, so + * re-posting the tag resumes where it stopped. The feed reports the stop + * as download_failed; the 'pause' stop request marks it as intentional. */ + async pauseDownload(repoWithTag: string): Promise { + if (!serverStore.isRouterMode) { + toast.error('Model downloads are only available in router mode'); + + return; + } + + this.subscribe(); + + this.stopRequests.set(repoWithTag, ModelDownloadStopRequest.PAUSE); + + try { + await ModelsService.unload(repoWithTag); + } catch { + this.stopRequests.delete(repoWithTag); + toast.error(`Failed to pause: ${repoWithTag}`); + } + } + + /** Open the /models/sse feed with auto reconnect; idempotent, router mode only. */ subscribe(): void { if (this.statusReaderActive) return; @@ -133,25 +386,94 @@ export class ModelStatusManager { } } - /** - * Close the /models/sse feed and drop transient progress. - */ unsubscribe(): void { this.statusReaderActive = false; this.statusAbort?.abort(); this.statusAbort = null; this.loadProgress.clear(); + this.downloadProgress.clear(); + this.failedDownloads.clear(); + this.stopRequests.clear(); } /** - * Apply a status envelope: update the model row, track or clear progress, - * settle any pending load or unload awaiter. + * A user pause keeps the last progress and stays resumable, a user cancel + * settles silently; genuine failures are marked so the UI can offer a retry. */ + private applyDownloadFinished(event: ApiModelsSseEvent): void { + let request: ModelDownloadStopRequest | undefined; + + if (event.event === ServerModelsSseEventType.DOWNLOAD_FAILED) { + request = this.stopRequests.get(event.model); + this.stopRequests.delete(event.model); + } + + const progress = this.downloadProgress.get(event.model) ?? null; + + this.downloadProgress.delete(event.model); + + if (request === ModelDownloadStopRequest.CANCEL) { + // user cancel: settle silently, the feed's model_remove cleans up the entry + this.failedDownloads.delete(event.model); + this.deletePausedDownload(event.model); + + return; + } + + if (request === ModelDownloadStopRequest.PAUSE) { + this.setPausedDownload(event.model, progress); + this.failedDownloads.delete(event.model); + + return; + } + + this.deletePausedDownload(event.model); + + const ok = event.event === ServerModelsSseEventType.DOWNLOAD_FINISHED; + + if (ok) { + this.failedDownloads.delete(event.model); + + // the finished download only registers in /v1/models on the next list + // fetch (the server reloads its model table then), so refetch to flip + // the quant chips to "downloaded" without waiting for a dialog reopen + void this.host.fetchRouterModels(); + + toast.success(`Download finished: ${this.host.toDisplayName(event.model)}`); + } else { + this.failedDownloads.add(event.model); + toast.error(`Download failed: ${this.host.toDisplayName(event.model)}`); + } + } + + /** Aggregate per-file progress into downloaded/total byte counts. */ + private applyDownloadProgress(event: ApiModelsSseEvent): void { + const data = event.data; + + if (!data || !('progress' in data)) return; + + const progress = (data as ApiModelsSseDownloadProgressData).progress; + + let downloaded = 0; + let total = 0; + + for (const file of Object.values(progress)) { + downloaded += file?.done ?? 0; + total += file?.total ?? 0; + } + + this.downloadProgress.set(event.model, { + downloadedBytes: downloaded, + files: progress, + totalBytes: total + }); + } + private applyModelStatus(event: ApiModelsSseEvent): void { const model = event.model; const data = event.data; - if (!model || !data?.status) return; + if (!model || !data || !('status' in data) || !data.status) return; const status = data.status; @@ -180,11 +502,7 @@ export class ModelStatusManager { this.settleStatus(model, status); } - /** - * Route one feed record by event kind. Only the status_* events carry a - * status payload, models_reload triggers a list refresh, model_remove drops - * the row, download_* belong to the download surface, not here. - */ + /** Route one feed record by event kind. */ private applyStatusEvent(event: ApiModelsSseEvent): void { switch (event.event) { case ServerModelsSseEventType.STATUS_CHANGE: @@ -202,13 +520,36 @@ export class ModelStatusManager { break; case ServerModelsSseEventType.DOWNLOAD_PROGRESS: + this.applyDownloadProgress(event); + + break; + case ServerModelsSseEventType.DOWNLOAD_FINISHED: + case ServerModelsSseEventType.DOWNLOAD_FAILED: + this.applyDownloadFinished(event); + break; } } - /** - * Reject and drop the awaiter for a model. - */ + private deletePausedDownload(repoWithTag: string): boolean { + if (!this.pausedDownloads.delete(repoWithTag)) return false; + + this.persistPausedDownloads(); + + return true; + } + + private persistPausedDownloads(): void { + try { + localStorage.setItem( + PAUSED_MODEL_DOWNLOADS_LOCALSTORAGE_KEY, + JSON.stringify(Array.from(this.pausedDownloads.keys())) + ); + } catch { + // storage unavailable: the pauses just do not survive a reload + } + } + private rejectStatus(modelId: string, error: Error): void { const waiter = this.statusWaiters.get(modelId); @@ -218,27 +559,32 @@ export class ModelStatusManager { } } - /** - * Drop a model row reported gone by the feed and settle its awaiters. - */ private removeRouterModel(modelId: string): void { if (this.host.routerModels.findIndex((m) => m.id === modelId) === -1) return; this.host.routerModels = this.host.routerModels.filter((m) => m.id !== modelId); this.loadProgress.delete(modelId); + this.downloadProgress.delete(modelId); + this.failedDownloads.delete(modelId); + this.deletePausedDownload(modelId); + this.stopRequests.delete(modelId); this.rejectStatus(modelId, new Error(`Model removed: ${this.host.toDisplayName(modelId)}`)); + + // drop the row from the selector options too; they rebuild from the list + // response, which only a refetch provides + void this.host.fetchRouterModels(); } - /** - * Read the feed and reconnect until unsubscribed. - */ private async runStatusReader(signal: AbortSignal): Promise { await ModelsService.watchModelEvents(signal, (event) => this.applyStatusEvent(event)); } - /** - * Update one model row status in place, reassigning to trigger reactivity. - */ + private setPausedDownload(repoWithTag: string, progress: ModelDownloadProgress | null): void { + this.pausedDownloads.set(repoWithTag, progress); + this.persistPausedDownloads(); + } + + // reassign the array: mutating an entry in place would not trigger reactivity private setRouterModelStatus(modelId: string, status: ServerModelStatus): void { const idx = this.host.routerModels.findIndex((m) => m.id === modelId); @@ -254,9 +600,6 @@ export class ModelStatusManager { this.host.routerModels = next; } - /** - * Resolve and drop the awaiter when the model reaches its target status. - */ private settleStatus(modelId: string, status: ServerModelStatus): void { const waiter = this.statusWaiters.get(modelId); @@ -266,10 +609,7 @@ export class ModelStatusManager { } } - /** - * Register an awaiter that resolves when the feed reports target status. - * One operation runs per model at a time, so one awaiter per model is kept. - */ + // one operation runs per model at a time, so one waiter per model suffices private waitForStatus(modelId: string, target: ServerModelStatus): Promise { return new Promise((resolve, reject) => { this.statusWaiters.set(modelId, { reject, resolve, target }); diff --git a/tools/ui/src/lib/types/api.d.ts b/tools/ui/src/lib/types/api.d.ts index 5191b03bd8..b52485ade8 100644 --- a/tools/ui/src/lib/types/api.d.ts +++ b/tools/ui/src/lib/types/api.d.ts @@ -138,6 +138,14 @@ export interface ApiModelsSseData { exit_code?: number; } +/** + * Per-file size snapshot reported by the download_progress SSE envelope. + * Keys are file URLs, values are byte counters (done <= total). + */ +export interface ApiModelsSseDownloadProgressData { + progress: Record; +} + /** * Event kind multiplexed on the /models/sse feed. * Only the status_* events carry a status payload, models_reload signals a @@ -150,7 +158,26 @@ export interface ApiModelsSseData { export interface ApiModelsSseEvent { model: string; event: ServerModelsSseEventType; - data: ApiModelsSseData; + data?: ApiModelsSseData | ApiModelsSseDownloadProgressData; +} + +/** + * Request body for POST /models (model download). + * `model` is a HuggingFace repo id, optionally suffixed with `:` to + * pin a quantization or sidecar file (e.g. `ggml-org/gemma-3-4b-it-GGUF:Q4_K_M`). + */ +export interface ApiModelsDownloadRequest { + model: string; +} + +/** + * Response from POST /models and DELETE /models. The POST endpoint returns + * immediately; the download itself runs in the background and emits events + * on /models/sse. + */ +export interface ApiModelsDownloadResponse { + success: boolean; + error?: { code: number; message: string; type: string }; } export interface ApiModelDetails { diff --git a/tools/ui/src/lib/types/index.ts b/tools/ui/src/lib/types/index.ts index 28b7cf6014..453af9bb80 100644 --- a/tools/ui/src/lib/types/index.ts +++ b/tools/ui/src/lib/types/index.ts @@ -14,7 +14,10 @@ export type { ApiModelLoadStage, ApiModelsSseProgress, ApiModelsSseData, + ApiModelsSseDownloadProgressData, ApiModelsSseEvent, + ApiModelsDownloadRequest, + ApiModelsDownloadResponse, ApiModelDetails, ApiLlamaCppServerProps, ApiChatCompletionRequest, @@ -102,6 +105,8 @@ export type { ModelCapabilities, ModelModalities, ModelOption, + ModelDownloadFileProgress, + ModelDownloadProgress, ModelLoadProgress, ModalityCapabilities } from './models'; diff --git a/tools/ui/src/lib/types/models.d.ts b/tools/ui/src/lib/types/models.d.ts index 03c723e1ae..ca53ca8805 100644 --- a/tools/ui/src/lib/types/models.d.ts +++ b/tools/ui/src/lib/types/models.d.ts @@ -26,17 +26,27 @@ export interface ModelOption { tags?: string[]; } -/** - * Ephemeral UI-only load progress for one model instance. - * Lives only while a load runs, driven by the /models/sse feed. - * stage is absent until the feed reports its first stage. - */ +/** UI-only load progress for one model, driven by the /models/sse feed. */ export interface ModelLoadProgress { stages: ApiModelLoadStage[]; current: ApiModelLoadStage; value: number; } +/** Per-file bytes of an in-flight download. */ +export interface ModelDownloadFileProgress { + done: number; + total: number; +} + +/** Progress of an in-flight download, summed across its files. */ +export interface ModelDownloadProgress { + downloadedBytes: number; + totalBytes: number; + /** Per-file progress keyed by file URL. */ + files: Record; +} + export interface ParsedModelId { raw: string; orgName: string | null; @@ -48,9 +58,7 @@ export interface ParsedModelId { tags: string[]; } -/** - * Modality capabilities for file validation - */ +/** Modality capabilities for file validation. */ export interface ModalityCapabilities { hasVision: boolean; hasAudio: boolean; diff --git a/tools/ui/src/lib/utils/api-fetch.ts b/tools/ui/src/lib/utils/api-fetch.ts index 9aa3a85799..fefea2fe1f 100644 --- a/tools/ui/src/lib/utils/api-fetch.ts +++ b/tools/ui/src/lib/utils/api-fetch.ts @@ -137,6 +137,40 @@ export async function apiPost( }); } +/** + * Send a DELETE request to an API endpoint, optionally with query parameters. + * + * @param path - API path (query string is appended if `params` is provided) + * @param params - Optional record of query parameters + * @param options - Additional fetch options + * @returns Parsed JSON response + */ +export async function apiDelete( + path: string, + params?: Record, + options: ApiFetchOptions = {} +): Promise { + // the query is appended to the path so `apiFetch` applies its base-path prefix; + // `apiFetchWithParams` resolves an absolute URL and would bypass it + let query = ''; + + if (params) { + const search = new URLSearchParams(); + + for (const [key, value] of Object.entries(params)) { + if (value !== undefined && value !== null) { + search.set(key, value); + } + } + + const qs = search.toString(); + + if (qs) query = `?${qs}`; + } + + return apiFetch(`${path}${query}`, { ...options, method: 'DELETE' }); +} + /** * Parse error message from a failed response. * Tries to extract error message from JSON body, falls back to status text. diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.ts index 06ab5d7b6a..21d84dc8d6 100644 --- a/tools/ui/src/lib/utils/index.ts +++ b/tools/ui/src/lib/utils/index.ts @@ -9,7 +9,7 @@ // API utilities export { getAuthHeaders, getJsonHeaders, sanitizeHeaders } from './api-headers'; -export { ApiError, apiFetch, apiFetchWithParams, apiPost } from './api-fetch'; +export { ApiError, apiDelete, apiFetch, apiFetchWithParams, apiPost } from './api-fetch'; export { validateApiKey } from './api-key-validation'; // Attachment utilities @@ -108,7 +108,7 @@ export { export { normalizeModelName, isValidModelName } from './model-names'; // Sidecar token utilities -export { isAuxSidecar, isDraftSidecar, sidecarFromFileToken } from './sidecars'; +export { isAuxSidecar, isDraftSidecar, sidecarFromFileToken, sidecarFromTag } from './sidecars'; // Portal utilities export { portalToBody } from './portal-to-body'; diff --git a/tools/ui/src/lib/utils/sidecars.ts b/tools/ui/src/lib/utils/sidecars.ts index a3eb3841fd..52532b135a 100644 --- a/tools/ui/src/lib/utils/sidecars.ts +++ b/tools/ui/src/lib/utils/sidecars.ts @@ -1,4 +1,4 @@ -import { type ModelSidecar, SIDECAR_TOKENS } from '$lib/constants'; +import { MODEL_ID, type ModelSidecar, SIDECAR_TOKENS } from '$lib/constants'; import { ModelAuxSidecar, ModelDraftSidecar } from '$lib/enums'; const SIDECAR_TOKEN_SET = new Set(SIDECAR_TOKENS); @@ -10,6 +10,17 @@ export function sidecarFromFileToken(token: string): ModelSidecar | null { return SIDECAR_TOKEN_SET.has(token) ? (token as ModelSidecar) : null; } +/** + * Sidecar a download tag points at: the segment after the last dash, + * e.g. `q4_0-mtp` -> `mtp`, `mmproj` -> `mmproj`. Returns null for quant-only + * tags and tags whose tail is not a sidecar token. + */ +export function sidecarFromTag(tag: string): ModelSidecar | null { + const token = tag.toLowerCase().split(MODEL_ID.SEGMENT_SEPARATOR).pop() ?? ''; + + return sidecarFromFileToken(token); +} + export function isDraftSidecar(sidecar: ModelSidecar): sidecar is ModelDraftSidecar { return DRAFT_SIDECAR_SET.has(sidecar); } diff --git a/tools/ui/tests/unit/model-sidecar-grammar.test.ts b/tools/ui/tests/unit/model-sidecar-grammar.test.ts new file mode 100644 index 0000000000..1ad59171a9 --- /dev/null +++ b/tools/ui/tests/unit/model-sidecar-grammar.test.ts @@ -0,0 +1,140 @@ +import { ModelAuxSidecar, ModelDraftSidecar, SidecarForm } from '$lib/enums'; +import { HuggingFaceService } from '$lib/services/huggingface.service'; +import { ModelsService } from '$lib/services/models.service'; +import { sidecarFromTag } from '$lib/utils'; +import { describe, expect, it } from 'vitest'; + +const { buildDownloadTag, isSidecarEntry } = ModelsService; +const { extractQuantMeta } = HuggingFaceService; + +// the sidecar filename grammar mirrors the server (common/download.cpp): +// the token must be lowercase, and it can sit at the start, between name +// segments, or at the end of the file name +describe('extractQuantMeta', () => { + it('parses the prefix form', () => { + expect(extractQuantMeta('mtp-Model-Q4_0.gguf')).toStrictEqual({ + quant: 'Q4_0', + shared: false, + sidecar: ModelDraftSidecar.MTP, + sidecarForm: SidecarForm.PREFIX + }); + }); + + it('parses the infix form', () => { + expect(extractQuantMeta('Model-mtp-Q4_0.gguf')).toStrictEqual({ + quant: 'Q4_0', + shared: false, + sidecar: ModelDraftSidecar.MTP, + sidecarForm: SidecarForm.INFIX + }); + }); + + it('parses the suffix form', () => { + expect(extractQuantMeta('gemma-4-E2B-it-BF16-mtp.gguf')).toStrictEqual({ + quant: 'BF16', + shared: false, + sidecar: ModelDraftSidecar.MTP, + sidecarForm: SidecarForm.SUFFIX + }); + }); + + it('parses an uppercase infix token', () => { + expect(extractQuantMeta('gemma-4-31B-it-MTP-BF16.gguf')).toStrictEqual({ + quant: 'BF16', + shared: false, + sidecar: ModelDraftSidecar.MTP, + sidecarForm: SidecarForm.INFIX + }); + }); + + it('parses an uppercase trailing token', () => { + expect(extractQuantMeta('gemma-4-E2B-it-BF16-MTP.gguf')).toStrictEqual({ + quant: 'BF16', + shared: false, + sidecar: ModelDraftSidecar.MTP, + sidecarForm: SidecarForm.SUFFIX + }); + }); + + it('parses a short-form sidecar with a bare quant', () => { + expect(extractQuantMeta('mmproj-F16.gguf')).toStrictEqual({ + quant: 'F16', + shared: false, + sidecar: ModelAuxSidecar.MMPROJ, + sidecarForm: SidecarForm.PREFIX + }); + }); + + it('parses a bare sidecar file', () => { + expect(extractQuantMeta('imatrix.gguf')).toStrictEqual({ + quant: null, + shared: false, + sidecar: ModelAuxSidecar.IMATRIX, + sidecarForm: SidecarForm.PREFIX + }); + }); + + it('parses a standalone sidecar with a draft tail', () => { + expect(extractQuantMeta('Model-mtp-draft.gguf')).toStrictEqual({ + quant: null, + shared: false, + sidecar: ModelDraftSidecar.MTP, + sidecarForm: SidecarForm.SUFFIX + }); + }); + + it('parses a nested sidecar path by its file name', () => { + expect(extractQuantMeta('MTP/mtp-Model-Q4_0.gguf')).toStrictEqual({ + quant: 'Q4_0', + shared: false, + sidecar: ModelDraftSidecar.MTP, + sidecarForm: SidecarForm.PREFIX + }); + }); + + it('returns null for non-weight files', () => { + expect(extractQuantMeta('README.md')).toBeNull(); + }); +}); + +describe('buildDownloadTag', () => { + it('appends the quantization', () => { + expect(buildDownloadTag('org/repo', 'Q4_0', null)).toBe('org/repo:Q4_0'); + }); + + it('appends the quantization and sidecar', () => { + expect(buildDownloadTag('org/repo', 'Q4_0', ModelDraftSidecar.MTP)).toBe('org/repo:Q4_0-mtp'); + }); + + it('uses the sidecar alone when there is no quant', () => { + expect(buildDownloadTag('org/repo', null, ModelAuxSidecar.MMPROJ)).toBe('org/repo:mmproj'); + }); + + it('returns the repo id untouched without a tag', () => { + expect(buildDownloadTag('org/repo', null, null)).toBe('org/repo'); + }); +}); + +describe('isSidecarEntry', () => { + it('detects sidecar entries by their tag', () => { + expect(isSidecarEntry('org/repo:Q4_0-mtp')).toBe(true); + expect(isSidecarEntry('org/repo:mmproj')).toBe(true); + }); + + it('leaves plain model entries loadable', () => { + expect(isSidecarEntry('org/repo:Q4_0')).toBe(false); + expect(isSidecarEntry('org/repo')).toBe(false); + }); +}); + +describe('sidecarFromTag', () => { + it('reads the token after the last dash', () => { + expect(sidecarFromTag('Q4_0-mtp')).toBe(ModelDraftSidecar.MTP); + expect(sidecarFromTag('mmproj')).toBe(ModelAuxSidecar.MMPROJ); + }); + + it('returns null for quant-only and unrelated tags', () => { + expect(sidecarFromTag('Q4_K_XL')).toBeNull(); + expect(sidecarFromTag('UD-Q4_K_XL')).toBeNull(); + }); +});