diff --git a/CHANGELOG.md b/CHANGELOG.md index e36cb18e..7f5c01e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ the frozen-backend fallback mirror it for their toolchains. **Highlights** +- Model-load GPU exhaustion now returns a sanitized, actionable dubbing error, and readiness correctly attributes the shared model status to TTS (#1695) - Source-mode development now restarts an isolated backend crash without tearing down the UI, while repeated crash loops still stop loudly with diagnostics (#1690) - Dubbing playback now keeps an audible companion source when a WebView can render the preview picture but cannot decode its audio (#1692) - Model Catalogue engine rows now use the available desktop width and keep identity, runtime state, and actions from crowding one another (#1689) diff --git a/backend/api/routers/dub_generate.py b/backend/api/routers/dub_generate.py index 314e9eec..58fdd48b 100644 --- a/backend/api/routers/dub_generate.py +++ b/backend/api/routers/dub_generate.py @@ -503,6 +503,18 @@ async def dub_generate(job_id: str, req: DubRequest): backend = await resolve_generation_backend(require_cloning=True) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + from core.failure import is_gpu_oom + + if not is_gpu_oom(e): + raise + from core.public_errors import public_exception_response + + payload = public_exception_response( + e, + fallback="The TTS model could not be loaded.", + ) + raise HTTPException(status_code=503, detail=payload["detail"]) from e async def _stream(task_id): total = len(req.segments) @@ -1291,7 +1303,15 @@ async def dub_generate(job_id: str, req: DubRequest): pass _release_audio_tensors() except Exception as e: - yield f"data: {json.dumps({'type': 'error', 'segment': i, 'error': str(e)})}\n\n" + # A task-stream error bypasses the global exception handler. + # Never publish engine exception text here: allocator errors + # carry process tables and arbitrary failures can carry paths, + # tokens, or source text. The shared helper enriches recognized + # classes using VoiceStudio-owned constants only. + from core.public_errors import stream_generation_failure + + error_detail = stream_generation_failure(e)["detail"] + yield f"data: {json.dumps({'type': 'error', 'segment': i, 'error': error_detail})}\n\n" sr = backend.sample_rate all_segment_wavs.append(_store_mix_wav(seg.start, seg.end, torch.zeros(1, max(0, int(seg_duration * sr))), sr, f"mix_{seg_id}")) sync_scores.append(1.0) diff --git a/backend/core/failure.py b/backend/core/failure.py index 2d72d192..2590b9ef 100644 --- a/backend/core/failure.py +++ b/backend/core/failure.py @@ -52,6 +52,7 @@ _REDACTED_VALUE = "***REDACTED***" # One-line "what to do" per docs-taxonomy key. Keys mirror error_docs_map's # taxonomy; the docs URL itself stays owned by error_docs_map. _HINTS: dict[str, str] = { + "GPU_OOM": "Close other GPU-heavy apps or unload models, then retry. You can also choose CPU in Settings → Performance & Device or select a smaller TTS engine.", "WORKER_AT_CAPACITY": "Wait for a running job on that worker to finish, or choose another available worker and retry.", "MODEL_NOT_INSTALLED": "Install or enable this engine on the worker machine, then refresh its capabilities and retry.", "MODEL_NOT_DOWNLOADED": "Open Models, install this model on the selected worker, then retry when the download completes.", @@ -290,6 +291,9 @@ def append_hf_mirror_hint(text: str) -> str: # must NOT be added: its bare "timed out" trigger would stamp a "video server" # hint on a model-load timeout that leaks through the 500 handler. _CONTEXT_FREE_HINT_CLASSES = frozenset({ + # Device allocator signatures are specific enough to attach the shared + # recovery without exposing CUDA's process table or filesystem paths. + "GPU_OOM", "SOCKS_PROXY_SUPPORT_MISSING", "SSL_HANDSHAKE_FAILURE", # Its trigger is an exact OpenSSL string, so it cannot be confused with @@ -323,6 +327,38 @@ def append_hint(text: str) -> str: return f"{text} — {hint}" if hint else text +_GPU_OOM_SIGNATURES = ( + "cuda out of memory", + "cuda error: out of memory", + "cuda_error_out_of_memory", + "mps backend out of memory", + "hip out of memory", + "out of memory on device", +) + + +def is_gpu_oom(error: BaseException | str) -> bool: + """Recognize device OOMs through wrappers without importing torch.""" + pending: list[BaseException] = [error] if isinstance(error, BaseException) else [] + seen: set[int] = set() + while pending: + current = pending.pop() + if id(current) in seen: + continue + seen.add(id(current)) + if type(current).__name__ == "OutOfMemoryError": + return True + if any(signature in str(current).lower() for signature in _GPU_OOM_SIGNATURES): + return True + if current.__cause__ is not None: + pending.append(current.__cause__) + if current.__context__ is not None: + pending.append(current.__context__) + if isinstance(error, str): + return any(signature in error.lower() for signature in _GPU_OOM_SIGNATURES) + return False + + def classify(reason: str) -> str: """Map a failure reason to a docs-taxonomy key, or "" when unknown. @@ -330,6 +366,8 @@ def classify(reason: str) -> str: backend log / diagnostic names the same class the UI deeplink will use. """ low = (reason or "").lower() + if is_gpu_oom(low): + return "GPU_OOM" if "pkg_resources" in low: return "PKG_RESOURCES_MISSING" if "quarantine" in low or "is damaged" in low or "gatekeeper" in low: diff --git a/backend/services/model_manager.py b/backend/services/model_manager.py index 4b47edd7..ef9bf4c5 100644 --- a/backend/services/model_manager.py +++ b/backend/services/model_manager.py @@ -2938,7 +2938,10 @@ async def preload_model(): "The TTS model could not be loaded. Settings → Logs → Backend " "has the full error." ) - _set_loading("failed", detail, error=detail) + # `sub_stage` is a public API enum and the frontend keys failure state + # off `error`. Keep the human-readable word "failed" in the detail, + # not in the state machine (#1695). + _set_loading("error", detail, error=detail) def get_model_status(): is_loaded = model is not None diff --git a/frontend/src/components/ReadinessChecklist.jsx b/frontend/src/components/ReadinessChecklist.jsx index 5857f06f..46952cab 100644 --- a/frontend/src/components/ReadinessChecklist.jsx +++ b/frontend/src/components/ReadinessChecklist.jsx @@ -37,6 +37,7 @@ export default function ReadinessChecklist({ compact = false, showWhenAllPass = const isLoading = preflightLoading || modelLoading; const modelStatus = modelData?.status ?? 'idle'; + const modelFailed = modelStatus === 'error' || ['error', 'failed'].includes(modelData?.sub_stage); // Build the checklist from preflight data + model status const checks = []; @@ -45,14 +46,14 @@ export default function ReadinessChecklist({ compact = false, showWhenAllPass = const modelDetail = modelData?.detail || ''; const modelErr = modelData?.error || null; const modelCheck = { - id: 'asr-model', - label: t('readiness.asr_model'), + id: 'tts-model', + label: t('readiness.tts_model'), status: modelStatus === 'ready' ? 'pass' : modelStatus === 'loading' ? 'loading' - : modelStatus === 'error' || modelData?.sub_stage === 'error' + : modelFailed ? 'fail' : 'warn', detail: @@ -60,15 +61,14 @@ export default function ReadinessChecklist({ compact = false, showWhenAllPass = ? t('readiness.loaded_ready') : modelStatus === 'loading' ? modelDetail || t('readiness.loading_first_run') - : modelData?.sub_stage === 'error' + : modelFailed ? modelErr || t('readiness.failed_to_load') - : t('readiness.not_loaded_yet'), - fix: - modelStatus === 'error' || modelData?.sub_stage === 'error' - ? modelErr - ? t('readiness.error_check_logs', { error: modelErr }) - : t('readiness.check_logs_restart') - : null, + : t('readiness.tts_not_loaded_yet'), + fix: modelFailed + ? modelErr + ? t('readiness.error_check_logs', { error: modelErr }) + : t('readiness.check_logs_restart') + : null, }; checks.push(modelCheck); diff --git a/frontend/src/i18n/locales/ar.json b/frontend/src/i18n/locales/ar.json index f1204b72..5b9d5409 100644 --- a/frontend/src/i18n/locales/ar.json +++ b/frontend/src/i18n/locales/ar.json @@ -1728,10 +1728,12 @@ "all_ready": "جميع الأنظمة جاهزة", "system_readiness": "جاهزية النظام", "asr_model": "نموذج ASR", + "tts_model": "نموذج TTS", "loaded_ready": "محملة وجاهزة", "loading_first_run": "جارٍ التحميل... (قد يستغرق هذا من دقيقة إلى دقيقتين عند التشغيل لأول مرة)", "failed_to_load": "فشل التحميل", "not_loaded_yet": "لم يتم تحميله بعد - سيتم تحميله عند النسخ الأول", + "tts_not_loaded_yet": "لم يتم تحميله بعد — سيُحمّل عند أول توليد صوتي", "error_check_logs": "خطأ: {{error}}. تحقق من السجلات وحاول إعادة التشغيل.", "check_logs_restart": "تحقق من السجلات بحثًا عن أخطاء تحميل النموذج. حاول إعادة التشغيل.", "llm_cinematic": "ماجستير في القانون (السينمائي)", diff --git a/frontend/src/i18n/locales/de.json b/frontend/src/i18n/locales/de.json index 8b6181b9..f1c8a203 100644 --- a/frontend/src/i18n/locales/de.json +++ b/frontend/src/i18n/locales/de.json @@ -1728,10 +1728,12 @@ "all_ready": "Alle Systeme bereit", "system_readiness": "Systembereitschaft", "asr_model": "ASR-Modell", + "tts_model": "TTS-Modell", "loaded_ready": "Geladen und fertig", "loading_first_run": "Wird geladen… (dies kann beim ersten Durchlauf 1-2 Minuten dauern)", "failed_to_load": "Laden fehlgeschlagen", "not_loaded_yet": "Noch nicht geladen – wird bei der ersten Transkription geladen", + "tts_not_loaded_yet": "Noch nicht geladen – wird bei der ersten Spracherzeugung geladen", "error_check_logs": "Fehler: {{error}}. Überprüfen Sie die Protokolle und versuchen Sie einen Neustart.", "check_logs_restart": "Überprüfen Sie die Protokolle auf Modellladefehler. Versuchen Sie einen Neustart.", "llm_cinematic": "LLM (Film)", diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index 201afb6f..5497f574 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -2261,10 +2261,12 @@ "all_ready": "All systems ready", "system_readiness": "System Readiness", "asr_model": "ASR Model", + "tts_model": "TTS Model", "loaded_ready": "Loaded and ready", "loading_first_run": "Loading… (this may take 1-2 minutes on first run)", "failed_to_load": "Failed to load", "not_loaded_yet": "Not loaded yet — will load on first transcription", + "tts_not_loaded_yet": "Not loaded yet — will load on first speech generation", "error_check_logs": "Error: {{error}}. Check logs and try restarting.", "check_logs_restart": "Check logs for model loading errors. Try restarting.", "llm_cinematic": "LLM (Cinematic)", diff --git a/frontend/src/i18n/locales/es.json b/frontend/src/i18n/locales/es.json index 57cb89c4..53b9181c 100644 --- a/frontend/src/i18n/locales/es.json +++ b/frontend/src/i18n/locales/es.json @@ -1728,10 +1728,12 @@ "all_ready": "Todos los sistemas listos", "system_readiness": "Preparación del sistema", "asr_model": "Modelo ASR", + "tts_model": "Modelo TTS", "loaded_ready": "Cargado y listo", "loading_first_run": "Cargando... (esto puede tardar entre 1 y 2 minutos en la primera ejecución)", "failed_to_load": "No se pudo cargar", "not_loaded_yet": "Aún no cargado: se cargará en la primera transcripción", + "tts_not_loaded_yet": "Aún no está cargado; se cargará al generar voz por primera vez", "error_check_logs": "Error: {{error}}. Verifique los registros e intente reiniciar.", "check_logs_restart": "Verifique los registros para detectar errores de carga del modelo. Intenta reiniciar.", "llm_cinematic": "LLM (Cinemático)", diff --git a/frontend/src/i18n/locales/fr.json b/frontend/src/i18n/locales/fr.json index ca899b11..7071d954 100644 --- a/frontend/src/i18n/locales/fr.json +++ b/frontend/src/i18n/locales/fr.json @@ -1728,10 +1728,12 @@ "all_ready": "Tous les systèmes sont prêts", "system_readiness": "État de préparation du système", "asr_model": "Modèle ASR", + "tts_model": "Modèle TTS", "loaded_ready": "Chargé et prêt", "loading_first_run": "Chargement… (cela peut prendre 1 à 2 minutes lors de la première exécution)", "failed_to_load": "Échec du chargement", "not_loaded_yet": "Pas encore chargé – se chargera lors de la première transcription", + "tts_not_loaded_yet": "Pas encore chargé — se chargera lors de la première génération vocale", "error_check_logs": "Erreur : {{error}}. Vérifiez les journaux et essayez de redémarrer.", "check_logs_restart": "Vérifiez les journaux pour détecter les erreurs de chargement du modèle. Essayez de redémarrer.", "llm_cinematic": "LLM (Cinématique)", diff --git a/frontend/src/i18n/locales/hi.json b/frontend/src/i18n/locales/hi.json index eb88f8db..60b22e96 100644 --- a/frontend/src/i18n/locales/hi.json +++ b/frontend/src/i18n/locales/hi.json @@ -1728,10 +1728,12 @@ "all_ready": "सभी सिस्टम तैयार", "system_readiness": "सिस्टम की तैयारी", "asr_model": "एएसआर मॉडल", + "tts_model": "टीटीएस मॉडल", "loaded_ready": "लोड और तैयार", "loading_first_run": "लोड हो रहा है... (पहली बार चलाने में 1-2 मिनट लग सकते हैं)", "failed_to_load": "लोड करने में विफल", "not_loaded_yet": "अभी तक लोड नहीं हुआ है - प्रथम प्रतिलेखन पर लोड होगा", + "tts_not_loaded_yet": "अभी लोड नहीं हुआ — पहली बार आवाज़ बनाने पर लोड होगा", "error_check_logs": "त्रुटि: {{error}}. लॉग जांचें और पुनः प्रारंभ करने का प्रयास करें।", "check_logs_restart": "मॉडल लोडिंग त्रुटियों के लिए लॉग की जाँच करें। पुनः आरंभ करने का प्रयास करें.", "llm_cinematic": "एलएलएम (सिनेमाई)", diff --git a/frontend/src/i18n/locales/id.json b/frontend/src/i18n/locales/id.json index cbe043c3..02d4ce1a 100644 --- a/frontend/src/i18n/locales/id.json +++ b/frontend/src/i18n/locales/id.json @@ -1728,10 +1728,12 @@ "all_ready": "Semua sistem siap", "system_readiness": "Kesiapan Sistem", "asr_model": "Model ASR", + "tts_model": "Model TTS", "loaded_ready": "Sudah terisi dan siap", "loading_first_run": "Memuat… (ini mungkin memakan waktu 1-2 menit saat pertama kali dijalankan)", "failed_to_load": "Gagal memuat", "not_loaded_yet": "Belum dimuat — akan dimuat pada transkripsi pertama", + "tts_not_loaded_yet": "Belum dimuat — akan dimuat saat pertama kali menghasilkan suara", "error_check_logs": "Kesalahan: {{error}}. Periksa log dan coba mulai ulang.", "check_logs_restart": "Periksa log untuk mengetahui kesalahan pemuatan model. Coba mulai ulang.", "llm_cinematic": "LLM (Sinematik)", diff --git a/frontend/src/i18n/locales/it.json b/frontend/src/i18n/locales/it.json index 94b1f020..b85128c0 100644 --- a/frontend/src/i18n/locales/it.json +++ b/frontend/src/i18n/locales/it.json @@ -1728,10 +1728,12 @@ "all_ready": "Tutti i sistemi pronti", "system_readiness": "Prontezza del sistema", "asr_model": "Modello ASR", + "tts_model": "Modello TTS", "loaded_ready": "Carico e pronto", "loading_first_run": "Caricamento... (l'operazione potrebbe richiedere 1-2 minuti alla prima esecuzione)", "failed_to_load": "Impossibile caricare", "not_loaded_yet": "Non ancora caricato: verrà caricato alla prima trascrizione", + "tts_not_loaded_yet": "Non ancora caricato: verrà caricato alla prima generazione vocale", "error_check_logs": "Errore: {{error}}. Controlla i log e prova a riavviare.", "check_logs_restart": "Controlla i log per eventuali errori di caricamento del modello. Prova a riavviare.", "llm_cinematic": "LLM (Cinematologico)", diff --git a/frontend/src/i18n/locales/ja.json b/frontend/src/i18n/locales/ja.json index 62378a84..36b50062 100644 --- a/frontend/src/i18n/locales/ja.json +++ b/frontend/src/i18n/locales/ja.json @@ -1728,10 +1728,12 @@ "all_ready": "すべてのシステムが準備完了", "system_readiness": "システムの準備状況", "asr_model": "ASRモデル", + "tts_model": "TTSモデル", "loaded_ready": "ロードされて準備完了", "loading_first_run": "読み込み中… (初回実行時は 1 ~ 2 分かかる場合があります)", "failed_to_load": "ロードに失敗しました", "not_loaded_yet": "まだロードされていません - 最初の文字起こし時にロードされます", + "tts_not_loaded_yet": "未ロード — 初回の音声生成時にロードされます", "error_check_logs": "エラー: {{error}}。ログを確認して再起動してみてください。", "check_logs_restart": "モデルの読み込みエラーがないかログを確認します。再起動してみてください。", "llm_cinematic": "LLM (映画)", diff --git a/frontend/src/i18n/locales/ko.json b/frontend/src/i18n/locales/ko.json index d861bfb0..48c8bd37 100644 --- a/frontend/src/i18n/locales/ko.json +++ b/frontend/src/i18n/locales/ko.json @@ -1728,10 +1728,12 @@ "all_ready": "모든 시스템 준비", "system_readiness": "시스템 준비", "asr_model": "ASR 모델", + "tts_model": "TTS 모델", "loaded_ready": "로드 및 준비됨", "loading_first_run": "로드 중… (처음 실행 시 1~2분 정도 소요될 수 있음)", "failed_to_load": "로드하지 못했습니다.", "not_loaded_yet": "아직 로드되지 않음 - 첫 번째 기록 시 로드됩니다.", + "tts_not_loaded_yet": "아직 로드되지 않음 — 첫 음성 생성 시 로드됩니다", "error_check_logs": "오류: {{error}}. 로그를 확인하고 다시 시작해 보세요.", "check_logs_restart": "모델 로드 오류에 대한 로그를 확인하세요. 다시 시작해 보세요.", "llm_cinematic": "LLM (영화)", diff --git a/frontend/src/i18n/locales/nl.json b/frontend/src/i18n/locales/nl.json index a7edb226..82a131e3 100644 --- a/frontend/src/i18n/locales/nl.json +++ b/frontend/src/i18n/locales/nl.json @@ -1728,10 +1728,12 @@ "all_ready": "Alle systemen klaar", "system_readiness": "Systeemgereedheid", "asr_model": "ASR-model", + "tts_model": "TTS-model", "loaded_ready": "Geladen en klaar", "loading_first_run": "Laden... (dit kan bij de eerste run 1-2 minuten duren)", "failed_to_load": "Kan niet laden", "not_loaded_yet": "Nog niet geladen: wordt geladen bij de eerste transcriptie", + "tts_not_loaded_yet": "Nog niet geladen — wordt geladen bij de eerste spraakgeneratie", "error_check_logs": "Fout: {{error}}. Controleer de logboeken en probeer opnieuw op te starten.", "check_logs_restart": "Controleer logboeken op fouten bij het laden van modellen. Probeer opnieuw op te starten.", "llm_cinematic": "LLM (filmisch)", diff --git a/frontend/src/i18n/locales/pl.json b/frontend/src/i18n/locales/pl.json index 4dabd876..34a78ecf 100644 --- a/frontend/src/i18n/locales/pl.json +++ b/frontend/src/i18n/locales/pl.json @@ -1728,10 +1728,12 @@ "all_ready": "Wszystkie systemy gotowe", "system_readiness": "Gotowość systemu", "asr_model": "Model ASR", + "tts_model": "Model TTS", "loaded_ready": "Załadowany i gotowy", "loading_first_run": "Ładowanie… (przy pierwszym uruchomieniu może to zająć 1–2 minuty)", "failed_to_load": "Nie udało się załadować", "not_loaded_yet": "Jeszcze nie załadowano — zostanie załadowane przy pierwszej transkrypcji", + "tts_not_loaded_yet": "Jeszcze nie załadowano — zostanie załadowany przy pierwszym generowaniu mowy", "error_check_logs": "Błąd: {{error}}. Sprawdź dzienniki i spróbuj uruchomić ponownie.", "check_logs_restart": "Sprawdź dzienniki pod kątem błędów ładowania modelu. Spróbuj uruchomić ponownie.", "llm_cinematic": "LLM (film)", diff --git a/frontend/src/i18n/locales/pt.json b/frontend/src/i18n/locales/pt.json index 17b78723..b4a1c08b 100644 --- a/frontend/src/i18n/locales/pt.json +++ b/frontend/src/i18n/locales/pt.json @@ -1728,10 +1728,12 @@ "all_ready": "Todos os sistemas prontos", "system_readiness": "Preparação do sistema", "asr_model": "Modelo ASR", + "tts_model": "Modelo TTS", "loaded_ready": "Carregado e pronto", "loading_first_run": "Carregando… (isso pode levar de 1 a 2 minutos na primeira execução)", "failed_to_load": "Falha ao carregar", "not_loaded_yet": "Ainda não carregado — será carregado na primeira transcrição", + "tts_not_loaded_yet": "Ainda não carregado — será carregado na primeira geração de voz", "error_check_logs": "Erro: {{error}}. Verifique os logs e tente reiniciar.", "check_logs_restart": "Verifique os logs para erros de carregamento do modelo. Tente reiniciar.", "llm_cinematic": "LLM (Cinemático)", diff --git a/frontend/src/i18n/locales/ru.json b/frontend/src/i18n/locales/ru.json index c382c78c..2e8d97ba 100644 --- a/frontend/src/i18n/locales/ru.json +++ b/frontend/src/i18n/locales/ru.json @@ -1728,10 +1728,12 @@ "all_ready": "Все системы готовы", "system_readiness": "Готовность системы", "asr_model": "Модель ASR", + "tts_model": "Модель TTS", "loaded_ready": "Загружено и готово", "loading_first_run": "Загрузка… (при первом запуске это может занять 1–2 минуты)", "failed_to_load": "Не удалось загрузить", "not_loaded_yet": "Еще не загружено — загрузится при первой транскрипции", + "tts_not_loaded_yet": "Ещё не загружена — загрузится при первой генерации речи", "error_check_logs": "Ошибка: {{error}}. Проверьте журналы и попробуйте перезагрузить компьютер.", "check_logs_restart": "Проверьте журналы на предмет ошибок загрузки модели. Попробуйте перезапустить.", "llm_cinematic": "LLM (кинематографический)", diff --git a/frontend/src/i18n/locales/sv.json b/frontend/src/i18n/locales/sv.json index 0207e79b..7cff7f1d 100644 --- a/frontend/src/i18n/locales/sv.json +++ b/frontend/src/i18n/locales/sv.json @@ -1728,10 +1728,12 @@ "all_ready": "Alla system redo", "system_readiness": "Systemberedskap", "asr_model": "ASR modell", + "tts_model": "TTS-modell", "loaded_ready": "Laddat och klart", "loading_first_run": "Laddar... (detta kan ta 1-2 minuter vid första körningen)", "failed_to_load": "Det gick inte att ladda", "not_loaded_yet": "Inte laddad än – kommer att laddas vid första transkription", + "tts_not_loaded_yet": "Inte inläst ännu – läses in vid den första röstgenereringen", "error_check_logs": "Fel: {{error}}. Kontrollera loggar och försök starta om.", "check_logs_restart": "Kontrollera loggar för modellladdningsfel. Testa att starta om.", "llm_cinematic": "LLM (Cinematic)", diff --git a/frontend/src/i18n/locales/th.json b/frontend/src/i18n/locales/th.json index 3b4822ac..ea1691e8 100644 --- a/frontend/src/i18n/locales/th.json +++ b/frontend/src/i18n/locales/th.json @@ -1728,10 +1728,12 @@ "all_ready": "พร้อมทุกระบบ", "system_readiness": "ความพร้อมของระบบ", "asr_model": "รุ่น ASR", + "tts_model": "โมเดล TTS", "loaded_ready": "โหลดแล้วพร้อมครับ", "loading_first_run": "กำลังโหลด... (อาจใช้เวลา 1-2 นาทีในการเรียกใช้ครั้งแรก)", "failed_to_load": "โหลดไม่สำเร็จ", "not_loaded_yet": "ยังไม่ได้โหลด — จะโหลดเมื่อถอดเสียงเป็นคำครั้งแรก", + "tts_not_loaded_yet": "ยังไม่ได้โหลด — จะโหลดเมื่อสร้างเสียงครั้งแรก", "error_check_logs": "ข้อผิดพลาด: {{error}} ตรวจสอบบันทึกและลองรีสตาร์ท", "check_logs_restart": "ตรวจสอบบันทึกเพื่อหาข้อผิดพลาดในการโหลดโมเดล ลองรีสตาร์ท", "llm_cinematic": "LLM (ภาพยนตร์)", diff --git a/frontend/src/i18n/locales/tr.json b/frontend/src/i18n/locales/tr.json index 29819332..be88ce0c 100644 --- a/frontend/src/i18n/locales/tr.json +++ b/frontend/src/i18n/locales/tr.json @@ -1728,10 +1728,12 @@ "all_ready": "Tüm sistemler hazır", "system_readiness": "Sistem Hazırlığı", "asr_model": "ASR Modeli", + "tts_model": "TTS Modeli", "loaded_ready": "Yüklendi ve hazır", "loading_first_run": "Yükleniyor… (ilk çalıştırmada bu işlem 1-2 dakika sürebilir)", "failed_to_load": "Yüklenemedi", "not_loaded_yet": "Henüz yüklenmedi — ilk transkripsiyonda yüklenecek", + "tts_not_loaded_yet": "Henüz yüklenmedi — ilk ses oluşturmada yüklenecek", "error_check_logs": "Hata: {{error}}. Günlükleri kontrol edin ve yeniden başlatmayı deneyin.", "check_logs_restart": "Model yükleme hataları için günlükleri kontrol edin. Yeniden başlatmayı deneyin.", "llm_cinematic": "Yüksek Lisans (Sinematik)", diff --git a/frontend/src/i18n/locales/uk.json b/frontend/src/i18n/locales/uk.json index 0914763d..fd76eebf 100644 --- a/frontend/src/i18n/locales/uk.json +++ b/frontend/src/i18n/locales/uk.json @@ -1728,10 +1728,12 @@ "all_ready": "Всі системи готові", "system_readiness": "Готовність системи", "asr_model": "Модель ASR", + "tts_model": "Модель TTS", "loaded_ready": "Завантажений і готовий", "loading_first_run": "Завантаження… (це може зайняти 1-2 хвилини під час першого запуску)", "failed_to_load": "Не вдалося завантажити", "not_loaded_yet": "Ще не завантажено — буде завантажено під час першої транскрипції", + "tts_not_loaded_yet": "Ще не завантажена — завантажиться під час першої генерації мовлення", "error_check_logs": "Помилка: {{error}}. Перевірте журнали та спробуйте перезапустити.", "check_logs_restart": "Перевірте журнали на наявність помилок завантаження моделі. Спробуйте перезапустити.", "llm_cinematic": "LLM (кінематографічний)", diff --git a/frontend/src/i18n/locales/vi.json b/frontend/src/i18n/locales/vi.json index b2b17bcc..8d4e1fc0 100644 --- a/frontend/src/i18n/locales/vi.json +++ b/frontend/src/i18n/locales/vi.json @@ -1728,10 +1728,12 @@ "all_ready": "Tất cả các hệ thống đã sẵn sàng", "system_readiness": "Sự sẵn sàng của hệ thống", "asr_model": "Mô hình ASR", + "tts_model": "Mô hình TTS", "loaded_ready": "Đã tải và sẵn sàng", "loading_first_run": "Đang tải… (việc này có thể mất 1-2 phút trong lần chạy đầu tiên)", "failed_to_load": "Không tải được", "not_loaded_yet": "Chưa được tải - sẽ tải vào lần phiên âm đầu tiên", + "tts_not_loaded_yet": "Chưa tải — sẽ tải khi tạo giọng nói lần đầu", "error_check_logs": "Lỗi: {{error}}. Kiểm tra nhật ký và thử khởi động lại.", "check_logs_restart": "Kiểm tra nhật ký để tìm lỗi tải mô hình. Hãy thử khởi động lại.", "llm_cinematic": "LLM (Điện ảnh)", diff --git a/frontend/src/i18n/locales/zh-CN.json b/frontend/src/i18n/locales/zh-CN.json index 5c9d575c..447ef99a 100644 --- a/frontend/src/i18n/locales/zh-CN.json +++ b/frontend/src/i18n/locales/zh-CN.json @@ -1734,10 +1734,12 @@ "all_ready": "所有系统准备就绪", "system_readiness": "系统准备情况", "asr_model": "ASR模型", + "tts_model": "TTS 模型", "loaded_ready": "已加载并准备就绪", "loading_first_run": "正在加载...(第一次运行可能需要 1-2 分钟)", "failed_to_load": "加载失败", "not_loaded_yet": "尚未加载 - 将在第一次转录时加载", + "tts_not_loaded_yet": "尚未加载 — 将在首次语音生成时加载", "error_check_logs": "错误:{{error}}。检查日志并尝试重新启动。", "check_logs_restart": "检查日志中是否有模型加载错误。尝试重新启动。", "llm_cinematic": "LLM(Cinematic 精译)", diff --git a/frontend/src/i18n/locales/zh-TW.json b/frontend/src/i18n/locales/zh-TW.json index 6ced059a..42bd5cfa 100644 --- a/frontend/src/i18n/locales/zh-TW.json +++ b/frontend/src/i18n/locales/zh-TW.json @@ -1728,10 +1728,12 @@ "all_ready": "所有系統準備就緒", "system_readiness": "系統準備狀況", "asr_model": "ASR模型", + "tts_model": "TTS 模型", "loaded_ready": "已載入並準備就緒", "loading_first_run": "正在加載...(第一次運行可能需要 1-2 分鐘)", "failed_to_load": "載入失敗", "not_loaded_yet": "尚未加載 - 將在第一次轉錄時加載", + "tts_not_loaded_yet": "尚未載入 — 將在首次語音生成時載入", "error_check_logs": "錯誤:{{error}}。檢查日誌並嘗試重新啟動。", "check_logs_restart": "檢查日誌中是否有模型載入錯誤。嘗試重新啟動。", "llm_cinematic": "法學碩士(電影)", diff --git a/frontend/src/test/ReadinessChecklist.test.jsx b/frontend/src/test/ReadinessChecklist.test.jsx new file mode 100644 index 00000000..743de02b --- /dev/null +++ b/frontend/src/test/ReadinessChecklist.test.jsx @@ -0,0 +1,29 @@ +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import { describe, expect, it, vi } from 'vitest'; + +const hookState = vi.hoisted(() => ({ + model: { + status: 'idle', + sub_stage: 'failed', + error: 'Close other GPU-heavy apps or unload models, then retry.', + }, + preflight: { checks: [] }, +})); + +vi.mock('../api/hooks', () => ({ + useModelStatus: () => ({ data: hookState.model, isLoading: false }), + usePreflight: () => ({ data: hookState.preflight, isLoading: false }), +})); + +import ReadinessChecklist from '../components/ReadinessChecklist'; + +describe('ReadinessChecklist model attribution', () => { + it('labels /model/status as TTS and never attributes its failure to ASR', () => { + render(); + expect(screen.getByText('TTS Model')).toBeInTheDocument(); + expect(screen.queryByText('ASR Model')).not.toBeInTheDocument(); + expect(screen.getAllByText(/Close other GPU-heavy apps/)).toHaveLength(2); + expect(screen.queryByText(/transcription/i)).not.toBeInTheDocument(); + }); +}); diff --git a/tests/test_broken_dependency_diagnosis_1415.py b/tests/test_broken_dependency_diagnosis_1415.py index f1d0690a..4fccb6cf 100644 --- a/tests/test_broken_dependency_diagnosis_1415.py +++ b/tests/test_broken_dependency_diagnosis_1415.py @@ -206,6 +206,33 @@ def test_a_successful_preload_clears_a_previous_failure(mm, monkeypatch): mm._set_loading("", "") +def test_preload_oom_status_is_actionable_and_does_not_publish_allocator_details( + mm, monkeypatch, +): + import asyncio + + monkeypatch.setattr(mm, "model", None, raising=False) + monkeypatch.setattr(mm, "resolve_omnivoice_checkpoint", lambda: "org/model") + monkeypatch.setattr(mm, "_checkpoint_in_local_cache", lambda *a, **kw: True) + private = ( + "CUDA out of memory. Tried to allocate 1.14 GiB. " + "Process 1031664 has 22.02 GiB memory in use. " + "/home/alice/private/model.safetensors" + ) + + async def _boom(): + raise RuntimeError(private) + + monkeypatch.setattr(mm, "_load_model_with_timeout", _boom) + asyncio.run(mm.preload_model()) + + status = mm.get_model_status() + assert status["sub_stage"] == "error" + assert "Close other GPU-heavy apps" in status["error"] + assert "1031664" not in status["error"] + assert "/home/alice" not in status["error"] + + def test_the_fallback_detail_does_not_leak_a_path(mm, monkeypatch): """If building the classified failure itself fails, what lands on the status must not be the raw exception — those carry absolute paths, i.e. diff --git a/tests/test_dub_batch_engine_selection.py b/tests/test_dub_batch_engine_selection.py index a8656e97..7c98ada0 100644 --- a/tests/test_dub_batch_engine_selection.py +++ b/tests/test_dub_batch_engine_selection.py @@ -204,6 +204,67 @@ def test_dub_generate_fails_fast_for_non_cloning_engine( assert "omnivoice" in detail # names a real alternative +def test_dub_generate_model_load_oom_is_a_sanitized_resource_error( + dub_job_env, monkeypatch, +): + dg, _job = dub_job_env + private = ( + "CUDA out of memory. Tried to allocate 1.14 GiB. " + "Process 1031664 has 22.02 GiB memory in use. " + "/home/alice/private/model.safetensors" + ) + + async def _oom(**_kwargs): + raise RuntimeError(private) + + monkeypatch.setattr(dg, "resolve_generation_backend", _oom) + with pytest.raises(HTTPException) as exc_info: + asyncio.run(dg.dub_generate("jobX", _one_seg_request())) + + assert exc_info.value.status_code == 503 + detail = exc_info.value.detail + assert "Close other GPU-heavy apps" in detail + assert "1031664" not in detail + assert "/home/alice" not in detail + + +def test_dub_generate_retry_oom_stream_is_sanitized( + dub_job_env, fake_registry, monkeypatch, +): + dg, _job = dub_job_env + fake = fake_registry("fake-oom", supports_cloning=True) + monkeypatch.setenv("OMNIVOICE_TTS_BACKEND", "fake-oom") + private = ( + "CUDA out of memory. Process 1031664 has 22.02 GiB in use. " + "/home/alice/private/model.safetensors" + ) + calls = [] + + def _oom(self, text, **kwargs): + calls.append((text, kwargs)) + raise RuntimeError(private) + + monkeypatch.setattr(fake, "generate", _oom) + events = [] + + class _CaptureTaskManager: + def is_cancelled(self, _task_id): + return False + + async def add_task(self, _task_id, _task_type, func, *args, **_kwargs): + async for event in func(*args): + events.append(event) + + monkeypatch.setattr(dg, "task_manager", _CaptureTaskManager()) + asyncio.run(dg.dub_generate("jobX", _one_seg_request())) + + body = "".join(events) + assert len(calls) == 2, "the initial generation and low-step retry must both run" + assert "Close other GPU-heavy apps" in body + assert "1031664" not in body + assert "/home/alice" not in body + + def test_dub_generate_uses_selected_cloning_engine_not_omnivoice( dub_job_env, fake_registry, no_omnivoice_model_manager, monkeypatch, ): diff --git a/tests/test_failure_helper.py b/tests/test_failure_helper.py index 49b88fd8..52f79f5b 100644 --- a/tests/test_failure_helper.py +++ b/tests/test_failure_helper.py @@ -78,6 +78,40 @@ def test_diagnostic_has_context_and_no_secrets(monkeypatch): # ── Classification → docs topic + hint (US1, FR-005) ──────────────────────── +def test_gpu_oom_gets_a_stable_remedy_without_allocator_details(): + private = ( + "CUDA out of memory. Tried to allocate 1.14 GiB. " + "Process 1031664 has 22.02 GiB memory in use. " + "/home/alice/private/model.safetensors" + ) + evt = failure.build_failure(RuntimeError(private), stage="model-preload") + assert evt["docs_topic"] == "GPU_OOM" + assert "Close other GPU-heavy apps" in evt["hint"] + assert "1031664" not in evt["hint"] + assert "/home/alice" not in evt["hint"] + + +def test_gpu_oom_classifier_covers_typed_and_wrapped_failures(): + typed_oom = type("OutOfMemoryError", (RuntimeError,), {}) + try: + try: + raise typed_oom("allocator failed") + except RuntimeError as inner: + raise RuntimeError("model load failed") from inner + except RuntimeError as wrapped: + assert failure.is_gpu_oom(wrapped) + assert failure.is_gpu_oom(RuntimeError("MPS backend out of memory")) + assert not failure.is_gpu_oom(RuntimeError("model load failed")) + + +def test_gpu_oom_classifier_visits_cause_and_context_branches(): + outer = RuntimeError("model load failed") + outer.__cause__ = ValueError("cleanup failed") + outer.__context__ = RuntimeError("HIP out of memory") + + assert failure.is_gpu_oom(outer) + + def test_docs_topic_and_hint_for_known_class(): evt = failure.build_failure( ModuleNotFoundError("No module named 'pkg_resources'"), stage="task"