fix(models): handle load OOMs safely (#1696)

* fix(models): handle load OOMs safely (#1695)

* fix(dub): sanitize streamed generation failures

* style(ui): format readiness checklist
This commit is contained in:
Palash Debnath
2026-08-28 16:00:24 +05:30
committed by GitHub
parent 3e5aa90e3f
commit de51120d6a
30 changed files with 268 additions and 13 deletions
+1
View File
@@ -10,6 +10,7 @@ the frozen-backend fallback mirror it for their toolchains.
**Highlights** **Highlights**
- Model-load GPU exhaustion now returns a sanitized, actionable dubbing error, and readiness correctly attributes the shared model status to TTS (#1695)
- Source-mode development now restarts an isolated backend crash without tearing down the UI, while repeated crash loops still stop loudly with diagnostics (#1690) - Source-mode development now restarts an isolated backend crash without tearing down the UI, while repeated crash loops still stop loudly with diagnostics (#1690)
- Dubbing playback now keeps an audible companion source when a WebView can render the preview picture but cannot decode its audio (#1692) - Dubbing playback now keeps an audible companion source when a WebView can render the preview picture but cannot decode its audio (#1692)
- Model Catalogue engine rows now use the available desktop width and keep identity, runtime state, and actions from crowding one another (#1689) - Model Catalogue engine rows now use the available desktop width and keep identity, runtime state, and actions from crowding one another (#1689)
+21 -1
View File
@@ -503,6 +503,18 @@ async def dub_generate(job_id: str, req: DubRequest):
backend = await resolve_generation_backend(require_cloning=True) backend = await resolve_generation_backend(require_cloning=True)
except ValueError as e: except ValueError as e:
raise HTTPException(status_code=400, detail=str(e)) raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
from core.failure import is_gpu_oom
if not is_gpu_oom(e):
raise
from core.public_errors import public_exception_response
payload = public_exception_response(
e,
fallback="The TTS model could not be loaded.",
)
raise HTTPException(status_code=503, detail=payload["detail"]) from e
async def _stream(task_id): async def _stream(task_id):
total = len(req.segments) total = len(req.segments)
@@ -1291,7 +1303,15 @@ async def dub_generate(job_id: str, req: DubRequest):
pass pass
_release_audio_tensors() _release_audio_tensors()
except Exception as e: except Exception as e:
yield f"data: {json.dumps({'type': 'error', 'segment': i, 'error': str(e)})}\n\n" # A task-stream error bypasses the global exception handler.
# Never publish engine exception text here: allocator errors
# carry process tables and arbitrary failures can carry paths,
# tokens, or source text. The shared helper enriches recognized
# classes using VoiceStudio-owned constants only.
from core.public_errors import stream_generation_failure
error_detail = stream_generation_failure(e)["detail"]
yield f"data: {json.dumps({'type': 'error', 'segment': i, 'error': error_detail})}\n\n"
sr = backend.sample_rate sr = backend.sample_rate
all_segment_wavs.append(_store_mix_wav(seg.start, seg.end, torch.zeros(1, max(0, int(seg_duration * sr))), sr, f"mix_{seg_id}")) all_segment_wavs.append(_store_mix_wav(seg.start, seg.end, torch.zeros(1, max(0, int(seg_duration * sr))), sr, f"mix_{seg_id}"))
sync_scores.append(1.0) sync_scores.append(1.0)
+38
View File
@@ -52,6 +52,7 @@ _REDACTED_VALUE = "***REDACTED***"
# One-line "what to do" per docs-taxonomy key. Keys mirror error_docs_map's # One-line "what to do" per docs-taxonomy key. Keys mirror error_docs_map's
# taxonomy; the docs URL itself stays owned by error_docs_map. # taxonomy; the docs URL itself stays owned by error_docs_map.
_HINTS: dict[str, str] = { _HINTS: dict[str, str] = {
"GPU_OOM": "Close other GPU-heavy apps or unload models, then retry. You can also choose CPU in Settings → Performance & Device or select a smaller TTS engine.",
"WORKER_AT_CAPACITY": "Wait for a running job on that worker to finish, or choose another available worker and retry.", "WORKER_AT_CAPACITY": "Wait for a running job on that worker to finish, or choose another available worker and retry.",
"MODEL_NOT_INSTALLED": "Install or enable this engine on the worker machine, then refresh its capabilities and retry.", "MODEL_NOT_INSTALLED": "Install or enable this engine on the worker machine, then refresh its capabilities and retry.",
"MODEL_NOT_DOWNLOADED": "Open Models, install this model on the selected worker, then retry when the download completes.", "MODEL_NOT_DOWNLOADED": "Open Models, install this model on the selected worker, then retry when the download completes.",
@@ -290,6 +291,9 @@ def append_hf_mirror_hint(text: str) -> str:
# must NOT be added: its bare "timed out" trigger would stamp a "video server" # must NOT be added: its bare "timed out" trigger would stamp a "video server"
# hint on a model-load timeout that leaks through the 500 handler. # hint on a model-load timeout that leaks through the 500 handler.
_CONTEXT_FREE_HINT_CLASSES = frozenset({ _CONTEXT_FREE_HINT_CLASSES = frozenset({
# Device allocator signatures are specific enough to attach the shared
# recovery without exposing CUDA's process table or filesystem paths.
"GPU_OOM",
"SOCKS_PROXY_SUPPORT_MISSING", "SOCKS_PROXY_SUPPORT_MISSING",
"SSL_HANDSHAKE_FAILURE", "SSL_HANDSHAKE_FAILURE",
# Its trigger is an exact OpenSSL string, so it cannot be confused with # Its trigger is an exact OpenSSL string, so it cannot be confused with
@@ -323,6 +327,38 @@ def append_hint(text: str) -> str:
return f"{text}{hint}" if hint else text return f"{text}{hint}" if hint else text
_GPU_OOM_SIGNATURES = (
"cuda out of memory",
"cuda error: out of memory",
"cuda_error_out_of_memory",
"mps backend out of memory",
"hip out of memory",
"out of memory on device",
)
def is_gpu_oom(error: BaseException | str) -> bool:
"""Recognize device OOMs through wrappers without importing torch."""
pending: list[BaseException] = [error] if isinstance(error, BaseException) else []
seen: set[int] = set()
while pending:
current = pending.pop()
if id(current) in seen:
continue
seen.add(id(current))
if type(current).__name__ == "OutOfMemoryError":
return True
if any(signature in str(current).lower() for signature in _GPU_OOM_SIGNATURES):
return True
if current.__cause__ is not None:
pending.append(current.__cause__)
if current.__context__ is not None:
pending.append(current.__context__)
if isinstance(error, str):
return any(signature in error.lower() for signature in _GPU_OOM_SIGNATURES)
return False
def classify(reason: str) -> str: def classify(reason: str) -> str:
"""Map a failure reason to a docs-taxonomy key, or "" when unknown. """Map a failure reason to a docs-taxonomy key, or "" when unknown.
@@ -330,6 +366,8 @@ def classify(reason: str) -> str:
backend log / diagnostic names the same class the UI deeplink will use. backend log / diagnostic names the same class the UI deeplink will use.
""" """
low = (reason or "").lower() low = (reason or "").lower()
if is_gpu_oom(low):
return "GPU_OOM"
if "pkg_resources" in low: if "pkg_resources" in low:
return "PKG_RESOURCES_MISSING" return "PKG_RESOURCES_MISSING"
if "quarantine" in low or "is damaged" in low or "gatekeeper" in low: if "quarantine" in low or "is damaged" in low or "gatekeeper" in low:
+4 -1
View File
@@ -2938,7 +2938,10 @@ async def preload_model():
"The TTS model could not be loaded. Settings → Logs → Backend " "The TTS model could not be loaded. Settings → Logs → Backend "
"has the full error." "has the full error."
) )
_set_loading("failed", detail, error=detail) # `sub_stage` is a public API enum and the frontend keys failure state
# off `error`. Keep the human-readable word "failed" in the detail,
# not in the state machine (#1695).
_set_loading("error", detail, error=detail)
def get_model_status(): def get_model_status():
is_loaded = model is not None is_loaded = model is not None
+11 -11
View File
@@ -37,6 +37,7 @@ export default function ReadinessChecklist({ compact = false, showWhenAllPass =
const isLoading = preflightLoading || modelLoading; const isLoading = preflightLoading || modelLoading;
const modelStatus = modelData?.status ?? 'idle'; const modelStatus = modelData?.status ?? 'idle';
const modelFailed = modelStatus === 'error' || ['error', 'failed'].includes(modelData?.sub_stage);
// Build the checklist from preflight data + model status // Build the checklist from preflight data + model status
const checks = []; const checks = [];
@@ -45,14 +46,14 @@ export default function ReadinessChecklist({ compact = false, showWhenAllPass =
const modelDetail = modelData?.detail || ''; const modelDetail = modelData?.detail || '';
const modelErr = modelData?.error || null; const modelErr = modelData?.error || null;
const modelCheck = { const modelCheck = {
id: 'asr-model', id: 'tts-model',
label: t('readiness.asr_model'), label: t('readiness.tts_model'),
status: status:
modelStatus === 'ready' modelStatus === 'ready'
? 'pass' ? 'pass'
: modelStatus === 'loading' : modelStatus === 'loading'
? 'loading' ? 'loading'
: modelStatus === 'error' || modelData?.sub_stage === 'error' : modelFailed
? 'fail' ? 'fail'
: 'warn', : 'warn',
detail: detail:
@@ -60,15 +61,14 @@ export default function ReadinessChecklist({ compact = false, showWhenAllPass =
? t('readiness.loaded_ready') ? t('readiness.loaded_ready')
: modelStatus === 'loading' : modelStatus === 'loading'
? modelDetail || t('readiness.loading_first_run') ? modelDetail || t('readiness.loading_first_run')
: modelData?.sub_stage === 'error' : modelFailed
? modelErr || t('readiness.failed_to_load') ? modelErr || t('readiness.failed_to_load')
: t('readiness.not_loaded_yet'), : t('readiness.tts_not_loaded_yet'),
fix: fix: modelFailed
modelStatus === 'error' || modelData?.sub_stage === 'error' ? modelErr
? modelErr ? t('readiness.error_check_logs', { error: modelErr })
? t('readiness.error_check_logs', { error: modelErr }) : t('readiness.check_logs_restart')
: t('readiness.check_logs_restart') : null,
: null,
}; };
checks.push(modelCheck); checks.push(modelCheck);
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "جميع الأنظمة جاهزة", "all_ready": "جميع الأنظمة جاهزة",
"system_readiness": "جاهزية النظام", "system_readiness": "جاهزية النظام",
"asr_model": "نموذج ASR", "asr_model": "نموذج ASR",
"tts_model": "نموذج TTS",
"loaded_ready": "محملة وجاهزة", "loaded_ready": "محملة وجاهزة",
"loading_first_run": "جارٍ التحميل... (قد يستغرق هذا من دقيقة إلى دقيقتين عند التشغيل لأول مرة)", "loading_first_run": "جارٍ التحميل... (قد يستغرق هذا من دقيقة إلى دقيقتين عند التشغيل لأول مرة)",
"failed_to_load": "فشل التحميل", "failed_to_load": "فشل التحميل",
"not_loaded_yet": "لم يتم تحميله بعد - سيتم تحميله عند النسخ الأول", "not_loaded_yet": "لم يتم تحميله بعد - سيتم تحميله عند النسخ الأول",
"tts_not_loaded_yet": "لم يتم تحميله بعد — سيُحمّل عند أول توليد صوتي",
"error_check_logs": "خطأ: {{error}}. تحقق من السجلات وحاول إعادة التشغيل.", "error_check_logs": "خطأ: {{error}}. تحقق من السجلات وحاول إعادة التشغيل.",
"check_logs_restart": "تحقق من السجلات بحثًا عن أخطاء تحميل النموذج. حاول إعادة التشغيل.", "check_logs_restart": "تحقق من السجلات بحثًا عن أخطاء تحميل النموذج. حاول إعادة التشغيل.",
"llm_cinematic": "ماجستير في القانون (السينمائي)", "llm_cinematic": "ماجستير في القانون (السينمائي)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "Alle Systeme bereit", "all_ready": "Alle Systeme bereit",
"system_readiness": "Systembereitschaft", "system_readiness": "Systembereitschaft",
"asr_model": "ASR-Modell", "asr_model": "ASR-Modell",
"tts_model": "TTS-Modell",
"loaded_ready": "Geladen und fertig", "loaded_ready": "Geladen und fertig",
"loading_first_run": "Wird geladen… (dies kann beim ersten Durchlauf 1-2 Minuten dauern)", "loading_first_run": "Wird geladen… (dies kann beim ersten Durchlauf 1-2 Minuten dauern)",
"failed_to_load": "Laden fehlgeschlagen", "failed_to_load": "Laden fehlgeschlagen",
"not_loaded_yet": "Noch nicht geladen wird bei der ersten Transkription geladen", "not_loaded_yet": "Noch nicht geladen wird bei der ersten Transkription geladen",
"tts_not_loaded_yet": "Noch nicht geladen wird bei der ersten Spracherzeugung geladen",
"error_check_logs": "Fehler: {{error}}. Überprüfen Sie die Protokolle und versuchen Sie einen Neustart.", "error_check_logs": "Fehler: {{error}}. Überprüfen Sie die Protokolle und versuchen Sie einen Neustart.",
"check_logs_restart": "Überprüfen Sie die Protokolle auf Modellladefehler. Versuchen Sie einen Neustart.", "check_logs_restart": "Überprüfen Sie die Protokolle auf Modellladefehler. Versuchen Sie einen Neustart.",
"llm_cinematic": "LLM (Film)", "llm_cinematic": "LLM (Film)",
+2
View File
@@ -2261,10 +2261,12 @@
"all_ready": "All systems ready", "all_ready": "All systems ready",
"system_readiness": "System Readiness", "system_readiness": "System Readiness",
"asr_model": "ASR Model", "asr_model": "ASR Model",
"tts_model": "TTS Model",
"loaded_ready": "Loaded and ready", "loaded_ready": "Loaded and ready",
"loading_first_run": "Loading… (this may take 1-2 minutes on first run)", "loading_first_run": "Loading… (this may take 1-2 minutes on first run)",
"failed_to_load": "Failed to load", "failed_to_load": "Failed to load",
"not_loaded_yet": "Not loaded yet — will load on first transcription", "not_loaded_yet": "Not loaded yet — will load on first transcription",
"tts_not_loaded_yet": "Not loaded yet — will load on first speech generation",
"error_check_logs": "Error: {{error}}. Check logs and try restarting.", "error_check_logs": "Error: {{error}}. Check logs and try restarting.",
"check_logs_restart": "Check logs for model loading errors. Try restarting.", "check_logs_restart": "Check logs for model loading errors. Try restarting.",
"llm_cinematic": "LLM (Cinematic)", "llm_cinematic": "LLM (Cinematic)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "Todos los sistemas listos", "all_ready": "Todos los sistemas listos",
"system_readiness": "Preparación del sistema", "system_readiness": "Preparación del sistema",
"asr_model": "Modelo ASR", "asr_model": "Modelo ASR",
"tts_model": "Modelo TTS",
"loaded_ready": "Cargado y listo", "loaded_ready": "Cargado y listo",
"loading_first_run": "Cargando... (esto puede tardar entre 1 y 2 minutos en la primera ejecución)", "loading_first_run": "Cargando... (esto puede tardar entre 1 y 2 minutos en la primera ejecución)",
"failed_to_load": "No se pudo cargar", "failed_to_load": "No se pudo cargar",
"not_loaded_yet": "Aún no cargado: se cargará en la primera transcripción", "not_loaded_yet": "Aún no cargado: se cargará en la primera transcripción",
"tts_not_loaded_yet": "Aún no está cargado; se cargará al generar voz por primera vez",
"error_check_logs": "Error: {{error}}. Verifique los registros e intente reiniciar.", "error_check_logs": "Error: {{error}}. Verifique los registros e intente reiniciar.",
"check_logs_restart": "Verifique los registros para detectar errores de carga del modelo. Intenta reiniciar.", "check_logs_restart": "Verifique los registros para detectar errores de carga del modelo. Intenta reiniciar.",
"llm_cinematic": "LLM (Cinemático)", "llm_cinematic": "LLM (Cinemático)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "Tous les systèmes sont prêts", "all_ready": "Tous les systèmes sont prêts",
"system_readiness": "État de préparation du système", "system_readiness": "État de préparation du système",
"asr_model": "Modèle ASR", "asr_model": "Modèle ASR",
"tts_model": "Modèle TTS",
"loaded_ready": "Chargé et prêt", "loaded_ready": "Chargé et prêt",
"loading_first_run": "Chargement… (cela peut prendre 1 à 2 minutes lors de la première exécution)", "loading_first_run": "Chargement… (cela peut prendre 1 à 2 minutes lors de la première exécution)",
"failed_to_load": "Échec du chargement", "failed_to_load": "Échec du chargement",
"not_loaded_yet": "Pas encore chargé  se chargera lors de la première transcription", "not_loaded_yet": "Pas encore chargé  se chargera lors de la première transcription",
"tts_not_loaded_yet": "Pas encore chargé — se chargera lors de la première génération vocale",
"error_check_logs": "Erreur : {{error}}. Vérifiez les journaux et essayez de redémarrer.", "error_check_logs": "Erreur : {{error}}. Vérifiez les journaux et essayez de redémarrer.",
"check_logs_restart": "Vérifiez les journaux pour détecter les erreurs de chargement du modèle. Essayez de redémarrer.", "check_logs_restart": "Vérifiez les journaux pour détecter les erreurs de chargement du modèle. Essayez de redémarrer.",
"llm_cinematic": "LLM (Cinématique)", "llm_cinematic": "LLM (Cinématique)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "सभी सिस्टम तैयार", "all_ready": "सभी सिस्टम तैयार",
"system_readiness": "सिस्टम की तैयारी", "system_readiness": "सिस्टम की तैयारी",
"asr_model": "एएसआर मॉडल", "asr_model": "एएसआर मॉडल",
"tts_model": "टीटीएस मॉडल",
"loaded_ready": "लोड और तैयार", "loaded_ready": "लोड और तैयार",
"loading_first_run": "लोड हो रहा है... (पहली बार चलाने में 1-2 मिनट लग सकते हैं)", "loading_first_run": "लोड हो रहा है... (पहली बार चलाने में 1-2 मिनट लग सकते हैं)",
"failed_to_load": "लोड करने में विफल", "failed_to_load": "लोड करने में विफल",
"not_loaded_yet": "अभी तक लोड नहीं हुआ है - प्रथम प्रतिलेखन पर लोड होगा", "not_loaded_yet": "अभी तक लोड नहीं हुआ है - प्रथम प्रतिलेखन पर लोड होगा",
"tts_not_loaded_yet": "अभी लोड नहीं हुआ — पहली बार आवाज़ बनाने पर लोड होगा",
"error_check_logs": "त्रुटि: {{error}}. लॉग जांचें और पुनः प्रारंभ करने का प्रयास करें।", "error_check_logs": "त्रुटि: {{error}}. लॉग जांचें और पुनः प्रारंभ करने का प्रयास करें।",
"check_logs_restart": "मॉडल लोडिंग त्रुटियों के लिए लॉग की जाँच करें। पुनः आरंभ करने का प्रयास करें.", "check_logs_restart": "मॉडल लोडिंग त्रुटियों के लिए लॉग की जाँच करें। पुनः आरंभ करने का प्रयास करें.",
"llm_cinematic": "एलएलएम (सिनेमाई)", "llm_cinematic": "एलएलएम (सिनेमाई)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "Semua sistem siap", "all_ready": "Semua sistem siap",
"system_readiness": "Kesiapan Sistem", "system_readiness": "Kesiapan Sistem",
"asr_model": "Model ASR", "asr_model": "Model ASR",
"tts_model": "Model TTS",
"loaded_ready": "Sudah terisi dan siap", "loaded_ready": "Sudah terisi dan siap",
"loading_first_run": "Memuat… (ini mungkin memakan waktu 1-2 menit saat pertama kali dijalankan)", "loading_first_run": "Memuat… (ini mungkin memakan waktu 1-2 menit saat pertama kali dijalankan)",
"failed_to_load": "Gagal memuat", "failed_to_load": "Gagal memuat",
"not_loaded_yet": "Belum dimuat — akan dimuat pada transkripsi pertama", "not_loaded_yet": "Belum dimuat — akan dimuat pada transkripsi pertama",
"tts_not_loaded_yet": "Belum dimuat — akan dimuat saat pertama kali menghasilkan suara",
"error_check_logs": "Kesalahan: {{error}}. Periksa log dan coba mulai ulang.", "error_check_logs": "Kesalahan: {{error}}. Periksa log dan coba mulai ulang.",
"check_logs_restart": "Periksa log untuk mengetahui kesalahan pemuatan model. Coba mulai ulang.", "check_logs_restart": "Periksa log untuk mengetahui kesalahan pemuatan model. Coba mulai ulang.",
"llm_cinematic": "LLM (Sinematik)", "llm_cinematic": "LLM (Sinematik)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "Tutti i sistemi pronti", "all_ready": "Tutti i sistemi pronti",
"system_readiness": "Prontezza del sistema", "system_readiness": "Prontezza del sistema",
"asr_model": "Modello ASR", "asr_model": "Modello ASR",
"tts_model": "Modello TTS",
"loaded_ready": "Carico e pronto", "loaded_ready": "Carico e pronto",
"loading_first_run": "Caricamento... (l'operazione potrebbe richiedere 1-2 minuti alla prima esecuzione)", "loading_first_run": "Caricamento... (l'operazione potrebbe richiedere 1-2 minuti alla prima esecuzione)",
"failed_to_load": "Impossibile caricare", "failed_to_load": "Impossibile caricare",
"not_loaded_yet": "Non ancora caricato: verrà caricato alla prima trascrizione", "not_loaded_yet": "Non ancora caricato: verrà caricato alla prima trascrizione",
"tts_not_loaded_yet": "Non ancora caricato: verrà caricato alla prima generazione vocale",
"error_check_logs": "Errore: {{error}}. Controlla i log e prova a riavviare.", "error_check_logs": "Errore: {{error}}. Controlla i log e prova a riavviare.",
"check_logs_restart": "Controlla i log per eventuali errori di caricamento del modello. Prova a riavviare.", "check_logs_restart": "Controlla i log per eventuali errori di caricamento del modello. Prova a riavviare.",
"llm_cinematic": "LLM (Cinematologico)", "llm_cinematic": "LLM (Cinematologico)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "すべてのシステムが準備完了", "all_ready": "すべてのシステムが準備完了",
"system_readiness": "システムの準備状況", "system_readiness": "システムの準備状況",
"asr_model": "ASRモデル", "asr_model": "ASRモデル",
"tts_model": "TTSモデル",
"loaded_ready": "ロードされて準備完了", "loaded_ready": "ロードされて準備完了",
"loading_first_run": "読み込み中… (初回実行時は 1 ~ 2 分かかる場合があります)", "loading_first_run": "読み込み中… (初回実行時は 1 ~ 2 分かかる場合があります)",
"failed_to_load": "ロードに失敗しました", "failed_to_load": "ロードに失敗しました",
"not_loaded_yet": "まだロードされていません - 最初の文字起こし時にロードされます", "not_loaded_yet": "まだロードされていません - 最初の文字起こし時にロードされます",
"tts_not_loaded_yet": "未ロード — 初回の音声生成時にロードされます",
"error_check_logs": "エラー: {{error}}。ログを確認して再起動してみてください。", "error_check_logs": "エラー: {{error}}。ログを確認して再起動してみてください。",
"check_logs_restart": "モデルの読み込みエラーがないかログを確認します。再起動してみてください。", "check_logs_restart": "モデルの読み込みエラーがないかログを確認します。再起動してみてください。",
"llm_cinematic": "LLM (映画)", "llm_cinematic": "LLM (映画)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "모든 시스템 준비", "all_ready": "모든 시스템 준비",
"system_readiness": "시스템 준비", "system_readiness": "시스템 준비",
"asr_model": "ASR 모델", "asr_model": "ASR 모델",
"tts_model": "TTS 모델",
"loaded_ready": "로드 및 준비됨", "loaded_ready": "로드 및 준비됨",
"loading_first_run": "로드 중… (처음 실행 시 1~2분 정도 소요될 수 있음)", "loading_first_run": "로드 중… (처음 실행 시 1~2분 정도 소요될 수 있음)",
"failed_to_load": "로드하지 못했습니다.", "failed_to_load": "로드하지 못했습니다.",
"not_loaded_yet": "아직 로드되지 않음 - 첫 번째 기록 시 로드됩니다.", "not_loaded_yet": "아직 로드되지 않음 - 첫 번째 기록 시 로드됩니다.",
"tts_not_loaded_yet": "아직 로드되지 않음 — 첫 음성 생성 시 로드됩니다",
"error_check_logs": "오류: {{error}}. 로그를 확인하고 다시 시작해 보세요.", "error_check_logs": "오류: {{error}}. 로그를 확인하고 다시 시작해 보세요.",
"check_logs_restart": "모델 로드 오류에 대한 로그를 확인하세요. 다시 시작해 보세요.", "check_logs_restart": "모델 로드 오류에 대한 로그를 확인하세요. 다시 시작해 보세요.",
"llm_cinematic": "LLM (영화)", "llm_cinematic": "LLM (영화)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "Alle systemen klaar", "all_ready": "Alle systemen klaar",
"system_readiness": "Systeemgereedheid", "system_readiness": "Systeemgereedheid",
"asr_model": "ASR-model", "asr_model": "ASR-model",
"tts_model": "TTS-model",
"loaded_ready": "Geladen en klaar", "loaded_ready": "Geladen en klaar",
"loading_first_run": "Laden... (dit kan bij de eerste run 1-2 minuten duren)", "loading_first_run": "Laden... (dit kan bij de eerste run 1-2 minuten duren)",
"failed_to_load": "Kan niet laden", "failed_to_load": "Kan niet laden",
"not_loaded_yet": "Nog niet geladen: wordt geladen bij de eerste transcriptie", "not_loaded_yet": "Nog niet geladen: wordt geladen bij de eerste transcriptie",
"tts_not_loaded_yet": "Nog niet geladen — wordt geladen bij de eerste spraakgeneratie",
"error_check_logs": "Fout: {{error}}. Controleer de logboeken en probeer opnieuw op te starten.", "error_check_logs": "Fout: {{error}}. Controleer de logboeken en probeer opnieuw op te starten.",
"check_logs_restart": "Controleer logboeken op fouten bij het laden van modellen. Probeer opnieuw op te starten.", "check_logs_restart": "Controleer logboeken op fouten bij het laden van modellen. Probeer opnieuw op te starten.",
"llm_cinematic": "LLM (filmisch)", "llm_cinematic": "LLM (filmisch)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "Wszystkie systemy gotowe", "all_ready": "Wszystkie systemy gotowe",
"system_readiness": "Gotowość systemu", "system_readiness": "Gotowość systemu",
"asr_model": "Model ASR", "asr_model": "Model ASR",
"tts_model": "Model TTS",
"loaded_ready": "Załadowany i gotowy", "loaded_ready": "Załadowany i gotowy",
"loading_first_run": "Ładowanie… (przy pierwszym uruchomieniu może to zająć 12 minuty)", "loading_first_run": "Ładowanie… (przy pierwszym uruchomieniu może to zająć 12 minuty)",
"failed_to_load": "Nie udało się załadować", "failed_to_load": "Nie udało się załadować",
"not_loaded_yet": "Jeszcze nie załadowano — zostanie załadowane przy pierwszej transkrypcji", "not_loaded_yet": "Jeszcze nie załadowano — zostanie załadowane przy pierwszej transkrypcji",
"tts_not_loaded_yet": "Jeszcze nie załadowano — zostanie załadowany przy pierwszym generowaniu mowy",
"error_check_logs": "Błąd: {{error}}. Sprawdź dzienniki i spróbuj uruchomić ponownie.", "error_check_logs": "Błąd: {{error}}. Sprawdź dzienniki i spróbuj uruchomić ponownie.",
"check_logs_restart": "Sprawdź dzienniki pod kątem błędów ładowania modelu. Spróbuj uruchomić ponownie.", "check_logs_restart": "Sprawdź dzienniki pod kątem błędów ładowania modelu. Spróbuj uruchomić ponownie.",
"llm_cinematic": "LLM (film)", "llm_cinematic": "LLM (film)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "Todos os sistemas prontos", "all_ready": "Todos os sistemas prontos",
"system_readiness": "Preparação do sistema", "system_readiness": "Preparação do sistema",
"asr_model": "Modelo ASR", "asr_model": "Modelo ASR",
"tts_model": "Modelo TTS",
"loaded_ready": "Carregado e pronto", "loaded_ready": "Carregado e pronto",
"loading_first_run": "Carregando… (isso pode levar de 1 a 2 minutos na primeira execução)", "loading_first_run": "Carregando… (isso pode levar de 1 a 2 minutos na primeira execução)",
"failed_to_load": "Falha ao carregar", "failed_to_load": "Falha ao carregar",
"not_loaded_yet": "Ainda não carregado — será carregado na primeira transcrição", "not_loaded_yet": "Ainda não carregado — será carregado na primeira transcrição",
"tts_not_loaded_yet": "Ainda não carregado — será carregado na primeira geração de voz",
"error_check_logs": "Erro: {{error}}. Verifique os logs e tente reiniciar.", "error_check_logs": "Erro: {{error}}. Verifique os logs e tente reiniciar.",
"check_logs_restart": "Verifique os logs para erros de carregamento do modelo. Tente reiniciar.", "check_logs_restart": "Verifique os logs para erros de carregamento do modelo. Tente reiniciar.",
"llm_cinematic": "LLM (Cinemático)", "llm_cinematic": "LLM (Cinemático)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "Все системы готовы", "all_ready": "Все системы готовы",
"system_readiness": "Готовность системы", "system_readiness": "Готовность системы",
"asr_model": "Модель ASR", "asr_model": "Модель ASR",
"tts_model": "Модель TTS",
"loaded_ready": "Загружено и готово", "loaded_ready": "Загружено и готово",
"loading_first_run": "Загрузка… (при первом запуске это может занять 1–2 минуты)", "loading_first_run": "Загрузка… (при первом запуске это может занять 1–2 минуты)",
"failed_to_load": "Не удалось загрузить", "failed_to_load": "Не удалось загрузить",
"not_loaded_yet": "Еще не загружено — загрузится при первой транскрипции", "not_loaded_yet": "Еще не загружено — загрузится при первой транскрипции",
"tts_not_loaded_yet": "Ещё не загружена — загрузится при первой генерации речи",
"error_check_logs": "Ошибка: {{error}}. Проверьте журналы и попробуйте перезагрузить компьютер.", "error_check_logs": "Ошибка: {{error}}. Проверьте журналы и попробуйте перезагрузить компьютер.",
"check_logs_restart": "Проверьте журналы на предмет ошибок загрузки модели. Попробуйте перезапустить.", "check_logs_restart": "Проверьте журналы на предмет ошибок загрузки модели. Попробуйте перезапустить.",
"llm_cinematic": "LLM (кинематографический)", "llm_cinematic": "LLM (кинематографический)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "Alla system redo", "all_ready": "Alla system redo",
"system_readiness": "Systemberedskap", "system_readiness": "Systemberedskap",
"asr_model": "ASR modell", "asr_model": "ASR modell",
"tts_model": "TTS-modell",
"loaded_ready": "Laddat och klart", "loaded_ready": "Laddat och klart",
"loading_first_run": "Laddar... (detta kan ta 1-2 minuter vid första körningen)", "loading_first_run": "Laddar... (detta kan ta 1-2 minuter vid första körningen)",
"failed_to_load": "Det gick inte att ladda", "failed_to_load": "Det gick inte att ladda",
"not_loaded_yet": "Inte laddad än kommer att laddas vid första transkription", "not_loaded_yet": "Inte laddad än kommer att laddas vid första transkription",
"tts_not_loaded_yet": "Inte inläst ännu läses in vid den första röstgenereringen",
"error_check_logs": "Fel: {{error}}. Kontrollera loggar och försök starta om.", "error_check_logs": "Fel: {{error}}. Kontrollera loggar och försök starta om.",
"check_logs_restart": "Kontrollera loggar för modellladdningsfel. Testa att starta om.", "check_logs_restart": "Kontrollera loggar för modellladdningsfel. Testa att starta om.",
"llm_cinematic": "LLM (Cinematic)", "llm_cinematic": "LLM (Cinematic)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "พร้อมทุกระบบ", "all_ready": "พร้อมทุกระบบ",
"system_readiness": "ความพร้อมของระบบ", "system_readiness": "ความพร้อมของระบบ",
"asr_model": "รุ่น ASR", "asr_model": "รุ่น ASR",
"tts_model": "โมเดล TTS",
"loaded_ready": "โหลดแล้วพร้อมครับ", "loaded_ready": "โหลดแล้วพร้อมครับ",
"loading_first_run": "กำลังโหลด... (อาจใช้เวลา 1-2 นาทีในการเรียกใช้ครั้งแรก)", "loading_first_run": "กำลังโหลด... (อาจใช้เวลา 1-2 นาทีในการเรียกใช้ครั้งแรก)",
"failed_to_load": "โหลดไม่สำเร็จ", "failed_to_load": "โหลดไม่สำเร็จ",
"not_loaded_yet": "ยังไม่ได้โหลด — จะโหลดเมื่อถอดเสียงเป็นคำครั้งแรก", "not_loaded_yet": "ยังไม่ได้โหลด — จะโหลดเมื่อถอดเสียงเป็นคำครั้งแรก",
"tts_not_loaded_yet": "ยังไม่ได้โหลด — จะโหลดเมื่อสร้างเสียงครั้งแรก",
"error_check_logs": "ข้อผิดพลาด: {{error}} ตรวจสอบบันทึกและลองรีสตาร์ท", "error_check_logs": "ข้อผิดพลาด: {{error}} ตรวจสอบบันทึกและลองรีสตาร์ท",
"check_logs_restart": "ตรวจสอบบันทึกเพื่อหาข้อผิดพลาดในการโหลดโมเดล ลองรีสตาร์ท", "check_logs_restart": "ตรวจสอบบันทึกเพื่อหาข้อผิดพลาดในการโหลดโมเดล ลองรีสตาร์ท",
"llm_cinematic": "LLM (ภาพยนตร์)", "llm_cinematic": "LLM (ภาพยนตร์)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "Tüm sistemler hazır", "all_ready": "Tüm sistemler hazır",
"system_readiness": "Sistem Hazırlığı", "system_readiness": "Sistem Hazırlığı",
"asr_model": "ASR Modeli", "asr_model": "ASR Modeli",
"tts_model": "TTS Modeli",
"loaded_ready": "Yüklendi ve hazır", "loaded_ready": "Yüklendi ve hazır",
"loading_first_run": "Yükleniyor… (ilk çalıştırmada bu işlem 1-2 dakika sürebilir)", "loading_first_run": "Yükleniyor… (ilk çalıştırmada bu işlem 1-2 dakika sürebilir)",
"failed_to_load": "Yüklenemedi", "failed_to_load": "Yüklenemedi",
"not_loaded_yet": "Henüz yüklenmedi — ilk transkripsiyonda yüklenecek", "not_loaded_yet": "Henüz yüklenmedi — ilk transkripsiyonda yüklenecek",
"tts_not_loaded_yet": "Henüz yüklenmedi — ilk ses oluşturmada yüklenecek",
"error_check_logs": "Hata: {{error}}. Günlükleri kontrol edin ve yeniden başlatmayı deneyin.", "error_check_logs": "Hata: {{error}}. Günlükleri kontrol edin ve yeniden başlatmayı deneyin.",
"check_logs_restart": "Model yükleme hataları için günlükleri kontrol edin. Yeniden başlatmayı deneyin.", "check_logs_restart": "Model yükleme hataları için günlükleri kontrol edin. Yeniden başlatmayı deneyin.",
"llm_cinematic": "Yüksek Lisans (Sinematik)", "llm_cinematic": "Yüksek Lisans (Sinematik)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "Всі системи готові", "all_ready": "Всі системи готові",
"system_readiness": "Готовність системи", "system_readiness": "Готовність системи",
"asr_model": "Модель ASR", "asr_model": "Модель ASR",
"tts_model": "Модель TTS",
"loaded_ready": "Завантажений і готовий", "loaded_ready": "Завантажений і готовий",
"loading_first_run": "Завантаження… (це може зайняти 1-2 хвилини під час першого запуску)", "loading_first_run": "Завантаження… (це може зайняти 1-2 хвилини під час першого запуску)",
"failed_to_load": "Не вдалося завантажити", "failed_to_load": "Не вдалося завантажити",
"not_loaded_yet": "Ще не завантажено — буде завантажено під час першої транскрипції", "not_loaded_yet": "Ще не завантажено — буде завантажено під час першої транскрипції",
"tts_not_loaded_yet": "Ще не завантажена — завантажиться під час першої генерації мовлення",
"error_check_logs": "Помилка: {{error}}. Перевірте журнали та спробуйте перезапустити.", "error_check_logs": "Помилка: {{error}}. Перевірте журнали та спробуйте перезапустити.",
"check_logs_restart": "Перевірте журнали на наявність помилок завантаження моделі. Спробуйте перезапустити.", "check_logs_restart": "Перевірте журнали на наявність помилок завантаження моделі. Спробуйте перезапустити.",
"llm_cinematic": "LLM (кінематографічний)", "llm_cinematic": "LLM (кінематографічний)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "Tất cả các hệ thống đã sẵn sàng", "all_ready": "Tất cả các hệ thống đã sẵn sàng",
"system_readiness": "Sự sẵn sàng của hệ thống", "system_readiness": "Sự sẵn sàng của hệ thống",
"asr_model": "Mô hình ASR", "asr_model": "Mô hình ASR",
"tts_model": "Mô hình TTS",
"loaded_ready": "Đã tải và sẵn sàng", "loaded_ready": "Đã tải và sẵn sàng",
"loading_first_run": "Đang tải… (việc này có thể mất 1-2 phút trong lần chạy đầu tiên)", "loading_first_run": "Đang tải… (việc này có thể mất 1-2 phút trong lần chạy đầu tiên)",
"failed_to_load": "Không tải được", "failed_to_load": "Không tải được",
"not_loaded_yet": "Chưa được tải - sẽ tải vào lần phiên âm đầu tiên", "not_loaded_yet": "Chưa được tải - sẽ tải vào lần phiên âm đầu tiên",
"tts_not_loaded_yet": "Chưa tải — sẽ tải khi tạo giọng nói lần đầu",
"error_check_logs": "Lỗi: {{error}}. Kiểm tra nhật ký và thử khởi động lại.", "error_check_logs": "Lỗi: {{error}}. Kiểm tra nhật ký và thử khởi động lại.",
"check_logs_restart": "Kiểm tra nhật ký để tìm lỗi tải mô hình. Hãy thử khởi động lại.", "check_logs_restart": "Kiểm tra nhật ký để tìm lỗi tải mô hình. Hãy thử khởi động lại.",
"llm_cinematic": "LLM (Điện ảnh)", "llm_cinematic": "LLM (Điện ảnh)",
+2
View File
@@ -1734,10 +1734,12 @@
"all_ready": "所有系统准备就绪", "all_ready": "所有系统准备就绪",
"system_readiness": "系统准备情况", "system_readiness": "系统准备情况",
"asr_model": "ASR模型", "asr_model": "ASR模型",
"tts_model": "TTS 模型",
"loaded_ready": "已加载并准备就绪", "loaded_ready": "已加载并准备就绪",
"loading_first_run": "正在加载...(第一次运行可能需要 1-2 分钟)", "loading_first_run": "正在加载...(第一次运行可能需要 1-2 分钟)",
"failed_to_load": "加载失败", "failed_to_load": "加载失败",
"not_loaded_yet": "尚未加载 - 将在第一次转录时加载", "not_loaded_yet": "尚未加载 - 将在第一次转录时加载",
"tts_not_loaded_yet": "尚未加载 — 将在首次语音生成时加载",
"error_check_logs": "错误:{{error}}。检查日志并尝试重新启动。", "error_check_logs": "错误:{{error}}。检查日志并尝试重新启动。",
"check_logs_restart": "检查日志中是否有模型加载错误。尝试重新启动。", "check_logs_restart": "检查日志中是否有模型加载错误。尝试重新启动。",
"llm_cinematic": "LLMCinematic 精译)", "llm_cinematic": "LLMCinematic 精译)",
+2
View File
@@ -1728,10 +1728,12 @@
"all_ready": "所有系統準備就緒", "all_ready": "所有系統準備就緒",
"system_readiness": "系統準備狀況", "system_readiness": "系統準備狀況",
"asr_model": "ASR模型", "asr_model": "ASR模型",
"tts_model": "TTS 模型",
"loaded_ready": "已載入並準備就緒", "loaded_ready": "已載入並準備就緒",
"loading_first_run": "正在加載...(第一次運行可能需要 1-2 分鐘)", "loading_first_run": "正在加載...(第一次運行可能需要 1-2 分鐘)",
"failed_to_load": "載入失敗", "failed_to_load": "載入失敗",
"not_loaded_yet": "尚未加載 - 將在第一次轉錄時加載", "not_loaded_yet": "尚未加載 - 將在第一次轉錄時加載",
"tts_not_loaded_yet": "尚未載入 — 將在首次語音生成時載入",
"error_check_logs": "錯誤:{{error}}。檢查日誌並嘗試重新啟動。", "error_check_logs": "錯誤:{{error}}。檢查日誌並嘗試重新啟動。",
"check_logs_restart": "檢查日誌中是否有模型載入錯誤。嘗試重新啟動。", "check_logs_restart": "檢查日誌中是否有模型載入錯誤。嘗試重新啟動。",
"llm_cinematic": "法學碩士(電影)", "llm_cinematic": "法學碩士(電影)",
@@ -0,0 +1,29 @@
import React from 'react';
import { render, screen } from '@testing-library/react';
import { describe, expect, it, vi } from 'vitest';
const hookState = vi.hoisted(() => ({
model: {
status: 'idle',
sub_stage: 'failed',
error: 'Close other GPU-heavy apps or unload models, then retry.',
},
preflight: { checks: [] },
}));
vi.mock('../api/hooks', () => ({
useModelStatus: () => ({ data: hookState.model, isLoading: false }),
usePreflight: () => ({ data: hookState.preflight, isLoading: false }),
}));
import ReadinessChecklist from '../components/ReadinessChecklist';
describe('ReadinessChecklist model attribution', () => {
it('labels /model/status as TTS and never attributes its failure to ASR', () => {
render(<ReadinessChecklist showWhenAllPass />);
expect(screen.getByText('TTS Model')).toBeInTheDocument();
expect(screen.queryByText('ASR Model')).not.toBeInTheDocument();
expect(screen.getAllByText(/Close other GPU-heavy apps/)).toHaveLength(2);
expect(screen.queryByText(/transcription/i)).not.toBeInTheDocument();
});
});
@@ -206,6 +206,33 @@ def test_a_successful_preload_clears_a_previous_failure(mm, monkeypatch):
mm._set_loading("", "") mm._set_loading("", "")
def test_preload_oom_status_is_actionable_and_does_not_publish_allocator_details(
mm, monkeypatch,
):
import asyncio
monkeypatch.setattr(mm, "model", None, raising=False)
monkeypatch.setattr(mm, "resolve_omnivoice_checkpoint", lambda: "org/model")
monkeypatch.setattr(mm, "_checkpoint_in_local_cache", lambda *a, **kw: True)
private = (
"CUDA out of memory. Tried to allocate 1.14 GiB. "
"Process 1031664 has 22.02 GiB memory in use. "
"/home/alice/private/model.safetensors"
)
async def _boom():
raise RuntimeError(private)
monkeypatch.setattr(mm, "_load_model_with_timeout", _boom)
asyncio.run(mm.preload_model())
status = mm.get_model_status()
assert status["sub_stage"] == "error"
assert "Close other GPU-heavy apps" in status["error"]
assert "1031664" not in status["error"]
assert "/home/alice" not in status["error"]
def test_the_fallback_detail_does_not_leak_a_path(mm, monkeypatch): def test_the_fallback_detail_does_not_leak_a_path(mm, monkeypatch):
"""If building the classified failure itself fails, what lands on the """If building the classified failure itself fails, what lands on the
status must not be the raw exception those carry absolute paths, i.e. status must not be the raw exception those carry absolute paths, i.e.
+61
View File
@@ -204,6 +204,67 @@ def test_dub_generate_fails_fast_for_non_cloning_engine(
assert "omnivoice" in detail # names a real alternative assert "omnivoice" in detail # names a real alternative
def test_dub_generate_model_load_oom_is_a_sanitized_resource_error(
dub_job_env, monkeypatch,
):
dg, _job = dub_job_env
private = (
"CUDA out of memory. Tried to allocate 1.14 GiB. "
"Process 1031664 has 22.02 GiB memory in use. "
"/home/alice/private/model.safetensors"
)
async def _oom(**_kwargs):
raise RuntimeError(private)
monkeypatch.setattr(dg, "resolve_generation_backend", _oom)
with pytest.raises(HTTPException) as exc_info:
asyncio.run(dg.dub_generate("jobX", _one_seg_request()))
assert exc_info.value.status_code == 503
detail = exc_info.value.detail
assert "Close other GPU-heavy apps" in detail
assert "1031664" not in detail
assert "/home/alice" not in detail
def test_dub_generate_retry_oom_stream_is_sanitized(
dub_job_env, fake_registry, monkeypatch,
):
dg, _job = dub_job_env
fake = fake_registry("fake-oom", supports_cloning=True)
monkeypatch.setenv("OMNIVOICE_TTS_BACKEND", "fake-oom")
private = (
"CUDA out of memory. Process 1031664 has 22.02 GiB in use. "
"/home/alice/private/model.safetensors"
)
calls = []
def _oom(self, text, **kwargs):
calls.append((text, kwargs))
raise RuntimeError(private)
monkeypatch.setattr(fake, "generate", _oom)
events = []
class _CaptureTaskManager:
def is_cancelled(self, _task_id):
return False
async def add_task(self, _task_id, _task_type, func, *args, **_kwargs):
async for event in func(*args):
events.append(event)
monkeypatch.setattr(dg, "task_manager", _CaptureTaskManager())
asyncio.run(dg.dub_generate("jobX", _one_seg_request()))
body = "".join(events)
assert len(calls) == 2, "the initial generation and low-step retry must both run"
assert "Close other GPU-heavy apps" in body
assert "1031664" not in body
assert "/home/alice" not in body
def test_dub_generate_uses_selected_cloning_engine_not_omnivoice( def test_dub_generate_uses_selected_cloning_engine_not_omnivoice(
dub_job_env, fake_registry, no_omnivoice_model_manager, monkeypatch, dub_job_env, fake_registry, no_omnivoice_model_manager, monkeypatch,
): ):
+34
View File
@@ -78,6 +78,40 @@ def test_diagnostic_has_context_and_no_secrets(monkeypatch):
# ── Classification → docs topic + hint (US1, FR-005) ──────────────────────── # ── Classification → docs topic + hint (US1, FR-005) ────────────────────────
def test_gpu_oom_gets_a_stable_remedy_without_allocator_details():
private = (
"CUDA out of memory. Tried to allocate 1.14 GiB. "
"Process 1031664 has 22.02 GiB memory in use. "
"/home/alice/private/model.safetensors"
)
evt = failure.build_failure(RuntimeError(private), stage="model-preload")
assert evt["docs_topic"] == "GPU_OOM"
assert "Close other GPU-heavy apps" in evt["hint"]
assert "1031664" not in evt["hint"]
assert "/home/alice" not in evt["hint"]
def test_gpu_oom_classifier_covers_typed_and_wrapped_failures():
typed_oom = type("OutOfMemoryError", (RuntimeError,), {})
try:
try:
raise typed_oom("allocator failed")
except RuntimeError as inner:
raise RuntimeError("model load failed") from inner
except RuntimeError as wrapped:
assert failure.is_gpu_oom(wrapped)
assert failure.is_gpu_oom(RuntimeError("MPS backend out of memory"))
assert not failure.is_gpu_oom(RuntimeError("model load failed"))
def test_gpu_oom_classifier_visits_cause_and_context_branches():
outer = RuntimeError("model load failed")
outer.__cause__ = ValueError("cleanup failed")
outer.__context__ = RuntimeError("HIP out of memory")
assert failure.is_gpu_oom(outer)
def test_docs_topic_and_hint_for_known_class(): def test_docs_topic_and_hint_for_known_class():
evt = failure.build_failure( evt = failure.build_failure(
ModuleNotFoundError("No module named 'pkg_resources'"), stage="task" ModuleNotFoundError("No module named 'pkg_resources'"), stage="task"