From 36e7fb12fc2134bc48da9ca478b2b6f3f2d1e5c4 Mon Sep 17 00:00:00 2001 From: Palash Debnath Date: Sat, 13 Jun 2026 16:17:28 +0530 Subject: [PATCH] =?UTF-8?q?feat(longform):=20job=20library=20=E2=80=94=20f?= =?UTF-8?q?inished=20books/stories=20in=20Projects=20(PR=207/8)=20(#417)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surfaces finished Audiobook + Story renders so they're re-downloadable from the Projects view — closing the resume/history loop of the convergence. Backend (new, no migration — reads existing job_store rows): - routers/longform_jobs.py: GET /longform/jobs lists finished audiobook/story jobs newest-first, recovering output/chapters/duration from each job's persisted 'done' SSE event. Pure build_longform_library() over the job_store callables; defensive (skips unparseable jobs, never 500s). Registered in main.py. Frontend: - Projects.jsx: new "Audiobooks" category fed by /longform/jobs; each row opens the rendered file (/audio/) with type/chapters/duration. Offline-safe (empty on fetch failure). en.json keys added. Built via parallel worktree agent; backend tests/test_longform_jobs.py (9) green; 334 frontend tests + build clean. Co-authored-by: Claude Opus 4.8 (1M context) --- backend/api/routers/longform_jobs.py | 158 ++++++++++++++++++++++++ backend/main.py | 2 + frontend/src/i18n/locales/en.json | 3 + frontend/src/pages/Projects.jsx | 35 +++++- tests/test_longform_jobs.py | 176 +++++++++++++++++++++++++++ 5 files changed, 372 insertions(+), 2 deletions(-) create mode 100644 backend/api/routers/longform_jobs.py create mode 100644 tests/test_longform_jobs.py diff --git a/backend/api/routers/longform_jobs.py b/backend/api/routers/longform_jobs.py new file mode 100644 index 00000000..864c4880 --- /dev/null +++ b/backend/api/routers/longform_jobs.py @@ -0,0 +1,158 @@ +"""Longform Job Library (PR 7). + +``GET /longform/jobs`` — list finished Audiobook + Story renders so the user can +re-download them from the Projects view. The render itself (the m4b/mp3) already +landed in ``OUTPUTS_DIR`` and is served at ``/audio/``; here we just +recover, from each finished job's persisted SSE tail, the output filename plus +the chapter count and duration the ``done`` event carried. + +Pure recovery, no synthesis. Defensive by construction: a job whose ``done`` +event is missing or unparseable is skipped, never surfaced and never a 500. + +The work lives in :func:`build_longform_library`, a pure function over the +job-store callables, so it's unit-testable without importing ``main`` (and the +torch graph behind it). +""" +from __future__ import annotations + +import json +import logging +from typing import Callable, Optional + +from fastapi import APIRouter, Query + +logger = logging.getLogger("omnivoice.longform_jobs") +router = APIRouter() + +#: Job types this library surfaces. Both flow through the shared longform +#: renderer (``_render_longform_sse``) and emit the same ``done`` event shape. +_LONGFORM_TYPES = ("audiobook", "story") + + +def _done_payload_from_events(events: list[dict]) -> Optional[dict]: + """Recover the final ``{"type": "done", ...}`` payload from a job's SSE tail. + + Each row's ``payload`` is the JSON the renderer stored via + ``job_store.append_event(job_id, json.dumps(payload))``. We scan newest-first + and return the first parseable ``done`` event. Anything malformed is skipped + — this never raises. + """ + for ev in reversed(events): + raw = ev.get("payload") if isinstance(ev, dict) else None + if not raw or not isinstance(raw, str): + continue + try: + obj = json.loads(raw) + except (ValueError, TypeError): + continue + if isinstance(obj, dict) and obj.get("type") == "done": + return obj + return None + + +def _coerce_int(value, default: int = 0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _coerce_float(value, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +def build_longform_library( + list_jobs: Callable[..., list[dict]], + events_since: Callable[..., list[dict]], + *, + limit: int = 50, +) -> list[dict]: + """Build the newest-first list of finished longform renders. + + Pure over the two job-store callables so tests can pass them directly: + + * ``list_jobs(status="done", limit=...)`` → all done jobs, newest-first. + * ``events_since(job_id)`` → that job's persisted SSE events. + + Returns ``[{job_id, type, title?, output, duration_s, chapters, + created_at}]``. Jobs that aren't a longform type, or whose ``done`` event / + output filename can't be recovered, are silently skipped — the library only + ever lists things the user can actually re-download. + """ + limit = max(1, min(_coerce_int(limit, 50), 500)) + try: + # Over-fetch: non-longform done jobs (dub, etc.) get filtered out below, + # so ask for more rows than the caller's limit to still fill the page. + rows = list_jobs(status="done", limit=limit * 4) + except Exception: + logger.warning("longform library: list_jobs failed", exc_info=True) + return [] + + out: list[dict] = [] + for row in rows or []: + if len(out) >= limit: + break + try: + job_type = row.get("type") + job_id = row.get("id") + if job_type not in _LONGFORM_TYPES or not job_id: + continue + try: + events = events_since(job_id) + except Exception: + logger.warning("longform library: events_since failed for %s", + job_id, exc_info=True) + continue + done = _done_payload_from_events(events or []) + if not done: + continue + output = done.get("output") + if not output or not isinstance(output, str): + continue # nothing to re-download → not worth listing + + item = { + "job_id": job_id, + "type": job_type, + "output": output, + "duration_s": round(_coerce_float(done.get("duration_s")), 2), + "chapters": _coerce_int(done.get("chapters")), + "created_at": row.get("created_at"), + } + # Title is optional — prefer the done event, fall back to job meta. + title = done.get("title") + if not title: + meta_raw = row.get("meta_json") + if isinstance(meta_raw, str) and meta_raw: + try: + meta = json.loads(meta_raw) + if isinstance(meta, dict): + title = meta.get("title") + except (ValueError, TypeError): + title = None + if title: + item["title"] = title + out.append(item) + except Exception: + # Per-row isolation: one bad row never sinks the whole list. + logger.warning("longform library: skipping unparseable job row", + exc_info=True) + continue + return out + + +@router.get("/longform/jobs") +def longform_jobs(limit: int = Query(50, ge=1, le=500)) -> dict: + """Finished Audiobook + Story renders, newest-first, ready to re-download. + + Each item's ``output`` is served at ``/audio/``. Never 500s — on any + backend hiccup it returns an empty list rather than an error. + """ + from core import job_store + + jobs = build_longform_library( + job_store.list_jobs, job_store.events_since, limit=limit, + ) + return {"jobs": jobs} diff --git a/backend/main.py b/backend/main.py index f541e16f..a46bdf2b 100644 --- a/backend/main.py +++ b/backend/main.py @@ -330,6 +330,7 @@ from api.routers import ( marketplace, sonitranslate, audiobook, + longform_jobs, settings as settings_router, # Phase 1 AUTH-03: HF token save/clear/state ) from utils import hf_progress @@ -766,6 +767,7 @@ app.include_router(tts_stream.router) app.include_router(marketplace.router) app.include_router(sonitranslate.router) app.include_router(audiobook.router) +app.include_router(longform_jobs.router) app.include_router(settings_router.router) # Phase 1 AUTH-03 endpoints from api.routers import mcp_bindings as _mcp_bindings_router # noqa: E402 app.include_router(_mcp_bindings_router.router) # Wave 2.2 per-agent voice bindings diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index e418c01e..550364d4 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -810,6 +810,9 @@ "dub_projects": "Dub Projects", "voice_profiles": "Voice Profiles", "transcripts": "Transcripts", + "audiobooks": "Audiobooks", + "audiobook": "Audiobook", + "story": "Story", "history": "History", "exports": "Exports", "search_placeholder": "Search dubs, clones, transcripts, exports…", diff --git a/frontend/src/pages/Projects.jsx b/frontend/src/pages/Projects.jsx index 67055fa0..51311d60 100644 --- a/frontend/src/pages/Projects.jsx +++ b/frontend/src/pages/Projects.jsx @@ -3,8 +3,10 @@ import { copyText } from "../utils/copyText"; import { useTranslation } from 'react-i18next'; import { Search, FolderOpen, Film, Fingerprint, Wand2, Music, Download, - LayoutGrid, List as ListIcon, Clock, FileText, Mic, + LayoutGrid, List as ListIcon, Clock, FileText, Mic, BookMarked, } from 'lucide-react'; +import { apiFetch } from '../api/client'; +import { audioUrl } from '../api/generate'; import './Projects.css'; /** @@ -88,10 +90,25 @@ export default function Projects({ { id: 'dubs', label: t('projects.dub_projects'), Icon: Film }, { id: 'profiles', label: t('projects.voice_profiles'), Icon: Fingerprint }, { id: 'transcripts', label: t('projects.transcripts'), Icon: Mic }, + { id: 'audiobooks', label: t('projects.audiobooks'), Icon: BookMarked }, { id: 'history', label: t('projects.history'), Icon: Music }, { id: 'exports', label: t('projects.exports'), Icon: Download }, ]; + // Finished Audiobook + Story renders (server-side longform library). + const [longformJobs, setLongformJobs] = useState([]); + React.useEffect(() => { + let alive = true; + (async () => { + try { + const res = await apiFetch('/longform/jobs'); + const data = await res.json(); + if (alive) setLongformJobs(data.jobs || []); + } catch { /* offline / no backend — leave empty */ } + })(); + return () => { alive = false; }; + }, []); + // Load transcriptions from localStorage (same source as TranscriptionsPage) const [transcriptions, setTranscriptions] = useState(() => { try { return JSON.parse(localStorage.getItem('omni_transcriptions') || '[]'); } @@ -160,6 +177,20 @@ export default function Projects({ onClick: () => e.path && onRevealExport?.(e.path), }); } + for (const j of longformJobs) { + const mins = j.duration_s ? `${Math.round(j.duration_s / 60)} min` : ''; + list.push({ + type: 'audiobooks', + id: j.job_id, + title: j.title || j.output, + subtitle: [j.type === 'story' ? t('projects.story') : t('projects.audiobook'), + j.chapters ? `${j.chapters} ch` : '', mins].filter(Boolean).join(' · '), + ts: (j.created_at || 0) * 1000, + accent: '#d3869b', + Icon: BookMarked, + onClick: () => j.output && window.open(audioUrl(j.output), '_blank'), + }); + } for (const tr of transcriptions) { list.push({ type: 'transcripts', @@ -176,7 +207,7 @@ export default function Projects({ } list.sort((a, b) => (b.ts || 0) - (a.ts || 0)); return list; - }, [studioProjects, profiles, history, exportHistory, transcriptions, onOpenDub, onOpenProfile, onRevealExport]); + }, [studioProjects, profiles, history, exportHistory, transcriptions, longformJobs, onOpenDub, onOpenProfile, onRevealExport, t]); const counts = useMemo(() => { const c = { all: items.length }; diff --git a/tests/test_longform_jobs.py b/tests/test_longform_jobs.py new file mode 100644 index 00000000..2d7cb1ea --- /dev/null +++ b/tests/test_longform_jobs.py @@ -0,0 +1,176 @@ +"""PR 7 — Longform Job Library. Tests the recovery logic in +``api.routers.longform_jobs.build_longform_library`` against a seeded job_store. + +We call the pure builder (and the route handler) directly — no ``main``/torch +import — seeding the real job_store over its temp DB. +""" +import json +import os +import time + +os.environ.setdefault("OMNIVOICE_DISABLE_FILE_LOG", "1") + +import pytest +from core import job_store +from core.db import init_db + +from api.routers.longform_jobs import ( + build_longform_library, + _done_payload_from_events, + longform_jobs, +) + + +@pytest.fixture(autouse=True) +def _init_db_once(): + init_db() + yield + + +def _uid(prefix: str) -> str: + return f"{prefix}_{int(time.time()*1e6)}_{os.getpid()}" + + +def _seed_done(job_id: str, *, type: str, done_payload: dict | None, + meta: dict | None = None, extra_events: list[dict] | None = None): + """Create a job, append some progress events + a final done event, mark done.""" + job_store.create(job_id, type=type, meta=meta) + job_store.mark_running(job_id) + for ev in (extra_events or []): + job_store.append_event(job_id, json.dumps(ev)) + if done_payload is not None: + job_store.append_event(job_id, json.dumps(done_payload)) + job_store.mark_done(job_id) + + +# ── _done_payload_from_events ──────────────────────────────────────────────── + + +def test_done_payload_recovers_last_done(): + events = [ + {"payload": json.dumps({"type": "started", "chapters": 2})}, + {"payload": json.dumps({"type": "chapter", "index": 0})}, + {"payload": json.dumps({"type": "done", "output": "story_x.m4b", + "chapters": 2, "duration_s": 12.5})}, + ] + done = _done_payload_from_events(events) + assert done is not None + assert done["output"] == "story_x.m4b" + + +def test_done_payload_skips_malformed_json(): + events = [ + {"payload": "not json at all"}, + {"payload": json.dumps({"type": "done", "output": "ok.m4b"})}, + ] + assert _done_payload_from_events(events)["output"] == "ok.m4b" + + +def test_done_payload_none_when_absent(): + events = [{"payload": json.dumps({"type": "chapter"})}] + assert _done_payload_from_events(events) is None + + +# ── build_longform_library ─────────────────────────────────────────────────── + + +def test_library_lists_only_finished_longform(): + ab = _uid("ab") + st = _uid("st") + failed = _uid("fail") + dub = _uid("dub") + + _seed_done(ab, type="audiobook", meta={"title": "My Book"}, + done_payload={"type": "done", "output": f"{ab}.m4b", + "chapters": 3, "duration_s": 100.0}) + _seed_done(st, type="story", + done_payload={"type": "done", "output": f"{st}.m4b", + "chapters": 1, "duration_s": 42.0, "title": "A Tale"}) + # A failed audiobook job — has progress events but never a done event. + job_store.create(failed, type="audiobook") + job_store.mark_running(failed) + job_store.append_event(failed, json.dumps({"type": "chapter", "index": 0})) + job_store.mark_failed(failed, "boom") + # A finished dub job — done, but not a longform type → excluded. + _seed_done(dub, type="dub_generate", + done_payload={"type": "done", "output": f"{dub}.wav"}) + + lib = build_longform_library(job_store.list_jobs, job_store.events_since, limit=50) + by_id = {it["job_id"]: it for it in lib} + + assert ab in by_id + assert st in by_id + assert failed not in by_id # never finished + assert dub not in by_id # wrong type + + assert by_id[ab]["type"] == "audiobook" + assert by_id[ab]["output"] == f"{ab}.m4b" + assert by_id[ab]["chapters"] == 3 + assert by_id[ab]["duration_s"] == 100.0 + assert by_id[ab]["title"] == "My Book" # from job meta + assert by_id[ab]["created_at"] is not None + + assert by_id[st]["title"] == "A Tale" # from done event + assert by_id[st]["chapters"] == 1 + + +def test_library_skips_done_job_without_output(): + bad = _uid("noout") + _seed_done(bad, type="story", + done_payload={"type": "done", "chapters": 2}) # no "output" + lib = build_longform_library(job_store.list_jobs, job_store.events_since, limit=50) + assert bad not in {it["job_id"] for it in lib} + + +def test_library_newest_first(): + older = _uid("old") + newer = _uid("new") + _seed_done(older, type="audiobook", + done_payload={"type": "done", "output": f"{older}.m4b", "chapters": 1}) + time.sleep(0.01) + _seed_done(newer, type="audiobook", + done_payload={"type": "done", "output": f"{newer}.m4b", "chapters": 1}) + + lib = build_longform_library(job_store.list_jobs, job_store.events_since, limit=50) + ids = [it["job_id"] for it in lib if it["job_id"] in (older, newer)] + assert ids.index(newer) < ids.index(older) + + +def test_library_respects_limit(): + seeded = [] + for _ in range(5): + jid = _uid("lim") + _seed_done(jid, type="story", + done_payload={"type": "done", "output": f"{jid}.m4b", "chapters": 1}) + seeded.append(jid) + lib = build_longform_library(job_store.list_jobs, job_store.events_since, limit=2) + assert len(lib) == 2 + + +def test_library_never_raises_on_bad_callables(): + def boom(*a, **k): + raise RuntimeError("db down") + # list_jobs failing → empty list, no exception. + assert build_longform_library(boom, job_store.events_since, limit=10) == [] + # events_since failing for a row → that row skipped, no exception. + jid = _uid("ev") + _seed_done(jid, type="audiobook", + done_payload={"type": "done", "output": f"{jid}.m4b", "chapters": 1}) + + def evboom(*a, **k): + raise RuntimeError("events down") + lib = build_longform_library(job_store.list_jobs, evboom, limit=10) + assert jid not in {it["job_id"] for it in lib} + + +# ── route handler ──────────────────────────────────────────────────────────── + + +def test_route_handler_returns_jobs_envelope(): + jid = _uid("route") + _seed_done(jid, type="audiobook", + done_payload={"type": "done", "output": f"{jid}.m4b", + "chapters": 2, "duration_s": 9.0}) + resp = longform_jobs(limit=50) + assert "jobs" in resp + assert jid in {it["job_id"] for it in resp["jobs"]}