feat(longform): shared render core — loudness, metadata, cover art (PR 1/8) (#408)
First slice of the Stories+Audiobook convergence (spec: docs/specs/2026-06-13-stories-audiobook-maturity.md). Both features will compile to one server-side chapterized renderer; this lands the shared pure builders and wires them behind Audiobook. New `backend/services/longform_render.py` (all pure, unit-tested without ffmpeg/torch): - build_ffmetadata(chapters, global_meta) — FFMETADATA1 with an optional global tag block (title/author→artist/narrator→composer/year→date/genre/description→ comment) + chapter table. - build_loudnorm_filter(preset) — `-af loudnorm` for ACX (~-19 LUFS, -3 dBTP) or podcast (-16 LUFS); off/unknown → None. Opt-in, so default behavior stays platform-identical. - validate_cover_image — jpg/png + 8 MB cap guard. - build_render_cmd — generalizes the m4b mux: m4b|mp3, optional cover (attached_pic) + loudness, bitrate validated. - build_concat_list — moved here. `services/audiobook.py`: build_chapter_ffmetadata / build_m4b_cmd / build_concat_list are now backward-compatible wrappers over the core (existing imports + tests unchanged). `POST /audiobook`: now accepts optional `format` (m4b|mp3), `loudness`, `cover_path`, and `metadata` and passes them through — backend-complete; the UI for these lands in PR 2. Tests: tests/test_longform_render.py (28) + existing test_audiobook.py (11) green. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Palash Debnath
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
f86beb041c
commit
e9481ef307
@@ -24,12 +24,14 @@ from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from services.audiobook import (
|
||||
build_chapter_ffmetadata,
|
||||
build_concat_list,
|
||||
build_m4b_cmd,
|
||||
parse_audiobook_script,
|
||||
synthesize_chapter,
|
||||
)
|
||||
from services.longform_render import (
|
||||
build_concat_list,
|
||||
build_ffmetadata,
|
||||
build_render_cmd,
|
||||
)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -50,6 +52,12 @@ class AudiobookRequest(BaseModel):
|
||||
text: str
|
||||
default_voice: str | None = None # voice profile id; None = engine default
|
||||
bitrate: str = "128k"
|
||||
format: str = "m4b" # "m4b" | "mp3"
|
||||
loudness: str | None = None # None/"off" | "acx" | "podcast" (opt-in)
|
||||
cover_path: str | None = None # server-side path to a jpg/png cover
|
||||
# Global tags embedded in the output: {title, author, narrator, year,
|
||||
# genre, description}. Player-visible (Apple Books / Audible read these).
|
||||
metadata: dict | None = None
|
||||
|
||||
|
||||
def _resolve_voice(profile_id: str | None) -> dict:
|
||||
@@ -201,14 +209,19 @@ async def audiobook_synthesize(req: AudiobookRequest):
|
||||
yield _emit({"type": "assembling"})
|
||||
meta_path = os.path.join(work, "chapters.ffmeta")
|
||||
with open(meta_path, "w", encoding="utf-8") as f:
|
||||
f.write(build_chapter_ffmetadata(chapters_meta))
|
||||
f.write(build_ffmetadata(chapters_meta, global_meta=req.metadata))
|
||||
concat_path = os.path.join(work, "concat.txt")
|
||||
with open(concat_path, "w", encoding="utf-8") as f:
|
||||
f.write(build_concat_list(chapter_files))
|
||||
out_name = f"audiobook_{job_id}.m4b"
|
||||
ext = "mp3" if (req.format or "").lower() == "mp3" else "m4b"
|
||||
out_name = f"audiobook_{job_id}.{ext}"
|
||||
out_path = os.path.join(OUTPUTS_DIR, out_name)
|
||||
await run_ffmpeg(
|
||||
build_m4b_cmd(ffmpeg, concat_path, meta_path, out_path, bitrate=req.bitrate),
|
||||
build_render_cmd(
|
||||
ffmpeg, concat_path, meta_path, out_path,
|
||||
fmt=ext, bitrate=req.bitrate,
|
||||
cover_path=req.cover_path, loudness=req.loudness,
|
||||
),
|
||||
job_id=job_id,
|
||||
)
|
||||
|
||||
|
||||
@@ -40,7 +40,6 @@ _HEADING_RE = re.compile(r"^[ \t]*#[ \t]+(\S.*)$", re.MULTILINE)
|
||||
# ``finditer`` (the source of the polynomial-time ReDoS). A voice name never
|
||||
# contains a bracket; the value is stripped in code.
|
||||
_VOICE_RE = re.compile(r"\[voice:([^\]\[]*)\]")
|
||||
_BITRATE_RE = re.compile(r"^\d{2,3}k$")
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -178,44 +177,23 @@ def synthesize_chapter(
|
||||
return audio, audio.shape[-1] / float(sample_rate)
|
||||
|
||||
|
||||
def _escape_meta(value: str) -> str:
|
||||
"""Escape an FFMETADATA value (``=``, ``;``, ``#``, ``\\``, newline)."""
|
||||
return re.sub(r"([=;#\\\n])", r"\\\1", value or "")
|
||||
# ── ffmpeg / metadata builders ──────────────────────────────────────────────
|
||||
#
|
||||
# These now live in the shared ``longform_render`` core (Stories + Audiobook
|
||||
# converge on one mux). The thin wrappers below preserve the original
|
||||
# audiobook-only call sites/signatures; new callers should use
|
||||
# ``longform_render`` directly to reach global metadata, cover art, loudness,
|
||||
# and mp3 output.
|
||||
from services.longform_render import ( # noqa: E402
|
||||
build_concat_list,
|
||||
build_ffmetadata,
|
||||
build_render_cmd,
|
||||
)
|
||||
|
||||
|
||||
def build_chapter_ffmetadata(chapters: list[tuple[str, int]]) -> str:
|
||||
"""Build an FFMETADATA1 document with one ``[CHAPTER]`` per (title, ms).
|
||||
|
||||
``chapters`` is ordered ``(title, duration_ms)`` pairs; START/END are the
|
||||
cumulative millisecond offsets ffmpeg writes into the m4b chapter table.
|
||||
"""
|
||||
lines = [";FFMETADATA1"]
|
||||
start = 0
|
||||
for title, dur_ms in chapters:
|
||||
end = start + max(0, int(dur_ms))
|
||||
lines += [
|
||||
"[CHAPTER]",
|
||||
"TIMEBASE=1/1000",
|
||||
f"START={start}",
|
||||
f"END={end}",
|
||||
f"title={_escape_meta(title)}",
|
||||
]
|
||||
start = end
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def build_concat_list(wav_paths: list[str]) -> str:
|
||||
"""Build an ffmpeg concat-demuxer list for the chapter WAVs.
|
||||
|
||||
Each line is ``file '<path>'`` with single quotes escaped the ffmpeg way
|
||||
(``'`` → ``'\\''``), so paths with spaces/quotes can't break the list or
|
||||
inject arguments.
|
||||
"""
|
||||
lines = []
|
||||
for p in wav_paths:
|
||||
safe = str(p).replace("'", "'\\''")
|
||||
lines.append(f"file '{safe}'")
|
||||
return "\n".join(lines) + "\n"
|
||||
"""Backward-compatible alias: chapters-only FFMETADATA (no global tags)."""
|
||||
return build_ffmetadata(chapters)
|
||||
|
||||
|
||||
def build_m4b_cmd(
|
||||
@@ -226,17 +204,8 @@ def build_m4b_cmd(
|
||||
*,
|
||||
bitrate: str = "128k",
|
||||
) -> list[str]:
|
||||
"""Pure argv for muxing chapter WAVs + FFMETADATA into a faststart m4b.
|
||||
|
||||
Input 0 is the concat-demuxer list of chapter WAVs; input 1 is the
|
||||
FFMETADATA file (``-map_metadata 1`` pulls the chapter table from it).
|
||||
"""
|
||||
if not _BITRATE_RE.match(bitrate or ""):
|
||||
bitrate = "128k"
|
||||
return [
|
||||
ffmpeg, "-y", "-hide_banner", "-loglevel", "error",
|
||||
"-f", "concat", "-safe", "0", "-i", str(concat_list_path),
|
||||
"-i", str(metadata_path), "-map_metadata", "1",
|
||||
"-c:a", "aac", "-b:a", bitrate,
|
||||
"-movflags", "+faststart", "-f", "mp4", str(out_path),
|
||||
]
|
||||
"""Backward-compatible alias: a chapterized faststart m4b, no cover/loudness."""
|
||||
return build_render_cmd(
|
||||
ffmpeg, concat_list_path, metadata_path, out_path,
|
||||
fmt="m4b", bitrate=bitrate,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
"""Shared long-form render core (Stories + Audiobook convergence).
|
||||
|
||||
Both the Audiobook tab and the Stories Editor produce the *same* artifact: a
|
||||
chapter-marked audio file built from chapter WAVs. This module owns the pure,
|
||||
engine-agnostic ffmpeg/metadata builders for that mux so neither feature has to
|
||||
reimplement it:
|
||||
|
||||
* ``build_ffmetadata`` — FFMETADATA1 doc: an optional ``[global]`` tag block
|
||||
(title / author / narrator / year / genre / description) followed by one
|
||||
``[CHAPTER]`` per (title, duration_ms).
|
||||
* ``build_concat_list`` — ffmpeg concat-demuxer list of chapter WAVs.
|
||||
* ``build_loudnorm_filter`` — an ``-af loudnorm=…`` string for an ACX /
|
||||
podcast loudness preset (off by default — opt-in, so the default-behavior
|
||||
stays platform-identical).
|
||||
* ``validate_cover_image`` — guard a cover path (type + size) before it
|
||||
reaches ffmpeg.
|
||||
* ``build_render_cmd`` — pure argv for the mux: chapter WAVs + FFMETADATA
|
||||
(+ optional cover art, loudness filter), output as ``m4b`` or ``mp3``.
|
||||
|
||||
Every function here is pure (string/argv in, string/argv out) so it's unit
|
||||
tested without ffmpeg, torch, or a GPU. The impure ffmpeg run lives in the
|
||||
caller (the audiobook router today; the stories job tomorrow).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Optional
|
||||
|
||||
_BITRATE_RE = re.compile(r"^\d{2,3}k$")
|
||||
_COVER_EXTS = {".jpg", ".jpeg", ".png"}
|
||||
_COVER_MAX_BYTES = 8 * 1024 * 1024 # 8 MB — a book cover, not a payload
|
||||
|
||||
#: Our metadata field → FFMETADATA tag key. Order is stable for deterministic
|
||||
#: output (tested). ``author`` maps to ``artist`` and ``narrator`` to
|
||||
#: ``composer`` — the tags audiobook players (Apple Books, Audible) read for
|
||||
#: those roles.
|
||||
_GLOBAL_TAG_KEYS: list[tuple[str, str]] = [
|
||||
("title", "title"),
|
||||
("author", "artist"),
|
||||
("album", "album"),
|
||||
("narrator", "composer"),
|
||||
("year", "date"),
|
||||
("genre", "genre"),
|
||||
("description", "comment"),
|
||||
]
|
||||
|
||||
|
||||
def _escape_meta(value: str) -> str:
|
||||
"""Escape an FFMETADATA value (``=``, ``;``, ``#``, ``\\``, newline)."""
|
||||
return re.sub(r"([=;#\\\n])", r"\\\1", value or "")
|
||||
|
||||
|
||||
# ── Loudness normalization ──────────────────────────────────────────────────
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LoudnessPreset:
|
||||
"""A loudnorm target. ``i`` = integrated LUFS, ``tp`` = true-peak ceiling
|
||||
(dBTP), ``lra`` = loudness range."""
|
||||
key: str
|
||||
i: float
|
||||
tp: float
|
||||
lra: float
|
||||
|
||||
|
||||
#: ``acx`` targets Audible/ACX submission (≈ -19 LUFS integrated, ≤ -3 dBTP
|
||||
#: peak — inside ACX's -23…-18 dB RMS / -3 dB peak window). ``podcast`` targets
|
||||
#: the -16 LUFS streaming norm.
|
||||
LOUDNESS_PRESETS: dict[str, LoudnessPreset] = {
|
||||
"acx": LoudnessPreset("acx", -19.0, -3.0, 11.0),
|
||||
"podcast": LoudnessPreset("podcast", -16.0, -1.5, 11.0),
|
||||
}
|
||||
|
||||
|
||||
def build_loudnorm_filter(preset: Optional[str]) -> Optional[str]:
|
||||
"""Return an ``-af`` loudnorm filter string for ``preset``, or ``None`` for
|
||||
off / unknown (single-pass; two-pass measure→apply is a runner enhancement).
|
||||
"""
|
||||
if not preset:
|
||||
return None
|
||||
p = LOUDNESS_PRESETS.get(preset.lower())
|
||||
if p is None: # "off", "none", or anything unrecognized → no filter
|
||||
return None
|
||||
return f"loudnorm=I={p.i}:TP={p.tp}:LRA={p.lra}"
|
||||
|
||||
|
||||
# ── FFMETADATA ──────────────────────────────────────────────────────────────
|
||||
|
||||
def build_ffmetadata(
|
||||
chapters: Iterable[tuple[str, int]],
|
||||
global_meta: Optional[dict] = None,
|
||||
) -> str:
|
||||
"""Build an FFMETADATA1 doc: optional global tags + one ``[CHAPTER]`` per
|
||||
``(title, duration_ms)``. START/END are cumulative millisecond offsets.
|
||||
"""
|
||||
lines = [";FFMETADATA1"]
|
||||
if global_meta:
|
||||
for field_key, meta_key in _GLOBAL_TAG_KEYS:
|
||||
val = global_meta.get(field_key)
|
||||
if val is not None and str(val).strip():
|
||||
lines.append(f"{meta_key}={_escape_meta(str(val).strip())}")
|
||||
start = 0
|
||||
for title, dur_ms in chapters:
|
||||
end = start + max(0, int(dur_ms))
|
||||
lines += [
|
||||
"[CHAPTER]",
|
||||
"TIMEBASE=1/1000",
|
||||
f"START={start}",
|
||||
f"END={end}",
|
||||
f"title={_escape_meta(title)}",
|
||||
]
|
||||
start = end
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def build_concat_list(wav_paths: Iterable[str]) -> str:
|
||||
"""Build an ffmpeg concat-demuxer list. Single quotes in paths are escaped
|
||||
the ffmpeg way (``'`` → ``'\\''``) so paths can't break the list or inject
|
||||
arguments."""
|
||||
lines = []
|
||||
for p in wav_paths:
|
||||
safe = str(p).replace("'", "'\\''")
|
||||
lines.append(f"file '{safe}'")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# ── Cover art ───────────────────────────────────────────────────────────────
|
||||
|
||||
def validate_cover_image(path: Optional[str]) -> bool:
|
||||
"""True if ``path`` is a readable jpg/png within the size cap. Anything
|
||||
dubious (missing, wrong type, too big, unreadable) → False, and the caller
|
||||
simply omits the cover rather than failing the render."""
|
||||
if not path:
|
||||
return False
|
||||
try:
|
||||
p = Path(path)
|
||||
return (
|
||||
p.is_file()
|
||||
and p.suffix.lower() in _COVER_EXTS
|
||||
and 0 < p.stat().st_size <= _COVER_MAX_BYTES
|
||||
)
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
# ── Render command ──────────────────────────────────────────────────────────
|
||||
|
||||
def build_render_cmd(
|
||||
ffmpeg: str,
|
||||
concat_list_path: str,
|
||||
metadata_path: str,
|
||||
out_path: str,
|
||||
*,
|
||||
fmt: str = "m4b",
|
||||
bitrate: str = "128k",
|
||||
cover_path: Optional[str] = None,
|
||||
loudness: Optional[str] = None,
|
||||
) -> list[str]:
|
||||
"""Pure argv for muxing chapter WAVs + FFMETADATA into a tagged,
|
||||
chapter-marked audio file.
|
||||
|
||||
Inputs: 0 = concat-demuxer list of chapter WAVs, 1 = FFMETADATA (chapters +
|
||||
global tags), 2 = cover image (only when present + valid). ``fmt`` is
|
||||
``m4b`` (AAC in mp4, faststart) or ``mp3`` (libmp3lame). A loudness preset
|
||||
adds an ``-af loudnorm`` pass; an invalid/oversized cover is silently
|
||||
dropped (see :func:`validate_cover_image`).
|
||||
"""
|
||||
if not _BITRATE_RE.match(bitrate or ""):
|
||||
bitrate = "128k"
|
||||
is_mp3 = (fmt or "").lower() == "mp3"
|
||||
have_cover = validate_cover_image(cover_path)
|
||||
|
||||
cmd = [
|
||||
ffmpeg, "-y", "-hide_banner", "-loglevel", "error",
|
||||
"-f", "concat", "-safe", "0", "-i", str(concat_list_path),
|
||||
"-i", str(metadata_path),
|
||||
]
|
||||
if have_cover:
|
||||
cmd += ["-i", str(cover_path)]
|
||||
|
||||
cmd += ["-map", "0:a", "-map_metadata", "1"]
|
||||
if have_cover:
|
||||
cmd += ["-map", "2:v", "-disposition:v", "attached_pic"]
|
||||
|
||||
filt = build_loudnorm_filter(loudness)
|
||||
if filt:
|
||||
cmd += ["-af", filt]
|
||||
|
||||
if is_mp3:
|
||||
cmd += ["-c:a", "libmp3lame", "-b:a", bitrate]
|
||||
if have_cover:
|
||||
cmd += ["-c:v", "copy", "-id3v2_version", "3"]
|
||||
cmd += ["-f", "mp3", str(out_path)]
|
||||
else: # m4b — AAC in an mp4 container
|
||||
cmd += ["-c:a", "aac", "-b:a", bitrate]
|
||||
if have_cover:
|
||||
cmd += ["-c:v", "copy"]
|
||||
cmd += ["-movflags", "+faststart", "-f", "mp4", str(out_path)]
|
||||
return cmd
|
||||
@@ -0,0 +1,164 @@
|
||||
# Stories Editor & Audiobook — Maturity Spec
|
||||
|
||||
> Compiled 2026-06-13. Targets the v0.3.x continuous-to-main line. Grounds every
|
||||
> work item in the two features' current implementation (file:line refs below).
|
||||
|
||||
## 0. TL;DR — the central decision
|
||||
|
||||
Stories Editor and Audiobook are **two authoring frontends over the same job**:
|
||||
*chapterized long-form TTS → a chapter-marked audio file*. Today they're built
|
||||
on divergent stacks, and that divergence is the root cause of most maturity gaps:
|
||||
|
||||
| | Stories Editor | Audiobook |
|
||||
|---|---|---|
|
||||
| Render | **Client-side** Web Audio stitch (`decodeAudioData` per chunk, concat in RAM) | **Server-side** GPU pool → ffmpeg `.m4b` |
|
||||
| State | localStorage only (`omnivoice.app`) | best-effort `job_store`, discarded on exit |
|
||||
| Strength | multi-character dialogue, per-line voice/speed, auto-cast | chapter markers, SSE progress, scales to long books |
|
||||
| Ceiling | can't render a 10-hour book (browser RAM, no resume, no loudness norm) | single textarea, no cover/metadata, no preview/retry/resume |
|
||||
|
||||
**Recommendation: converge on one server-side chapterized render core; keep two
|
||||
distinct authoring frontends.** Stories Editor's "Generate full" compiles its
|
||||
cast/lines into the same `Chapter[]`/`Span[]` model the Audiobook backend already
|
||||
uses and submits the same job. This kills the duplication, lifts the browser
|
||||
scale ceiling off Stories, and gives *both* features resume, loudness
|
||||
normalization, cover art, and metadata for free.
|
||||
|
||||
Stories keeps its fast **client-side single-line preview** (low latency, no job
|
||||
overhead). Only the full export moves server-side.
|
||||
|
||||
> **Alternative considered:** keep them fully separate and mature each stack
|
||||
> independently. Rejected — it doubles the work (two metadata systems, two
|
||||
> normalization paths, two resume mechanisms) and leaves Stories permanently
|
||||
> unable to render book-length output. If the owner wants them kept separate,
|
||||
> only §2 + §3-frontend apply and §1 is dropped.
|
||||
|
||||
---
|
||||
|
||||
## 1. Shared render core (new) — backend
|
||||
|
||||
**New module `backend/services/longform_render.py`**, generalizing the audiobook
|
||||
pipeline (which already has the right shape).
|
||||
|
||||
- **Canonical job model** (already exists in `backend/services/audiobook.py:47-86`):
|
||||
`Project → Chapter(title, spans) → Span(voice_id, text, pause_ms_after)`.
|
||||
Promote it here; Audiobook and Stories both compile to it.
|
||||
- **Reuse** `synthesize_chapter` (audiobook.py:140), `build_chapter_ffmetadata`
|
||||
(:186), `build_concat_list` (:207), `build_m4b_cmd` (:221). Generalize
|
||||
`build_*_cmd` to take metadata + cover + format.
|
||||
- **Add capabilities** (each is a discrete, testable pure builder + a wiring step):
|
||||
1. **Global metadata** → FFMETADATA `[global]`: `title`, `artist` (author),
|
||||
`album`, `composer`/`narrator`, `date` (year), `genre`, `comment`.
|
||||
2. **Cover art** → ffmpeg `-i cover.jpg -map 2 -disposition:v attached_pic`
|
||||
(MP4 `COVR`). Validate image (size/type) server-side.
|
||||
3. **Loudness normalization** → two-pass ffmpeg `loudnorm` to an **ACX preset**
|
||||
(target ≈ -19…-21 LUFS integrated, ≤ -3 dB true peak, noise floor < -60 dB).
|
||||
Toggle + preset (`ACX` / `Podcast -16 LUFS` / `Off`).
|
||||
4. **Per-chapter checkpoint / resume**: cache each rendered chapter WAV keyed
|
||||
by `hash(span_texts + voice_ids + pauses + engine + voice refs)`. On retry,
|
||||
skip chapters whose hash already has a cached WAV → resume a failed/long job
|
||||
without re-rendering completed chapters.
|
||||
5. **Parallel chapter synth**: submit chapters to the existing `_gpu_pool`
|
||||
concurrently (bounded by pool size) instead of strictly sequential.
|
||||
6. **Output formats**: `m4b` (chaptered AAC, current), `mp3` (chaptered via
|
||||
ID3 CHAP frames), `per-chapter files` (zip), `stems` (one track per voice —
|
||||
Stories' existing stems concept, server-side).
|
||||
- **Job persistence / library**: new alembic table `longform_jobs`
|
||||
(id, kind=`audiobook`|`story`, title, status, output_path, chapters json,
|
||||
metadata json, created_at). A finished book/story reappears in Projects/Library
|
||||
and is re-downloadable. Backward-compatible migration (additive table).
|
||||
|
||||
---
|
||||
|
||||
## 2. Audiobook tab — maturity (frontend + thin backend)
|
||||
|
||||
Anchors: `frontend/src/pages/AudiobookTab.jsx`, `frontend/src/api/audiobook.ts`,
|
||||
`backend/api/routers/audiobook.py`, `backend/services/audiobook.py`.
|
||||
|
||||
**P0**
|
||||
- **Metadata panel**: title, author, narrator, year, genre, description, **cover
|
||||
image** picker. Pipes to §1.1/§1.2.
|
||||
- **Per-chapter preview**: render a single chapter (new `POST /audiobook/preview`
|
||||
reusing `synthesize_chapter`) so users audition before committing the full book.
|
||||
- **Chapter-level retry**: a failed chapter doesn't kill the job; mark it, let the
|
||||
user re-run just that chapter (uses §1.4 checkpoints).
|
||||
- **Resume**: reconnect to / restart an interrupted job, skipping cached chapters.
|
||||
- **Loudness-norm toggle** (ACX preset default-off; explicit opt-in).
|
||||
|
||||
**P1**
|
||||
- **Import → auto-chapter**: `.txt` (split on `# H1` / `^Chapter \d+`), **EPUB**
|
||||
(spine + TOC → chapters; local parse, no network), drag-drop file.
|
||||
- **Pronunciation lexicon**: per-project word→phoneme/respelling overrides applied
|
||||
pre-synthesis (shared with Stories).
|
||||
- **ETA + richer progress** (chapters done / total, elapsed, est. remaining).
|
||||
|
||||
**P2**
|
||||
- SSML-lite (`[emphasis]`, `[slow]`/`[fast]`, `[spell]`) mapped to engine instruct
|
||||
+ chunk rate. Batch (queue multiple books).
|
||||
|
||||
---
|
||||
|
||||
## 3. Stories Editor — maturity (frontend + backend)
|
||||
|
||||
Anchors: `frontend/src/components/StoriesEditor.jsx` (749-line monolith),
|
||||
`frontend/src/store/storiesSlice.ts`, `frontend/src/utils/storyExport.js`,
|
||||
`frontend/src/utils/parseScript.js`.
|
||||
|
||||
**P0**
|
||||
- **Move "Generate full" to the shared server-side job** (§1). Add a
|
||||
`storyToSpans()` compiler: cast + lines + `[voice:]`/`[pause]` markers →
|
||||
`Chapter[]`/`Span[]`. `storyExport.js` shrinks to **preview-only** (keep the
|
||||
snappy single-line client playback at `StoriesEditor.jsx:271-322`).
|
||||
- **Per-line regenerate** (currently only full export exists).
|
||||
- **Abort/cancel** an in-progress export (today the loop has no abort signal).
|
||||
- **Wire the emotion/tone field → `instruct`** (`storiesSlice.ts:17` "Phase 3"
|
||||
stub; tone chips insert text markers but never reach synthesis).
|
||||
|
||||
**P1**
|
||||
- **Component split**: `CastPanel.jsx` / `StoryLine.jsx` / `LineDrawer.jsx`
|
||||
(already flagged in `docs/superpowers/specs/2026-05-30-stories-editor-studio-design.md`).
|
||||
- **Per-line waveform + duration** thumbnail.
|
||||
- **Chaptered M4B export** with markers (reuse §1; Stories already detects
|
||||
`# ` chapter lines at `StoriesEditor.jsx:46`).
|
||||
- **Optional server-persisted projects** (DB) so large casts/long scripts survive
|
||||
localStorage limits; lazy-migrate existing localStorage projects.
|
||||
|
||||
**P2**
|
||||
- EPUB/screenplay import polish; richer auto-cast (gender/voice matching).
|
||||
|
||||
---
|
||||
|
||||
## 4. Cross-cutting
|
||||
|
||||
- **Loudness norm, metadata+cover, resume/checkpoint, parallel synth** all live in
|
||||
§1 and are consumed by both features — built once.
|
||||
- **Pronunciation lexicon** shared service (used by Audiobook P1 + Stories).
|
||||
- **DB**: `longform_jobs` (+ optional `story_projects`) via alembic, additive,
|
||||
tested upgrade path. Existing `omnivoice_data/` and localStorage untouched.
|
||||
- **i18n**: new keys under `audiobook.*` and `stories.*`; no hardcoded CJK.
|
||||
- **Tests**: extend `tests/test_audiobook.py` — loudnorm argv (two-pass shape),
|
||||
global-metadata + cover ffmeta/argv, `storyToSpans()` compiler, resume-skip
|
||||
(cached chapter hash → skipped), EPUB→chapter parse. Frontend: metadata panel,
|
||||
per-line regen, abort.
|
||||
|
||||
## 5. Constraints honored
|
||||
|
||||
- **Cross-platform parity**: ffmpeg `loudnorm` / `COVR` / `aac` work identically on
|
||||
mac/Win/Linux (ffmpeg already bundled). Loudness toggle ships **off by default**
|
||||
→ no default-behavior divergence. EPUB parse is local.
|
||||
- **Local-first**: no cloud, no accounts; all rendering/import on-device.
|
||||
- **Backward-compat data**: alembic additive tables; localStorage projects migrate
|
||||
lazily, never break.
|
||||
- **Versioning**: ships as continuous-to-main patches on the v0.3.x line; no RCs,
|
||||
no new minor unless the owner asks.
|
||||
|
||||
## 6. Suggested PR slicing (order, each independently shippable)
|
||||
|
||||
1. **Shared `longform_render` core** + loudnorm + global-metadata/cover builders
|
||||
(backend only, wired behind Audiobook). Pure builders + tests first.
|
||||
2. **Audiobook metadata + cover UI**.
|
||||
3. **Per-chapter preview + retry + resume** (checkpoints).
|
||||
4. **Text/EPUB import + auto-chapter**.
|
||||
5. **Stories "Generate" → shared job**; `storyToSpans()`; per-line regen; emotion→instruct.
|
||||
6. **Stories component split + per-line waveform**; chaptered M4B.
|
||||
7. **DB job library** (Projects/Library shows finished books/stories).
|
||||
8. (P2) SSML-lite, batch, pronunciation lexicon UI.
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Shared long-form render core (Stories + Audiobook convergence).
|
||||
|
||||
Pure builders for the chapterized mux: FFMETADATA (global tags + chapters),
|
||||
concat list, loudness filter, cover validation, and the ffmpeg render argv.
|
||||
All unit-testable without ffmpeg/torch/GPU.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from services.longform_render import (
|
||||
LOUDNESS_PRESETS,
|
||||
build_concat_list,
|
||||
build_ffmetadata,
|
||||
build_loudnorm_filter,
|
||||
build_render_cmd,
|
||||
validate_cover_image,
|
||||
)
|
||||
|
||||
|
||||
# ── loudness ────────────────────────────────────────────────────────────────
|
||||
|
||||
def test_loudnorm_acx_filter():
|
||||
f = build_loudnorm_filter("acx")
|
||||
assert f == "loudnorm=I=-19.0:TP=-3.0:LRA=11.0"
|
||||
|
||||
|
||||
def test_loudnorm_podcast_filter():
|
||||
assert build_loudnorm_filter("podcast") == "loudnorm=I=-16.0:TP=-1.5:LRA=11.0"
|
||||
|
||||
|
||||
def test_loudnorm_case_insensitive():
|
||||
assert build_loudnorm_filter("ACX") == build_loudnorm_filter("acx")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("val", [None, "", "off", "none", "bogus"])
|
||||
def test_loudnorm_off_or_unknown_is_none(val):
|
||||
assert build_loudnorm_filter(val) is None
|
||||
|
||||
|
||||
def test_loudness_presets_within_acx_window():
|
||||
# ACX wants integrated near -19 LUFS and a -3 dB peak ceiling.
|
||||
acx = LOUDNESS_PRESETS["acx"]
|
||||
assert -23.0 <= acx.i <= -18.0
|
||||
assert acx.tp == -3.0
|
||||
|
||||
|
||||
# ── FFMETADATA ──────────────────────────────────────────────────────────────
|
||||
|
||||
def test_ffmetadata_chapters_only_matches_legacy_shape():
|
||||
doc = build_ffmetadata([("One", 1000), ("Two", 500)])
|
||||
assert doc.startswith(";FFMETADATA1\n")
|
||||
assert "[CHAPTER]\nTIMEBASE=1/1000\nSTART=0\nEND=1000\ntitle=One" in doc
|
||||
assert "START=1000\nEND=1500\ntitle=Two" in doc
|
||||
# No global tags when none supplied.
|
||||
assert "artist=" not in doc
|
||||
|
||||
|
||||
def test_ffmetadata_global_tags_mapped_and_ordered():
|
||||
doc = build_ffmetadata(
|
||||
[("Ch", 1000)],
|
||||
global_meta={
|
||||
"title": "My Book", "author": "Ada", "narrator": "Grace",
|
||||
"year": "2026", "genre": "Sci-Fi", "description": "A tale",
|
||||
},
|
||||
)
|
||||
# field → tag mapping (author→artist, narrator→composer, year→date,
|
||||
# description→comment) and stable order (title before artist).
|
||||
head = doc.split("[CHAPTER]")[0]
|
||||
assert head.index("title=My Book") < head.index("artist=Ada")
|
||||
assert "composer=Grace" in head
|
||||
assert "date=2026" in head
|
||||
assert "genre=Sci-Fi" in head
|
||||
assert "comment=A tale" in head
|
||||
|
||||
|
||||
def test_ffmetadata_skips_empty_global_values():
|
||||
doc = build_ffmetadata([("Ch", 1)], global_meta={"title": "T", "author": " ", "genre": None})
|
||||
head = doc.split("[CHAPTER]")[0]
|
||||
assert "title=T" in head
|
||||
assert "artist=" not in head # whitespace-only dropped
|
||||
assert "genre=" not in head # None dropped
|
||||
|
||||
|
||||
def test_ffmetadata_escapes_special_chars():
|
||||
doc = build_ffmetadata([("a=b;c#d", 100)], global_meta={"title": "x=y"})
|
||||
assert r"title=x\=y" in doc
|
||||
assert r"title=a\=b\;c\#d" in doc
|
||||
|
||||
|
||||
# ── concat list ─────────────────────────────────────────────────────────────
|
||||
|
||||
def test_concat_list_quotes_and_escapes():
|
||||
out = build_concat_list(["/a/one.wav", "/weird/it's here.wav"])
|
||||
assert "file '/a/one.wav'" in out
|
||||
assert "file '/weird/it'\\''s here.wav'" in out
|
||||
|
||||
|
||||
# ── cover validation ────────────────────────────────────────────────────────
|
||||
|
||||
def test_cover_valid(tmp_path):
|
||||
p = tmp_path / "cover.jpg"
|
||||
p.write_bytes(b"\xff\xd8\xff" + b"x" * 100)
|
||||
assert validate_cover_image(str(p)) is True
|
||||
|
||||
|
||||
def test_cover_rejects_missing_and_bad_type(tmp_path):
|
||||
assert validate_cover_image(None) is False
|
||||
assert validate_cover_image(str(tmp_path / "nope.jpg")) is False
|
||||
txt = tmp_path / "c.txt"
|
||||
txt.write_bytes(b"hi")
|
||||
assert validate_cover_image(str(txt)) is False
|
||||
|
||||
|
||||
def test_cover_rejects_oversize(tmp_path):
|
||||
big = tmp_path / "big.png"
|
||||
big.write_bytes(b"\x89PNG" + b"0" * (8 * 1024 * 1024 + 1))
|
||||
assert validate_cover_image(str(big)) is False
|
||||
|
||||
|
||||
def test_cover_rejects_empty(tmp_path):
|
||||
empty = tmp_path / "empty.jpg"
|
||||
empty.write_bytes(b"")
|
||||
assert validate_cover_image(str(empty)) is False
|
||||
|
||||
|
||||
# ── render command ──────────────────────────────────────────────────────────
|
||||
|
||||
def test_render_cmd_m4b_default():
|
||||
cmd = build_render_cmd("ffmpeg", "concat.txt", "ch.ffmeta", "out.m4b")
|
||||
assert cmd[0] == "ffmpeg"
|
||||
assert "-f" in cmd and "concat" in cmd
|
||||
assert cmd[-3:] == ["-f", "mp4", "out.m4b"]
|
||||
assert "-c:a" in cmd and "aac" in cmd
|
||||
assert "+faststart" in cmd
|
||||
assert "-map_metadata" in cmd
|
||||
# no cover, no loudnorm by default
|
||||
assert "attached_pic" not in cmd
|
||||
assert "-af" not in cmd
|
||||
|
||||
|
||||
def test_render_cmd_mp3_format():
|
||||
cmd = build_render_cmd("ffmpeg", "c.txt", "m.ffmeta", "out.mp3", fmt="mp3")
|
||||
assert "libmp3lame" in cmd
|
||||
assert cmd[-3:] == ["-f", "mp3", "out.mp3"]
|
||||
assert "+faststart" not in cmd
|
||||
|
||||
|
||||
def test_render_cmd_bitrate_validation():
|
||||
ok = build_render_cmd("ffmpeg", "c", "m", "o", bitrate="192k")
|
||||
assert "192k" in ok
|
||||
bad = build_render_cmd("ffmpeg", "c", "m", "o", bitrate="; rm -rf /")
|
||||
assert "128k" in bad # rejected → default
|
||||
assert "; rm -rf /" not in bad
|
||||
|
||||
|
||||
def test_render_cmd_loudnorm_adds_af():
|
||||
cmd = build_render_cmd("ffmpeg", "c", "m", "o", loudness="acx")
|
||||
assert "-af" in cmd
|
||||
assert any(a.startswith("loudnorm=") for a in cmd)
|
||||
|
||||
|
||||
def test_render_cmd_with_cover(tmp_path):
|
||||
cover = tmp_path / "cover.jpg"
|
||||
cover.write_bytes(b"\xff\xd8\xff" + b"x" * 50)
|
||||
cmd = build_render_cmd("ffmpeg", "c", "m", "o.m4b", cover_path=str(cover))
|
||||
# cover becomes input 2, mapped as attached_pic, copied
|
||||
assert str(cover) in cmd
|
||||
assert "-map" in cmd and "2:v" in cmd
|
||||
assert "attached_pic" in cmd
|
||||
assert "-c:v" in cmd and "copy" in cmd
|
||||
|
||||
|
||||
def test_render_cmd_drops_invalid_cover(tmp_path):
|
||||
cmd = build_render_cmd("ffmpeg", "c", "m", "o.m4b", cover_path=str(tmp_path / "missing.jpg"))
|
||||
assert "attached_pic" not in cmd # silently dropped, render still proceeds
|
||||
Reference in New Issue
Block a user