test: backend route-inventory + webUI feature-coverage guards (#609)

* test: backend route-inventory snapshot + webUI feature-coverage guards

A reusable testing system that verifies every feature surface is present:
- tests/test_api_route_inventory.py: boots the app, diffs all 213 routes vs a
  committed snapshot (tests/fixtures/api_routes.txt), guards a critical-endpoint
  set, and floors the route count — any endpoint drift fails CI.
- scripts/dump_api_routes.py: regenerates the snapshot.
- frontend featureCoverage.test.js: every AppMode has a render branch, every
  lazy-imported page file exists, every feature has an i18n namespace.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* docs(changelog): note the feature-coverage test system

* test(api-inventory): isolate via subprocess + exclude env-dependent mounts

CI surfaced two flaws in the first cut:
- the in-process app import + sys.modules purge polluted later DB-touching
  tests (a cascade of 404s in test_dub_subtitles_309 etc.);
- the snapshot included StaticFiles mounts (/demo_audio) and a conditional
  GET / root that register based on filesystem state, so a macOS-generated
  snapshot didn't match a fresh Linux CI runner.

Compute routes in an isolated subprocess (scripts/dump_api_routes.py --print)
and cover only the deterministic router surface (drop Mounts + root). 209
routes; inventory + previously-polluted tests now pass together.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: mergetest <test@local>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Palash Debnath
2026-06-22 05:33:00 +05:30
committed by GitHub
co-authored by Claude Opus 4.8 mergetest
parent 1575baca36
commit de80856cd9
5 changed files with 475 additions and 0 deletions
+8
View File
@@ -118,6 +118,14 @@ The bundled TTS model package (`pyproject.toml`) is versioned independently.
worker pool is rebuilt on demand, so a reset can never strand an in-flight or
later request. No settings change; the recovery is automatic. (#589 #599)
### CI
- **Feature-coverage test system.** A backend route-inventory test diffs all 213
HTTP/WebSocket endpoints against a committed snapshot (plus a critical-endpoint
guard and a route-count floor), and a frontend feature-coverage test asserts
every app mode is wired to a page and every feature has its i18n namespace — so
an endpoint or page silently disappearing now fails CI on every PR.
## [0.3.7] — 2026-06-20
A stabilization release that clears the wave of issues reported on the 0.3.6
+67
View File
@@ -0,0 +1,67 @@
// WebUI feature-coverage guard — the frontend counterpart to the backend's
// route-inventory test. Verifies, by static analysis (no fragile full-page
// render), that every app feature is actually wired up:
//
// 1. every AppMode (minus documented legacy aliases) has a render branch in
// App.jsx — a feature can't be in the nav state yet unreachable;
// 2. every page component App.jsx lazy-imports resolves to a real file;
// 3. every major feature has its i18n namespace in en.json — so a shipped
// feature can't be missing its user-facing copy (localization rule).
//
// Self-maintaining: the mode list is read from the store and the branches from
// App.jsx, so adding a mode + page + branch keeps this green automatically.
import { describe, it, expect } from 'vitest';
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
const _dir = path.dirname(fileURLToPath(import.meta.url));
const SRC = path.resolve(_dir, '..'); // frontend/src
const read = (p) => fs.readFileSync(path.join(SRC, p), 'utf8');
// Legacy/alias modes intentionally kept in the union but routed elsewhere
// (see uiSlice.ts comments): clone/design → studio; generate/batch unused.
const LEGACY_MODES = new Set(['clone', 'design', 'generate', 'batch']);
function appModes() {
const ts = read('store/uiSlice.ts');
const block = ts.slice(ts.indexOf('export type AppMode'), ts.indexOf(';', ts.indexOf('export type AppMode')));
return [...block.matchAll(/\|\s*'([a-z]+)'/g)].map((m) => m[1]);
}
describe('webUI feature coverage', () => {
const app = read('App.jsx');
it('every non-legacy AppMode has a render branch in App.jsx', () => {
const modes = appModes();
expect(modes.length).toBeGreaterThan(10); // sanity: union parsed
const missing = modes.filter(
(m) => !LEGACY_MODES.has(m) && !app.includes(`mode === '${m}'`),
);
expect(missing, `AppModes with no render branch in App.jsx: ${missing.join(', ')}`).toEqual([]);
});
it('every page App.jsx lazy-imports exists on disk', () => {
const imports = [...app.matchAll(/import\(['"]\.\/(pages|components)\/([\w/]+)['"]\)/g)];
expect(imports.length).toBeGreaterThan(8);
const missing = imports
.map(([, dir, name]) => `${dir}/${name}`)
.filter((rel) => {
return !['.jsx', '.tsx', '.js', '.ts'].some((ext) => fs.existsSync(path.join(SRC, rel + ext)));
});
expect(missing, `lazy() imports with no file: ${missing.join(', ')}`).toEqual([]);
});
it('every major feature has an i18n namespace in en.json', () => {
const en = JSON.parse(read('i18n/locales/en.json'));
// One namespace per shipped feature surface.
const required = [
'launchpad', 'dub', 'dub_workflow', 'clone', 'stories', 'audiobook',
'gallery', 'projects', 'transcriptions', 'settings', 'engines',
'donate', 'enterprise', 'contact', 'tools', 'batch', 'voice',
'history', 'glossary', 'network', 'header', 'nav', 'common',
];
const missing = required.filter((k) => !(k in en));
expect(missing, `feature i18n namespaces missing from en.json: ${missing.join(', ')}`).toEqual([]);
});
});
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
"""Regenerate the backend API route snapshot.
The snapshot (`tests/fixtures/api_routes.txt`) is the committed inventory of
every HTTP/WebSocket route the FastAPI app exposes. `tests/test_api_route_
inventory.py` diffs the live app against it, so an accidentally removed or
renamed endpoint fails CI. When you intentionally add/remove/rename a route,
run this script and commit the updated snapshot.
OMNIVOICE_MODEL=test uv run python scripts/dump_api_routes.py
"""
import os
import sys
from pathlib import Path
_REPO = Path(__file__).resolve().parents[1]
SNAPSHOT = _REPO / "tests" / "fixtures" / "api_routes.txt"
_HEADER = (
"# OmniVoice backend API route snapshot — regenerate with "
"scripts/dump_api_routes.py\n"
"# Guards against accidental endpoint removal/rename "
"(tests/test_api_route_inventory.py).\n"
)
def route_lines(app):
"""Stable, sorted ``"METHODS /path"`` lines for every route on ``app``.
HEAD/OPTIONS are dropped (auto-added by Starlette); WebSocket routes use
``WS``. Excludes things that vary by environment and aren't part of the API
contract: ``Mount`` routes (StaticFiles / sub-app mounts like ``/demo_audio``,
``/outputs``, ``/mcp`` — they only register when their dir/sub-app exists, so
they differ between a dev checkout and a fresh CI runner) and the bare
``GET /`` root (a conditional frontend-serving fallback). HEAD/OPTIONS are
dropped (Starlette adds them automatically).
"""
from starlette.routing import Mount
rows = set()
for r in app.routes:
path = getattr(r, "path", None)
if not path or path == "/" or isinstance(r, Mount):
continue
methods = getattr(r, "methods", None)
if methods:
ms = ",".join(sorted(m for m in methods if m not in ("HEAD", "OPTIONS")))
elif "WebSocket" in type(r).__name__:
ms = "WS"
else:
continue # non-HTTP, non-WS, non-mount: not part of the API surface
rows.add(f"{ms} {path}")
return sorted(rows)
def load_app():
os.environ.setdefault("OMNIVOICE_MODEL", "test")
os.environ.setdefault("OMNIVOICE_DISABLE_FILE_LOG", "1")
sys.path.insert(0, str(_REPO / "backend"))
from main import app
return app
def main():
lines = route_lines(load_app())
# `--print` dumps to stdout (the inventory test captures this in a subprocess
# so importing the app never pollutes the pytest process); default writes the
# committed snapshot.
if "--print" in sys.argv:
sys.stdout.write("\n".join(lines) + "\n")
else:
SNAPSHOT.write_text(_HEADER + "\n".join(lines) + "\n", encoding="utf-8")
sys.stderr.write(f"Wrote {len(lines)} routes to {SNAPSHOT.relative_to(_REPO)}\n")
if __name__ == "__main__":
main()
+211
View File
@@ -0,0 +1,211 @@
# OmniVoice backend API route snapshot — regenerate with scripts/dump_api_routes.py
# Guards against accidental endpoint removal/rename (tests/test_api_route_inventory.py).
DELETE /api/mcp/bindings/{client_id}
DELETE /api/settings/hf-token
DELETE /batch/jobs/{job_id}
DELETE /dub/history
DELETE /dub/history/{history_id}
DELETE /engines/translation/{engine_id}
DELETE /gallery/voices/{voice_id}
DELETE /glossary/{project_id}
DELETE /glossary/{project_id}/{term_id}
DELETE /history
DELETE /history/{history_id}
DELETE /marketplace/{filename}
DELETE /models/{repo_id:path}
DELETE /profiles/{profile_id}
DELETE /profiles/{profile_id}/consent
DELETE /projects/{project_id}
GET /api/mcp/bindings
GET /api/settings/dictation-refinement
GET /api/settings/hf-mirror
GET /api/settings/hf-token/state
GET /api/settings/license/{engine_id}
GET /api/settings/llm-endpoint
GET /api/settings/perf/torch-compile-disabled
GET /api/settings/storage/models-dir
GET /archetypes
GET /archetypes/categories
GET /archetypes/{archetype_id}
GET /archetypes/{archetype_id}/preview
GET /audiobook/jobs
GET /batch/download/{job_id}/{lang}
GET /batch/jobs
GET /batch/jobs/{job_id}
GET /community/items
GET /community/manifest
GET /community/sources
GET /community/submit-url
GET /docs
GET /dub/audio/{job_id}
GET /dub/download-audio/{job_id}
GET /dub/download-audio/{job_id}/{filename}
GET /dub/download-mp3/{job_id}
GET /dub/download-mp3/{job_id}/{filename}
GET /dub/download/{job_id}
GET /dub/download/{job_id}/{filename}
GET /dub/export-segments/{job_id}
GET /dub/export-stems/{job_id}
GET /dub/history
GET /dub/media/{job_id}
GET /dub/onsets/{job_id}
GET /dub/preview-video/{job_id}
GET /dub/preview/{job_id}/{segment_index}
GET /dub/srt/{job_id}
GET /dub/srt/{job_id}/{filename}
GET /dub/thumb/{job_id}
GET /dub/tracks/{job_id}
GET /dub/transcribe-stream/{job_id}
GET /dub/vtt/{job_id}
GET /dub/vtt/{job_id}/{filename}
GET /engines
GET /engines/asr
GET /engines/effects/presets
GET /engines/llm
GET /engines/sonitranslate/status
GET /engines/translation
GET /engines/tts
GET /engines/{engine_id}/health
GET /export/history
GET /gallery/categories
GET /gallery/voices
GET /gallery/voices/{voice_id}
GET /gallery/voices/{voice_id}/preview
GET /glossary/{project_id}
GET /health
GET /history
GET /jobs
GET /jobs/{job_id}
GET /jobs/{job_id}/events
GET /longform/jobs
GET /marketplace/browse
GET /model/loaded
GET /model/status
GET /models
GET /openapi.json
GET /personalities
GET /preview/{filename}
GET /profiles
GET /profiles/{profile_id}
GET /profiles/{profile_id}/audio
GET /profiles/{profile_id}/usage
GET /projects
GET /projects/{project_id}
GET /setup/download-stream
GET /setup/preflight
GET /setup/recommendations
GET /setup/status
GET /sysinfo
GET /system/asr-backends
GET /system/diagnose
GET /system/errors/recent
GET /system/hf-token/state
GET /system/info
GET /system/logs
GET /system/logs/stream
GET /system/logs/tauri
GET /system/network/state
GET /system/notifications
GET /system/quarantine-status
GET /system/tailscale/status
GET /tasks/stream/{task_id}
GET /tools/effects
GET /tools/plugins
GET /v1/audio/voices
GET /watermark/status
PATCH /gallery/voices/{voice_id}
PATCH /projects/{project_id}
POST /api/settings/hf-token
POST /api/settings/license
POST /archetypes/{archetype_id}/use
POST /audiobook
POST /audiobook/cover
POST /audiobook/import
POST /audiobook/plan
POST /audiobook/preview
POST /audiobook/resume/{job_id}
POST /batch/enqueue
POST /batch/jobs/{job_id}/cancel
POST /clean-audio
POST /community/items/{item_id}/use
POST /design/describe
POST /dub/abort/{job_id}
POST /dub/cleanup-segments/{job_id}
POST /dub/generate/{job_id}
POST /dub/import-srt/{job_id}
POST /dub/ingest-url
POST /dub/preview-segment/{job_id}
POST /dub/qc/{job_id}
POST /dub/transcribe/{job_id}
POST /dub/translate
POST /dub/upload
POST /engines/select
POST /engines/sonitranslate/dub
POST /engines/sonitranslate/install
POST /engines/sonitranslate/start
POST /engines/sonitranslate/stop
POST /engines/translation/{engine_id}/install
POST /export
POST /export/record
POST /export/reveal
POST /gallery/download
POST /gallery/search/youtube
POST /gallery/upload
POST /gallery/voices/batch-delete
POST /gallery/voices/{voice_id}/save-as-profile
POST /gallery/voices/{voice_id}/to-profile
POST /generate
POST /glossary/{project_id}
POST /glossary/{project_id}/auto-extract
POST /longform/render
POST /marketplace/export/{profile_id}
POST /marketplace/import
POST /marketplace/install/{filename}
POST /marketplace/publish/{profile_id}
POST /model/unload/{model_id}
POST /models/install
POST /models/install/cancel
POST /personas/export/{profile_id}
POST /personas/import
POST /personas/inspect
POST /preview/upload
POST /profiles
POST /profiles/{profile_id}/consent
POST /profiles/{profile_id}/lock
POST /profiles/{profile_id}/unlock
POST /projects
POST /setup/warmup
POST /stories/encode
POST /system/crash/ack
POST /system/diagnostic-bundle
POST /system/flush-memory
POST /system/logs/clear
POST /system/logs/tauri/clear
POST /system/network/disable
POST /system/network/enable
POST /system/set-env
POST /system/tailscale/disable
POST /system/tailscale/enable
POST /tasks/cancel/{task_id}
POST /tools/direction
POST /tools/incremental
POST /tools/probe
POST /tools/rate-fit
POST /tools/video-context/{job_id}
POST /transcribe
POST /v1/audio/speech
POST /v1/audio/transcriptions
POST /watermark/detect
POST /watermark/settings
PUT /api/mcp/bindings
PUT /api/settings/dictation-refinement
PUT /api/settings/hf-mirror
PUT /api/settings/llm-endpoint
PUT /api/settings/perf/torch-compile-disabled
PUT /api/settings/storage/models-dir
PUT /glossary/{project_id}/{term_id}
PUT /profiles/{profile_id}
PUT /projects/{project_id}
WS /ws/events
WS /ws/transcribe
WS /ws/tts
+112
View File
@@ -0,0 +1,112 @@
"""Backend API surface coverage — the whole route inventory in one guard.
Two layers, both reusable across the project:
1. **Snapshot diff** — the live FastAPI app's router endpoints must equal the
committed snapshot (`tests/fixtures/api_routes.txt`). Any endpoint added,
removed, renamed, or with changed methods fails here, so the API surface can
never drift silently. Intentional changes: regenerate with
`uv run python scripts/dump_api_routes.py` and commit.
2. **Critical-endpoint guard** — a hardcoded set of must-exist endpoints (the
features every platform's prod use depends on). This can't be satisfied by
carelessly regenerating the snapshot — the features have to be present.
The route list is computed in a SUBPROCESS (`scripts/dump_api_routes.py
--print`) so importing the app never pollutes this pytest process's
`sys.modules` — which would break later DB-touching tests. The subprocess uses
the same code path as snapshot generation, so the comparison is deterministic
(no in-process / cross-platform skew). `OMNIVOICE_MODEL=test` skips the 2.4 GB
model load.
"""
import os
import subprocess
import sys
from pathlib import Path
import pytest
_REPO = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(_REPO / "scripts"))
from dump_api_routes import SNAPSHOT # noqa: E402
@pytest.fixture(scope="module")
def live_routes():
"""Route lines from a fresh subprocess — fully isolated from this process."""
env = dict(os.environ, OMNIVOICE_MODEL="test", OMNIVOICE_DISABLE_FILE_LOG="1")
proc = subprocess.run(
[sys.executable, str(_REPO / "scripts" / "dump_api_routes.py"), "--print"],
cwd=str(_REPO), env=env, capture_output=True, text=True, timeout=180,
)
assert proc.returncode == 0, f"route dump failed:\n{proc.stderr}"
return {ln.strip() for ln in proc.stdout.splitlines() if ln.strip()}
# Features that MUST stay reachable. Each entry is "METHOD /path" exactly as the
# snapshot encodes it (the snapshot diff covers the full method set).
_CRITICAL = [
"GET /health", "GET /system/info", "GET /model/status",
"POST /generate", "GET /history",
"GET /profiles", "POST /profiles", "PUT /profiles/{profile_id}",
"DELETE /profiles/{profile_id}", "POST /design/describe",
"POST /dub/upload", "POST /dub/ingest-url", "POST /dub/generate/{job_id}",
"POST /dub/translate",
"GET /engines", "POST /engines/select",
"GET /gallery/voices", "GET /archetypes",
"POST /audiobook", "POST /stories/encode", "POST /batch/enqueue",
"POST /transcribe", "GET /api/settings/hf-token/state",
"POST /v1/audio/speech", "POST /v1/audio/transcriptions",
"WS /ws/events", "WS /ws/tts", "WS /ws/transcribe",
]
def test_route_inventory_matches_snapshot(live_routes):
assert SNAPSHOT.is_file(), (
f"Missing route snapshot {SNAPSHOT} — run "
"`uv run python scripts/dump_api_routes.py`."
)
snap = {
ln.strip()
for ln in SNAPSHOT.read_text(encoding="utf-8").splitlines()
if ln.strip() and not ln.startswith("#")
}
missing = sorted(snap - live_routes) # in snapshot but gone from the app
added = sorted(live_routes - snap) # in the app but not yet snapshotted
msg = []
if missing:
msg.append("Routes REMOVED/renamed since the snapshot (regression?):\n "
+ "\n ".join(missing))
if added:
msg.append("Routes ADDED but not in the snapshot:\n " + "\n ".join(added))
if msg:
msg.append(
"\nIf this change is intentional, regenerate the snapshot:\n"
" OMNIVOICE_MODEL=test uv run python scripts/dump_api_routes.py\n"
"and commit tests/fixtures/api_routes.txt."
)
pytest.fail("\n\n".join(msg))
@pytest.mark.parametrize("entry", _CRITICAL, ids=lambda e: e.replace(" ", "_"))
def test_critical_endpoint_present(live_routes, entry):
"""Each must-exist feature endpoint is registered (method-aware)."""
method, path = entry.split(" ", 1)
served = set()
for ln in live_routes:
m, p = ln.split(" ", 1)
if p == path:
served.update(m.split(","))
assert served, f"Critical endpoint missing entirely: {path}"
assert method in served, (
f"{path} exists but does not serve {method} (serves: {sorted(served)})"
)
def test_route_count_is_sane(live_routes):
"""A floor so a broken router-mount (silently dropping routes) is caught even
if the snapshot were regenerated against the breakage."""
assert len(live_routes) >= 180, (
f"Only {len(live_routes)} routes registered — a router likely failed to "
"mount. Expected 200+."
)