ci(docs): daily docs-drift job — canonical inventory vs README/docs/registries (Wave 0.1) (#353)
docs/features.yaml is the curated single source of truth (12 features, 11 TTS + 7 ASR engine ids, required install docs). scripts/check-docs-drift.py diffs it against README.md, docs/, and the engine registries — parsing registry keys from source so the CI runner never imports torch. The daily workflow updates ONE rolling 'docs-drift' issue in place and auto-closes it when clean (pattern adapted from Patter, MIT). Self-test includes a real-repo-is-clean gate, so any PR that changes engines/features without updating the inventory fails CI too. Spec: docs/competitive-analysis.md Spec 9a / parity program Wave 0.1. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Palash Debnath
co-authored by
Claude Fable 5
Claude Fable 5
parent
73de4f9277
commit
11c498eeb5
@@ -0,0 +1,97 @@
|
||||
# Docs drift — daily inventory-vs-docs check with a single rolling issue.
|
||||
#
|
||||
# docs/features.yaml is the canonical inventory; scripts/check-docs-drift.py
|
||||
# diffs it against README.md, docs/, and the engine registries. On drift the
|
||||
# job updates (or creates) ONE issue labeled `docs-drift` in place — no issue
|
||||
# spam — and closes it automatically when the check is clean again.
|
||||
#
|
||||
# Companion to the PR-gating validate-install-docs.py step in ci.yml.
|
||||
# Spec: docs/competitive-analysis.md Spec 9a / parity program Wave 0.1.
|
||||
# Rolling-issue pattern adapted from Patter (MIT).
|
||||
|
||||
name: docs-drift
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Daily 03:30 UTC — after most merges, before EU morning triage.
|
||||
- cron: "30 3 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
drift:
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: Install checker deps
|
||||
run: pip install "pyyaml>=6"
|
||||
|
||||
- name: Check inventory vs README/docs/registries
|
||||
id: drift
|
||||
continue-on-error: true
|
||||
run: python scripts/check-docs-drift.py --output drift-report.md
|
||||
|
||||
- name: Update rolling docs-drift issue
|
||||
uses: actions/github-script@v7
|
||||
env:
|
||||
DRIFT_OUTCOME: ${{ steps.drift.outcome }}
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const drifted = process.env.DRIFT_OUTCOME === 'failure';
|
||||
const { owner, repo } = context.repo;
|
||||
const label = 'docs-drift';
|
||||
|
||||
const open = await github.rest.issues.listForRepo({
|
||||
owner, repo, state: 'open', labels: label, per_page: 5,
|
||||
});
|
||||
|
||||
if (drifted) {
|
||||
let body = '';
|
||||
try {
|
||||
body = fs.readFileSync('drift-report.md', 'utf8');
|
||||
} catch {
|
||||
body = '# Docs drift report\n\nThe checker failed before writing a report — see the workflow run logs.';
|
||||
}
|
||||
body += `\n\n---\n_Last checked by [run ${context.runId}](https://github.com/${owner}/${repo}/actions/runs/${context.runId})._\n`;
|
||||
|
||||
if (open.data.length > 0) {
|
||||
await github.rest.issues.update({
|
||||
owner, repo, issue_number: open.data[0].number, body,
|
||||
});
|
||||
core.info(`Updated rolling issue #${open.data[0].number}`);
|
||||
} else {
|
||||
const created = await github.rest.issues.create({
|
||||
owner, repo,
|
||||
title: 'docs-drift: feature inventory vs docs mismatch',
|
||||
body,
|
||||
labels: [label, 'documentation'],
|
||||
});
|
||||
core.info(`Created rolling issue #${created.data.number}`);
|
||||
}
|
||||
} else {
|
||||
for (const issue of open.data) {
|
||||
await github.rest.issues.createComment({
|
||||
owner, repo, issue_number: issue.number,
|
||||
body: 'Drift resolved — nightly check is clean again. Closing automatically.',
|
||||
});
|
||||
await github.rest.issues.update({
|
||||
owner, repo, issue_number: issue.number, state: 'closed',
|
||||
});
|
||||
core.info(`Closed rolling issue #${issue.number}`);
|
||||
}
|
||||
}
|
||||
|
||||
- name: Surface drift as a failed run
|
||||
if: steps.drift.outcome == 'failure'
|
||||
run: |
|
||||
echo "Docs drift detected — see the rolling docs-drift issue."
|
||||
exit 1
|
||||
@@ -0,0 +1,73 @@
|
||||
# Canonical feature inventory — the single source of truth that the daily
|
||||
# docs-drift job (.github/workflows/docs-drift.yml) diffs against README.md,
|
||||
# docs/, and the engine registries via scripts/check-docs-drift.py.
|
||||
#
|
||||
# When a PR adds or removes an engine or user-facing feature, update this
|
||||
# file in the same PR — otherwise the nightly job opens/updates the rolling
|
||||
# `docs-drift` issue. Spec: docs/competitive-analysis.md Spec 9a /
|
||||
# docs/specs/2026-06-12-elevenlabs-parity-program.md Wave 0.1.
|
||||
|
||||
# Each name must appear verbatim in README.md (the Features grid).
|
||||
features:
|
||||
- Voice Cloning
|
||||
- Voice Design
|
||||
- Video Dubbing
|
||||
- Dictation Widget
|
||||
- Vocal Isolation
|
||||
- Speaker Diarization
|
||||
- Batch Queue
|
||||
- MCP Server
|
||||
- AI Watermark
|
||||
- 100% Local
|
||||
- GPU Auto-Detect
|
||||
- Extensible
|
||||
|
||||
# id: must exactly match the registry keys in backend/services/tts_backend.py
|
||||
# (_REGISTRY eager entries + _LAZY_REGISTRY).
|
||||
# readme (optional): a string that must appear in README.md (engine table row).
|
||||
# doc (optional): a repo-relative doc file that must exist.
|
||||
tts_engines:
|
||||
- id: omnivoice
|
||||
readme: "**OmniVoice** (default)"
|
||||
- id: cosyvoice
|
||||
readme: CosyVoice 3
|
||||
doc: docs/engines/cosyvoice.md
|
||||
- id: kittentts
|
||||
readme: KittenTTS
|
||||
- id: mlx-audio
|
||||
readme: MLX-Audio
|
||||
- id: voxcpm2
|
||||
readme: VoxCPM2
|
||||
- id: moss-tts-nano
|
||||
readme: MOSS-TTS-Nano
|
||||
- id: gpt-sovits
|
||||
- id: sherpa-onnx
|
||||
- id: indextts2
|
||||
doc: docs/engines/indextts.md
|
||||
- id: omnivoice-gguf
|
||||
- id: supertonic3
|
||||
|
||||
# Same contract against backend/services/asr_backend.py _REGISTRY.
|
||||
asr_engines:
|
||||
- id: whisperx
|
||||
readme: "**WhisperX** (default)"
|
||||
- id: faster-whisper
|
||||
readme: Faster-Whisper
|
||||
- id: mlx-whisper
|
||||
readme: MLX Whisper
|
||||
- id: pytorch-whisper
|
||||
readme: PyTorch Whisper
|
||||
- id: nemo-parakeet
|
||||
readme: Parakeet TDT
|
||||
- id: moonshine
|
||||
readme: Moonshine
|
||||
- id: funasr
|
||||
readme: FunASR
|
||||
|
||||
# Doc files that must exist (the install path users are sent to).
|
||||
docs:
|
||||
- docs/install/macos.md
|
||||
- docs/install/windows.md
|
||||
- docs/install/linux.md
|
||||
- docs/install/docker.md
|
||||
- docs/install/troubleshooting.md
|
||||
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Diff the canonical feature inventory against README, docs, and registries.
|
||||
|
||||
The inventory (``docs/features.yaml``) is the curated truth. This checker
|
||||
verifies, without importing any backend module (the engine registries pull
|
||||
torch transitively, which the docs-drift CI runner does not have):
|
||||
|
||||
1. every ``features[]`` name appears verbatim in README.md;
|
||||
2. ``tts_engines[].id`` is exactly the set of registry keys parsed from
|
||||
``backend/services/tts_backend.py`` (eager ``_REGISTRY`` + lazy
|
||||
``_LAZY_REGISTRY``), both directions;
|
||||
3. ``asr_engines[].id`` likewise against ``backend/services/asr_backend.py``;
|
||||
4. every ``readme:`` string appears in README.md;
|
||||
5. every ``doc:`` / ``docs[]`` file exists.
|
||||
|
||||
Exit 0 = no drift. Exit 1 = drift; findings go to stderr and, with
|
||||
``--output``, to a Markdown report consumed by the rolling-issue automation
|
||||
in ``.github/workflows/docs-drift.yml``.
|
||||
|
||||
Companion to ``scripts/validate-install-docs.py`` (the PR-gating half).
|
||||
Rolling-issue pattern adapted from Patter (MIT) — see
|
||||
docs/competitive-analysis.md, Patter deep dive 4.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
# Markers locating the registry dicts whose keys we extract. Each marker is
|
||||
# matched at a line start; the block ends at the first line that is exactly
|
||||
# ``}`` or ``})`` (the registries are flat string-keyed dict literals).
|
||||
_TTS_MARKERS = ("_LAZY_REGISTRY: dict[str, tuple[str, str]] = {",
|
||||
"_REGISTRY: dict[str, type[TTSBackend]] = _LazyRegistry({")
|
||||
_ASR_MARKERS = ("_REGISTRY: dict[str, type[ASRBackend]] = {",)
|
||||
|
||||
_KEY_RE = re.compile(r'^\s*"([^"]+)"\s*:')
|
||||
|
||||
|
||||
def _registry_ids(source: str, markers: tuple[str, ...], *, path: str) -> set[str]:
|
||||
"""Parse string keys out of the dict literal(s) following each marker."""
|
||||
ids: set[str] = set()
|
||||
lines = source.splitlines()
|
||||
for marker in markers:
|
||||
try:
|
||||
start = next(i for i, ln in enumerate(lines) if ln.strip() == marker.strip())
|
||||
except StopIteration:
|
||||
raise SystemExit(
|
||||
f"check-docs-drift: marker not found in {path}: {marker!r} — "
|
||||
"the registry layout changed; update _TTS_MARKERS/_ASR_MARKERS."
|
||||
)
|
||||
for ln in lines[start + 1:]:
|
||||
stripped = ln.strip()
|
||||
if stripped in ("}", "})"):
|
||||
break
|
||||
if stripped.startswith("#"):
|
||||
continue
|
||||
m = _KEY_RE.match(ln)
|
||||
if m:
|
||||
ids.add(m.group(1))
|
||||
return ids
|
||||
|
||||
|
||||
def _check(root: Path) -> list[str]:
|
||||
drifts: list[str] = []
|
||||
|
||||
inv_path = root / "docs" / "features.yaml"
|
||||
if not inv_path.exists():
|
||||
return [f"`{inv_path.relative_to(root)}` is missing"]
|
||||
inv = yaml.safe_load(inv_path.read_text(encoding="utf-8")) or {}
|
||||
|
||||
readme = (root / "README.md").read_text(encoding="utf-8")
|
||||
|
||||
# 1. Features present in README.
|
||||
for name in inv.get("features", []):
|
||||
if name not in readme:
|
||||
drifts.append(f"feature `{name}` is in the inventory but not in README.md")
|
||||
|
||||
# 2–4. Engine ids vs registries; readme strings; per-engine docs.
|
||||
for section, src_rel, markers in (
|
||||
("tts_engines", "backend/services/tts_backend.py", _TTS_MARKERS),
|
||||
("asr_engines", "backend/services/asr_backend.py", _ASR_MARKERS),
|
||||
):
|
||||
entries = inv.get(section, [])
|
||||
inv_ids = {e["id"] for e in entries}
|
||||
code_ids = _registry_ids(
|
||||
(root / src_rel).read_text(encoding="utf-8"), markers, path=src_rel
|
||||
)
|
||||
for missing in sorted(code_ids - inv_ids):
|
||||
drifts.append(
|
||||
f"engine `{missing}` exists in `{src_rel}` but not in the "
|
||||
f"`{section}` inventory — document it (or list it deliberately)"
|
||||
)
|
||||
for gone in sorted(inv_ids - code_ids):
|
||||
drifts.append(
|
||||
f"engine `{gone}` is in the `{section}` inventory but no longer "
|
||||
f"in `{src_rel}` — remove it from the inventory and docs"
|
||||
)
|
||||
for entry in entries:
|
||||
readme_name = entry.get("readme")
|
||||
if readme_name and readme_name not in readme:
|
||||
drifts.append(
|
||||
f"engine `{entry['id']}`: expected `{readme_name}` in README.md"
|
||||
)
|
||||
doc = entry.get("doc")
|
||||
if doc and not (root / doc).exists():
|
||||
drifts.append(f"engine `{entry['id']}`: doc `{doc}` does not exist")
|
||||
|
||||
# 5. Required docs exist.
|
||||
for doc in inv.get("docs", []):
|
||||
if not (root / doc).exists():
|
||||
drifts.append(f"required doc `{doc}` does not exist")
|
||||
|
||||
return drifts
|
||||
|
||||
|
||||
def _report(drifts: list[str], checked: int) -> str:
|
||||
lines = ["# Docs drift report", ""]
|
||||
if drifts:
|
||||
lines.append(f"{len(drifts)} mismatch(es) between `docs/features.yaml`, "
|
||||
"README.md, docs/, and the engine registries:")
|
||||
lines.append("")
|
||||
lines += [f"- {d}" for d in drifts]
|
||||
lines.append("")
|
||||
lines.append("Fix by updating the docs **or** the inventory — whichever is "
|
||||
"stale. This issue updates in place and closes automatically "
|
||||
"when the nightly check is clean.")
|
||||
else:
|
||||
lines.append(f"No drift — {checked} inventory entries verified.")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None, root: Path | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, default=None,
|
||||
help="repo root (default: parent of this script's dir)")
|
||||
parser.add_argument("--output", type=Path, default=None,
|
||||
help="write a Markdown report to this path")
|
||||
args = parser.parse_args([] if argv is None else argv)
|
||||
|
||||
repo = args.root or root or Path(__file__).resolve().parents[1]
|
||||
drifts = _check(repo)
|
||||
|
||||
inv = yaml.safe_load((repo / "docs" / "features.yaml").read_text(encoding="utf-8")) \
|
||||
if (repo / "docs" / "features.yaml").exists() else {}
|
||||
checked = sum(len(inv.get(k, [])) for k in ("features", "tts_engines", "asr_engines", "docs"))
|
||||
|
||||
if args.output:
|
||||
args.output.write_text(_report(drifts, checked), encoding="utf-8")
|
||||
|
||||
if drifts:
|
||||
for d in drifts:
|
||||
print(f"docs-drift: {d}", file=sys.stderr)
|
||||
print(f"check-docs-drift: {len(drifts)} drift(s) across {checked} "
|
||||
"inventory entries.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"OK — {checked} inventory entries verified against README, docs, "
|
||||
"and engine registries.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Self-test for scripts/check-docs-drift.py (parity program Wave 0.1).
|
||||
|
||||
Mirrors the import pattern of tests/scripts/test_validate_install_docs.py:
|
||||
the hyphenated script is loaded via importlib against a tmp-path repo fixture,
|
||||
so no test depends on real repo state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPT_PATH = Path(__file__).resolve().parents[2] / "scripts" / "check-docs-drift.py"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def drift_module():
|
||||
spec = importlib.util.spec_from_file_location("check_docs_drift", SCRIPT_PATH)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
sys.modules["check_docs_drift"] = mod
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
|
||||
_TTS_SOURCE = '''
|
||||
_LAZY_REGISTRY: dict[str, tuple[str, str]] = {
|
||||
"indextts2": ("engines.indextts", "IndexTTS2Backend"),
|
||||
}
|
||||
|
||||
_REGISTRY: dict[str, type[TTSBackend]] = _LazyRegistry({
|
||||
"omnivoice": OmniVoiceBackend,
|
||||
# "indextts2": resolved lazily
|
||||
"cosyvoice": CosyVoiceBackend,
|
||||
})
|
||||
'''
|
||||
|
||||
_ASR_SOURCE = '''
|
||||
_REGISTRY: dict[str, type[ASRBackend]] = {
|
||||
"whisperx": WhisperXBackend,
|
||||
}
|
||||
'''
|
||||
|
||||
_INVENTORY = """
|
||||
features:
|
||||
- Voice Cloning
|
||||
tts_engines:
|
||||
- id: omnivoice
|
||||
readme: OmniVoice (default)
|
||||
- id: cosyvoice
|
||||
doc: docs/engines/cosyvoice.md
|
||||
- id: indextts2
|
||||
asr_engines:
|
||||
- id: whisperx
|
||||
readme: WhisperX (default)
|
||||
docs:
|
||||
- docs/install/macos.md
|
||||
"""
|
||||
|
||||
_README = """# App
|
||||
## Features
|
||||
Voice Cloning
|
||||
## Engines
|
||||
| OmniVoice (default) | ... |
|
||||
| WhisperX (default) | ... |
|
||||
"""
|
||||
|
||||
|
||||
def _make_root(tmp_path: Path, *, inventory: str = _INVENTORY, readme: str = _README,
|
||||
tts: str = _TTS_SOURCE, asr: str = _ASR_SOURCE) -> Path:
|
||||
root = tmp_path / "repo"
|
||||
(root / "docs" / "engines").mkdir(parents=True)
|
||||
(root / "docs" / "install").mkdir(parents=True)
|
||||
(root / "backend" / "services").mkdir(parents=True)
|
||||
(root / "docs" / "features.yaml").write_text(inventory, encoding="utf-8")
|
||||
(root / "README.md").write_text(readme, encoding="utf-8")
|
||||
(root / "backend" / "services" / "tts_backend.py").write_text(tts, encoding="utf-8")
|
||||
(root / "backend" / "services" / "asr_backend.py").write_text(asr, encoding="utf-8")
|
||||
(root / "docs" / "engines" / "cosyvoice.md").write_text("# CosyVoice\n", encoding="utf-8")
|
||||
(root / "docs" / "install" / "macos.md").write_text("# macOS\n", encoding="utf-8")
|
||||
return root
|
||||
|
||||
|
||||
def test_clean_state_passes(drift_module, tmp_path, capsys):
|
||||
root = _make_root(tmp_path)
|
||||
assert drift_module.main([], root=root) == 0
|
||||
assert "OK" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_feature_missing_from_readme_fails(drift_module, tmp_path, capsys):
|
||||
root = _make_root(tmp_path, readme=_README.replace("Voice Cloning", "Something Else"))
|
||||
assert drift_module.main([], root=root) == 1
|
||||
assert "Voice Cloning" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_engine_in_code_but_not_inventory_fails(drift_module, tmp_path, capsys):
|
||||
tts = _TTS_SOURCE.replace(
|
||||
'"cosyvoice": CosyVoiceBackend,',
|
||||
'"cosyvoice": CosyVoiceBackend,\n "newengine": NewBackend,',
|
||||
)
|
||||
root = _make_root(tmp_path, tts=tts)
|
||||
assert drift_module.main([], root=root) == 1
|
||||
assert "newengine" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_engine_in_inventory_but_not_code_fails(drift_module, tmp_path, capsys):
|
||||
asr = _ASR_SOURCE.replace('"whisperx": WhisperXBackend,', "")
|
||||
root = _make_root(tmp_path, asr=asr)
|
||||
assert drift_module.main([], root=root) == 1
|
||||
assert "whisperx" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_commented_registry_lines_ignored(drift_module, tmp_path):
|
||||
# `# "indextts2": resolved lazily` inside _REGISTRY must not count as a key
|
||||
# (it is also a real lazy key here, so a parser that read comments would
|
||||
# not fail — drop the lazy entry to make the assertion meaningful).
|
||||
tts = _TTS_SOURCE.replace(' "indextts2": ("engines.indextts", "IndexTTS2Backend"),\n', "")
|
||||
inventory = _INVENTORY.replace(" - id: indextts2\n", "")
|
||||
root = _make_root(tmp_path, tts=tts, inventory=inventory)
|
||||
assert drift_module.main([], root=root) == 0
|
||||
|
||||
|
||||
def test_missing_engine_doc_fails(drift_module, tmp_path, capsys):
|
||||
root = _make_root(tmp_path)
|
||||
(root / "docs" / "engines" / "cosyvoice.md").unlink()
|
||||
assert drift_module.main([], root=root) == 1
|
||||
assert "cosyvoice" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_missing_required_doc_fails(drift_module, tmp_path, capsys):
|
||||
root = _make_root(tmp_path)
|
||||
(root / "docs" / "install" / "macos.md").unlink()
|
||||
assert drift_module.main([], root=root) == 1
|
||||
assert "docs/install/macos.md" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_readme_string_missing_fails(drift_module, tmp_path, capsys):
|
||||
root = _make_root(tmp_path, readme=_README.replace("WhisperX (default)", "WhisperX"))
|
||||
assert drift_module.main([], root=root) == 1
|
||||
assert "WhisperX (default)" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_report_written_on_drift(drift_module, tmp_path):
|
||||
root = _make_root(tmp_path)
|
||||
(root / "docs" / "install" / "macos.md").unlink()
|
||||
out = tmp_path / "drift-report.md"
|
||||
assert drift_module.main(["--output", str(out)], root=root) == 1
|
||||
text = out.read_text(encoding="utf-8")
|
||||
assert "Docs drift report" in text and "docs/install/macos.md" in text
|
||||
|
||||
|
||||
def test_report_written_on_clean(drift_module, tmp_path):
|
||||
root = _make_root(tmp_path)
|
||||
out = tmp_path / "drift-report.md"
|
||||
assert drift_module.main(["--output", str(out)], root=root) == 0
|
||||
assert "No drift" in out.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_changed_registry_layout_is_loud(drift_module, tmp_path):
|
||||
root = _make_root(tmp_path, tts="_SOMETHING_ELSE = {}\n")
|
||||
with pytest.raises(SystemExit, match="marker not found"):
|
||||
drift_module.main([], root=root)
|
||||
|
||||
|
||||
def test_real_repo_is_clean(drift_module):
|
||||
"""The shipped inventory must match the shipped README/registries."""
|
||||
repo = Path(__file__).resolve().parents[2]
|
||||
assert drift_module.main([], root=repo) == 0
|
||||
Reference in New Issue
Block a user