feat(dictation): rebuild to Wispr-Flow quality — live waveform, streaming commits, honest insertion, polished text (#888)
* feat(dictation): rebuild to instant-feedback quality — waveform, streaming commits, honest insertion, text polish Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * docs(changelog): dictation rebuild entry Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(lint): Array.from over new Array(n) — oxlint no-array-constructor Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: mergetest <test@local> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Palash Debnath
co-authored by
Claude Fable 5
mergetest
Claude Fable 5
mergetest
parent
da9315815d
commit
d58010fe1b
@@ -10,6 +10,17 @@ The bundled TTS model package (`pyproject.toml`) is versioned independently.
|
||||
|
||||
### Added
|
||||
|
||||
- **Dictation, rebuilt.** The dictation pill now shows a live waveform the
|
||||
moment the mic opens, streams words as you speak with real download/loading
|
||||
progress on first use, and finishes what you say in about half a second of
|
||||
silence instead of two-and-a-half. Transcripts come out properly
|
||||
capitalized and punctuated. Text insertion is now honest and safe: your
|
||||
clipboard is preserved and restored, failures show what to do (including a
|
||||
one-click jump to macOS Accessibility settings when permission is missing)
|
||||
instead of a false "Pasted", and Esc cancels cleanly at any point. The
|
||||
dictation model also pre-warms in the background after launch, so the first
|
||||
press of the hotkey no longer sits on a cold model load.
|
||||
|
||||
- **LLM Providers: one-click connection testing with real diagnostics.** The
|
||||
Test button in Settings → LLM Providers now measures round-trip latency and
|
||||
turns failures into plain-language guidance — bad key (401/403), wrong
|
||||
|
||||
@@ -19,7 +19,15 @@ Protocol:
|
||||
"segments": [...], "language": "en",
|
||||
"duration_s": 4.2, "transcription_time_s": 0.8,
|
||||
"engine": "mlx-whisper"}
|
||||
{"type": "error", "detail": "..."} — error
|
||||
{"type": "status", "stage": "downloading"|"loading"|"ready"}
|
||||
— model cold-start
|
||||
{"type": "error", "message": "...", "kind": "...",
|
||||
"detail": "..."} — error ("detail"
|
||||
kept for legacy)
|
||||
|
||||
Every ``final`` text is normalised by services.text_polish (leading
|
||||
capital for Latin scripts, terminal punctuation, single-spaced) so the
|
||||
pasted result reads like typed text. Partials are raw.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -32,6 +40,7 @@ import time
|
||||
from fastapi import APIRouter, WebSocket, WebSocketDisconnect
|
||||
|
||||
from api.dependencies import _LOOPBACK_HOSTS, ws_remote_authorized
|
||||
from services.text_polish import polish_text
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger("omnivoice.capture_ws")
|
||||
@@ -302,6 +311,9 @@ async def ws_transcribe(websocket: WebSocket):
|
||||
if total_bytes > MIN_FINAL_BUFFER_BYTES:
|
||||
try:
|
||||
result = await _transcribe_buffer_full(audio_chunks, pcm_sr=pcm_sr)
|
||||
# Dictation v2: deterministic polish so the pasted final reads
|
||||
# like typed text (leading capital, terminal punctuation).
|
||||
result["text"] = polish_text(result.get("text", ""))
|
||||
# Wave 2.1: optional local-LLM refinement of the final text.
|
||||
# Off-thread (network call, not GPU); pass-through on any
|
||||
# failure or when no LLM backend is configured. The raw text
|
||||
@@ -315,7 +327,8 @@ async def ws_transcribe(websocket: WebSocket):
|
||||
logger.debug("Skipped final send — client already disconnected")
|
||||
except Exception as e:
|
||||
logger.error("Final transcription failed: %s", e)
|
||||
await _safe_send({"type": "error", "detail": str(e)})
|
||||
await _safe_send({"type": "error", "message": str(e),
|
||||
"kind": "transcribe", "detail": str(e)})
|
||||
else:
|
||||
await _safe_send({
|
||||
"type": "final",
|
||||
@@ -340,10 +353,19 @@ async def ws_transcribe(websocket: WebSocket):
|
||||
# opt-in 1-byte type prefix when ?aec=1, else bare PCM) at ?sr= (default 16000).
|
||||
# This is the low-latency transport — no WebM/ffmpeg in the hot path.
|
||||
|
||||
# How often the offline-kind handler re-decodes the growing buffer for a live
|
||||
# partial (streaming-kind decodes every frame, no cadence needed).
|
||||
# How often the offline-kind handler re-decodes the live window for a partial
|
||||
# (streaming-kind decodes every frame, no cadence needed).
|
||||
SHERPA_OFFLINE_PARTIAL_S = float(os.environ.get("OMNIVOICE_SHERPA_OFFLINE_PARTIAL", "0.8"))
|
||||
|
||||
# Utterance gate for the offline-kind handler: once the trailing this-many
|
||||
# seconds of the live buffer fall below the RMS floor, the utterance is
|
||||
# COMMITTED — decoded, flushed as a `final`, and dropped from the buffer. Each
|
||||
# decode is thereby bounded by one utterance instead of the whole session
|
||||
# (the old full-buffer re-decode was O(n²)), and a sentence commits ~0.6s
|
||||
# after the user stops speaking instead of only at EOF.
|
||||
SHERPA_OFFLINE_SILENCE_S = float(os.environ.get("OMNIVOICE_SHERPA_OFFLINE_SILENCE", "0.6"))
|
||||
SHERPA_OFFLINE_RMS_FLOOR = float(os.environ.get("OMNIVOICE_SHERPA_OFFLINE_RMS", "0.01"))
|
||||
|
||||
|
||||
def _pcm16_to_f32(pcm: bytes):
|
||||
"""int16 little-endian mono PCM bytes → float32 numpy in [-1, 1]."""
|
||||
@@ -411,6 +433,43 @@ async def _recv_pcm_frame(websocket: WebSocket, aec):
|
||||
return "skip", b""
|
||||
|
||||
|
||||
async def _sherpa_load_with_status(websocket: WebSocket, backend, spec) -> bool:
|
||||
"""Build the recognizer off the event loop, narrating cold-start progress.
|
||||
|
||||
Sends ``{"type":"status","stage":"downloading"|"loading"}`` before the
|
||||
load ("downloading" when the pinned assets aren't in the HF cache yet;
|
||||
stage-only — HF's per-file progress isn't worth a callback plumb-through)
|
||||
and ``{"type":"status","stage":"ready"}`` after, so the widget can show
|
||||
*why* the first dictation takes a moment. Returns False when the load
|
||||
failed (the error frame is sent and the socket closed here).
|
||||
"""
|
||||
try:
|
||||
from services import sherpa_dictation as _sd
|
||||
stage = "loading" if _sd.is_installed(spec) else "downloading"
|
||||
except Exception:
|
||||
stage = "loading"
|
||||
try:
|
||||
await websocket.send_json({"type": "status", "stage": stage})
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await asyncio.to_thread(backend.ensure_loaded)
|
||||
except Exception as e:
|
||||
logger.error("sherpa dictation load failed (%s): %s", spec.id, e)
|
||||
try:
|
||||
await websocket.send_json({"type": "error", "message": str(e),
|
||||
"kind": "load", "detail": str(e)})
|
||||
await websocket.close()
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
try:
|
||||
await websocket.send_json({"type": "status", "stage": "ready"})
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
async def _run_sherpa_streaming(websocket: WebSocket, spec):
|
||||
"""True streaming: feed the OnlineRecognizer frame-by-frame, emit `partial`
|
||||
every time the decoded text grows, and `final` on sherpa's endpoint (silence)
|
||||
@@ -425,16 +484,8 @@ async def _run_sherpa_streaming(websocket: WebSocket, spec):
|
||||
|
||||
backend = SherpaDictationBackend(model_id=spec.id)
|
||||
# Build the recognizer off the event loop (download-on-first-use + ONNX
|
||||
# session init can take a moment); keep the socket responsive.
|
||||
try:
|
||||
await asyncio.to_thread(backend.ensure_loaded)
|
||||
except Exception as e:
|
||||
logger.error("sherpa streaming load failed: %s", e)
|
||||
try:
|
||||
await websocket.send_json({"type": "error", "detail": str(e)})
|
||||
await websocket.close()
|
||||
except Exception:
|
||||
pass
|
||||
# session init can take a moment); status frames keep the widget honest.
|
||||
if not await _sherpa_load_with_status(websocket, backend, spec):
|
||||
return
|
||||
rec = backend._rec
|
||||
stream = rec.create_stream()
|
||||
@@ -484,7 +535,9 @@ async def _run_sherpa_streaming(websocket: WebSocket, spec):
|
||||
continue
|
||||
text, endpoint = await asyncio.to_thread(_decode_after_feed, pcm)
|
||||
if endpoint:
|
||||
# Commit this utterance; reset for the next one.
|
||||
# Commit this utterance (polished — it gets pasted); reset
|
||||
# for the next one.
|
||||
text = polish_text(text)
|
||||
if text:
|
||||
committed.append(text)
|
||||
await _send({"type": "final", "text": text,
|
||||
@@ -507,9 +560,11 @@ async def _run_sherpa_streaming(websocket: WebSocket, spec):
|
||||
except Exception as e:
|
||||
logger.debug("sherpa streaming flush failed: %s", e)
|
||||
tail_text = ""
|
||||
tail_text = polish_text(tail_text)
|
||||
if tail_text and tail_text != (committed[-1] if committed else None):
|
||||
committed.append(tail_text)
|
||||
|
||||
# Pieces are already polished; the join is too (polish is idempotent).
|
||||
full = " ".join(t for t in committed if t).strip()
|
||||
segments = [{"start": 0.0, "end": None, "text": t} for t in committed if t]
|
||||
if not client_disconnected:
|
||||
@@ -534,9 +589,15 @@ async def _run_sherpa_streaming(websocket: WebSocket, spec):
|
||||
|
||||
|
||||
async def _run_sherpa_offline(websocket: WebSocket, spec):
|
||||
"""Offline-kind sherpa model with live partials: buffer raw PCM and
|
||||
re-decode the growing buffer every ~800ms so the user still sees text
|
||||
appear while speaking; finalize on EOF/silence."""
|
||||
"""Offline-kind sherpa model with live partials, utterance-windowed.
|
||||
|
||||
Raw PCM accumulates in a *live* buffer holding only the current
|
||||
(uncommitted) utterance. Every ~800ms the live window is re-decoded for a
|
||||
``partial``; when the trailing ~0.6s of it fall below the RMS floor the
|
||||
utterance is committed — decoded once more, flushed as a ``final``, and
|
||||
its samples dropped — so per-partial cost is bounded by one utterance
|
||||
(not the whole session) and sentences commit as the user pauses instead
|
||||
of only at EOF."""
|
||||
from services.asr_backend import SherpaDictationBackend
|
||||
|
||||
pcm_sr, aec = await _sherpa_session(websocket)
|
||||
@@ -544,22 +605,17 @@ async def _run_sherpa_offline(websocket: WebSocket, spec):
|
||||
spec.id, pcm_sr, bool(aec))
|
||||
|
||||
backend = SherpaDictationBackend(model_id=spec.id)
|
||||
try:
|
||||
await asyncio.to_thread(backend.ensure_loaded)
|
||||
except Exception as e:
|
||||
logger.error("sherpa offline load failed: %s", e)
|
||||
try:
|
||||
await websocket.send_json({"type": "error", "detail": str(e)})
|
||||
await websocket.close()
|
||||
except Exception:
|
||||
pass
|
||||
if not await _sherpa_load_with_status(websocket, backend, spec):
|
||||
return
|
||||
|
||||
buf = bytearray()
|
||||
buf = bytearray() # live (uncommitted) PCM only
|
||||
committed: list[str] = [] # polished utterances already flushed
|
||||
last_partial = ""
|
||||
running = True
|
||||
client_disconnected = False
|
||||
last_audio = time.monotonic()
|
||||
# Trailing-silence gate window, in bytes of int16 mono PCM.
|
||||
sil_bytes = max(2, int(SHERPA_OFFLINE_SILENCE_S * pcm_sr) * 2)
|
||||
|
||||
async def _send(payload) -> bool:
|
||||
nonlocal client_disconnected
|
||||
@@ -572,8 +628,14 @@ async def _run_sherpa_offline(websocket: WebSocket, spec):
|
||||
client_disconnected = True
|
||||
return False
|
||||
|
||||
def _decode_buffer() -> str:
|
||||
samples = _pcm16_to_f32(bytes(buf))
|
||||
def _rms(pcm: bytes) -> float:
|
||||
samples = _pcm16_to_f32(pcm)
|
||||
if not len(samples):
|
||||
return 0.0
|
||||
return float((samples * samples).mean() ** 0.5)
|
||||
|
||||
def _decode_window(pcm: bytes) -> str:
|
||||
samples = _pcm16_to_f32(pcm)
|
||||
if not len(samples):
|
||||
return ""
|
||||
return backend._decode_offline(samples, pcm_sr)
|
||||
@@ -597,14 +659,43 @@ async def _run_sherpa_offline(websocket: WebSocket, spec):
|
||||
logger.debug("sherpa offline receive ended: %s", e)
|
||||
running = False
|
||||
|
||||
async def _commit(snapshot: bytes):
|
||||
"""Finalize one utterance: decode it off-thread, flush a polished
|
||||
`final`, drop its samples from the live buffer. `receive()` may
|
||||
append while we decode — only the snapshot's prefix is dropped."""
|
||||
nonlocal last_partial
|
||||
try:
|
||||
text = await asyncio.to_thread(_decode_window, snapshot)
|
||||
except Exception as e:
|
||||
logger.debug("sherpa offline commit decode failed: %s", e)
|
||||
return
|
||||
del buf[:len(snapshot)]
|
||||
last_partial = ""
|
||||
text = polish_text(text)
|
||||
if text:
|
||||
committed.append(text)
|
||||
await _send({"type": "final", "text": text,
|
||||
"segments": [{"start": 0.0, "end": None, "text": text}],
|
||||
"language": "auto", "engine": backend.id})
|
||||
|
||||
async def partials():
|
||||
nonlocal last_partial, running
|
||||
while running:
|
||||
await asyncio.sleep(SHERPA_OFFLINE_PARTIAL_S)
|
||||
if not running or len(buf) < 2000:
|
||||
continue
|
||||
snapshot = bytes(buf)
|
||||
if len(snapshot) > sil_bytes and \
|
||||
_rms(snapshot[-sil_bytes:]) < SHERPA_OFFLINE_RMS_FLOOR:
|
||||
if _rms(snapshot[:-sil_bytes]) >= SHERPA_OFFLINE_RMS_FLOOR:
|
||||
await _commit(snapshot)
|
||||
else:
|
||||
# Pure silence — drop it (keep the gate window for
|
||||
# continuity) so a long pause can't grow the buffer.
|
||||
del buf[:len(snapshot) - sil_bytes]
|
||||
continue
|
||||
try:
|
||||
text = await asyncio.to_thread(_decode_buffer)
|
||||
text = await asyncio.to_thread(_decode_window, snapshot)
|
||||
except Exception as e:
|
||||
logger.debug("sherpa offline partial failed: %s", e)
|
||||
continue
|
||||
@@ -624,13 +715,18 @@ async def _run_sherpa_offline(websocket: WebSocket, spec):
|
||||
except (asyncio.CancelledError, Exception):
|
||||
pass
|
||||
|
||||
# Drain the trailing (un-committed) utterance on EOF.
|
||||
try:
|
||||
full = await asyncio.to_thread(_decode_buffer)
|
||||
tail = await asyncio.to_thread(_decode_window, bytes(buf))
|
||||
except Exception as e:
|
||||
logger.error("sherpa offline final failed: %s", e)
|
||||
full = ""
|
||||
full = (full or "").strip()
|
||||
segments = [{"start": 0.0, "end": None, "text": full}] if full else []
|
||||
tail = ""
|
||||
tail = polish_text(tail)
|
||||
if tail:
|
||||
committed.append(tail)
|
||||
# Pieces are already polished; the join is too (polish is idempotent).
|
||||
full = " ".join(committed).strip()
|
||||
segments = [{"start": 0.0, "end": None, "text": t} for t in committed]
|
||||
if not client_disconnected:
|
||||
payload = {"type": "final", "text": full, "segments": segments,
|
||||
"language": "auto", "engine": backend.id}
|
||||
|
||||
+38
-4
@@ -387,6 +387,32 @@ def _env_flag(name: str, default: bool = False) -> bool:
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _capture_preload_delay_s() -> float:
|
||||
"""Seconds after boot before the dictation (capture ASR) model warms.
|
||||
|
||||
Late enough that it never competes with startup I/O or the TTS preload;
|
||||
overridable via OMNIVOICE_CAPTURE_PRELOAD_DELAY (mostly for tests)."""
|
||||
raw = os.environ.get("OMNIVOICE_CAPTURE_PRELOAD_DELAY", "")
|
||||
try:
|
||||
v = float(raw)
|
||||
if v >= 0:
|
||||
return v
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return 30.0
|
||||
|
||||
|
||||
def _capture_preload_ram_ok(min_free_bytes: int = 4 * 1024**3) -> bool:
|
||||
"""RAM guard for the dictation warm-up: skip below 4 GB free so the
|
||||
background load never pushes a small machine into swap. If free memory
|
||||
can't be measured, warm anyway (the load path has its own error handling)."""
|
||||
try:
|
||||
import psutil
|
||||
return psutil.virtual_memory().available >= min_free_bytes
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
|
||||
def _mcp_start_timeout_s() -> float:
|
||||
"""Seconds to wait for the MCP session manager to start before giving up
|
||||
and serving without it (#632). Overridable via OMNIVOICE_MCP_START_TIMEOUT_S."""
|
||||
@@ -519,11 +545,19 @@ async def lifespan(app: FastAPI):
|
||||
worker_task = asyncio.create_task(task_manager.worker())
|
||||
# Warm the TTS model in the background so first /generate is instant.
|
||||
preload_task = asyncio.create_task(preload_model())
|
||||
# Capture ASR is useful to keep warm, but it is another large model in
|
||||
# unified memory on Apple Silicon. Keep launch lean by default; users who
|
||||
# prefer instant dictation can opt in with OMNIVOICE_PRELOAD_CAPTURE_ASR=1.
|
||||
if _env_flag("OMNIVOICE_PRELOAD_CAPTURE_ASR"):
|
||||
# Dictation v2: the capture ASR warms in the background BY DEFAULT — a
|
||||
# deferred (~30s post-boot) load off the event loop, so startup stays
|
||||
# lean and the first dictation is instant instead of a cold model load.
|
||||
# OMNIVOICE_PRELOAD_CAPTURE_ASR=0 opts out; the warm-up is also skipped
|
||||
# under 4 GB free RAM (checked at warm time, not boot time).
|
||||
if _env_flag("OMNIVOICE_PRELOAD_CAPTURE_ASR", default=True):
|
||||
async def _preload_capture_asr():
|
||||
await asyncio.sleep(_capture_preload_delay_s())
|
||||
if not _capture_preload_ram_ok():
|
||||
logger.info(
|
||||
"Capture ASR preload skipped: <4GB free RAM; "
|
||||
"dictation ASR will load on first use.")
|
||||
return
|
||||
loading_detail = None
|
||||
prev_loading_detail = None
|
||||
try:
|
||||
|
||||
@@ -34,6 +34,23 @@ _PROVIDER = os.environ.get("OMNIVOICE_SHERPA_ASR_PROVIDER", "cpu")
|
||||
_NUM_THREADS = int(os.environ.get("OMNIVOICE_SHERPA_ASR_THREADS", "2"))
|
||||
|
||||
|
||||
def _endpoint_rules() -> tuple[float, float]:
|
||||
"""Trailing-silence endpoint rules (seconds) for streaming recognizers.
|
||||
|
||||
Wispr-Flow-speed defaults (dictation v2): rule2 commits ~0.6s after speech
|
||||
stops, rule1 flushes after 1.0s of trailing non-speech — down from the
|
||||
upstream 2.4/1.2, which made every committed sentence feel laggy. Read at
|
||||
call time so the env overrides apply without a restart.
|
||||
"""
|
||||
def _f(env: str, default: float) -> float:
|
||||
try:
|
||||
return float(os.environ.get(env, "") or default)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
return (_f("OMNIVOICE_DICTATION_ENDPOINT_R1", 1.0),
|
||||
_f("OMNIVOICE_DICTATION_ENDPOINT_R2", 0.6))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SherpaModelSpec:
|
||||
"""One downloadable sherpa-onnx dictation model.
|
||||
@@ -282,6 +299,7 @@ def build_online_recognizer(spec: SherpaModelSpec, *, download: bool = True):
|
||||
import sherpa_onnx
|
||||
|
||||
d = _resolve_model_dir(spec, download=download)
|
||||
rule1, rule2 = _endpoint_rules()
|
||||
|
||||
def p(role: str) -> str:
|
||||
return os.path.join(d, spec.files[role])
|
||||
@@ -296,8 +314,8 @@ def build_online_recognizer(spec: SherpaModelSpec, *, download: bool = True):
|
||||
provider=_PROVIDER,
|
||||
decoding_method="greedy_search",
|
||||
enable_endpoint_detection=True,
|
||||
rule1_min_trailing_silence=2.4,
|
||||
rule2_min_trailing_silence=1.2,
|
||||
rule1_min_trailing_silence=rule1,
|
||||
rule2_min_trailing_silence=rule2,
|
||||
rule3_min_utterance_length=20,
|
||||
)
|
||||
if spec.kind == "online-paraformer":
|
||||
@@ -309,8 +327,8 @@ def build_online_recognizer(spec: SherpaModelSpec, *, download: bool = True):
|
||||
provider=_PROVIDER,
|
||||
decoding_method="greedy_search",
|
||||
enable_endpoint_detection=True,
|
||||
rule1_min_trailing_silence=2.4,
|
||||
rule2_min_trailing_silence=1.2,
|
||||
rule1_min_trailing_silence=rule1,
|
||||
rule2_min_trailing_silence=rule2,
|
||||
rule3_min_utterance_length=20,
|
||||
)
|
||||
raise ValueError(f"{spec.id} is not a streaming model (kind={spec.kind})")
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""
|
||||
Deterministic polish for dictation finals (dictation v2).
|
||||
|
||||
Every ``final`` that leaves ``/ws/transcribe`` passes through
|
||||
:func:`polish_text` so pasted dictation reads like typed text:
|
||||
|
||||
* leading capital -- Latin scripts only (CJK/Cyrillic/etc. untouched),
|
||||
* terminal punctuation -- a period is appended unless the text already
|
||||
ends with sentence-terminal punctuation (incl. the CJK fullwidth forms),
|
||||
* doubled spaces collapsed, leading/trailing whitespace stripped.
|
||||
|
||||
Purely rule-based -- no model, no locale detection, no network -- so it is
|
||||
byte-for-byte reproducible and idempotent (``polish(polish(x)) == polish(x)``).
|
||||
|
||||
CJK codepoints below are ``\\u``-escaped on purpose: this is functional
|
||||
punctuation handling (allowed), and the escapes keep this file outside the
|
||||
literal-CJK scan in ``tests/test_no_hardcoded_cjk.py`` without growing its
|
||||
allowlist.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# Sentence-terminal punctuation that already "closes" a final -- Latin plus
|
||||
# the CJK fullwidth forms (U+3002 ideographic full stop, U+FF01 !, U+FF1F ?)
|
||||
# and ellipsis. A trailing closing quote/bracket after one of these still
|
||||
# counts as terminated ("He said \"hi.\"").
|
||||
_TERMINAL = ".!?\u2026\u3002\uff01\uff1f"
|
||||
_CLOSERS = "\"'\u201d\u2019\u00bb\u203a)]}\u300d\u300f\uff09\u3011"
|
||||
|
||||
# A dangling clause separator at the very end (ASR often stops mid-breath on
|
||||
# a comma) is swapped for a stop instead of stacking ",." punctuation.
|
||||
# Latin , ; : plus the CJK forms U+3001 U+FF0C U+FF1B U+FF1A.
|
||||
_DANGLING = ",;:\u3001\uff0c\uff1b\uff1a"
|
||||
|
||||
# CJK codepoints (kana, unified ideographs, compatibility + halfwidth forms)
|
||||
# -- used to pick the fullwidth stop U+3002 over "." for CJK sentences.
|
||||
_CJK = re.compile(
|
||||
"[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff66-\uff9f]"
|
||||
)
|
||||
|
||||
_MULTISPACE = re.compile(r"[ \t]{2,}")
|
||||
|
||||
|
||||
def _is_latin_lower(ch: str) -> bool:
|
||||
"""Lowercase letter in a Latin block (ASCII, Latin-1, Latin Extended-A/B).
|
||||
|
||||
Capitalization is meaningless (CJK) or presumptuous (Cyrillic, Greek --
|
||||
the model's casing is trusted) outside Latin scripts.
|
||||
"""
|
||||
return ch.islower() and ord(ch) <= 0x024F
|
||||
|
||||
|
||||
def polish_text(text: str) -> str:
|
||||
"""Normalise one dictation final. Empty/whitespace-only input -> ``""``."""
|
||||
if not text:
|
||||
return ""
|
||||
out = _MULTISPACE.sub(" ", text).strip()
|
||||
if not out:
|
||||
return ""
|
||||
|
||||
# Leading capital (Latin scripts only).
|
||||
if _is_latin_lower(out[0]):
|
||||
out = out[0].upper() + out[1:]
|
||||
|
||||
# Already terminated -- possibly behind a closing quote/bracket?
|
||||
body = out.rstrip(_CLOSERS)
|
||||
if body and body[-1] in _TERMINAL:
|
||||
return out
|
||||
|
||||
# Swap a dangling comma/colon for the stop instead of stacking ",.".
|
||||
if out[-1] in _DANGLING:
|
||||
out = out[:-1].rstrip()
|
||||
if not out:
|
||||
return ""
|
||||
|
||||
# Script-matched stop: fullwidth U+3002 when the sentence ends in CJK.
|
||||
out += "\u3002" if _CJK.search(out[-1]) else "."
|
||||
return out
|
||||
@@ -257,43 +257,121 @@ fn hf_hub_cache_dir() -> PathBuf {
|
||||
|
||||
use enigo::{Direction, Enigo, Key, Keyboard, Settings as EnigoSettings};
|
||||
|
||||
/// Error-kind builder the dictation widget switches on. Kinds are a plain
|
||||
/// string prefix ("a11y:" | "clipboard:" | "paste:") so the JS side can do
|
||||
/// `err.split(':')[0]` without a serde enum crossing the IPC boundary.
|
||||
fn kind_err(kind: &str, detail: impl std::fmt::Display) -> String {
|
||||
format!("{kind}:{detail}")
|
||||
}
|
||||
|
||||
/// How long the transcript must sit on the clipboard before the user's
|
||||
/// previous clipboard is restored: ~300ms covers slow paste consumers
|
||||
/// (Electron apps, remote desktops) without being user-noticeable.
|
||||
const CLIPBOARD_RESTORE_DELAY: Duration = Duration::from_millis(300);
|
||||
|
||||
/// macOS Accessibility grant check — CGEvent key synthesis silently no-ops
|
||||
/// without it. Direct FFI against ApplicationServices: one symbol, not worth
|
||||
/// a crate.
|
||||
#[cfg(target_os = "macos")]
|
||||
fn accessibility_trusted() -> bool {
|
||||
#[link(name = "ApplicationServices", kind = "framework")]
|
||||
extern "C" {
|
||||
fn AXIsProcessTrusted() -> bool;
|
||||
}
|
||||
unsafe { AXIsProcessTrusted() }
|
||||
}
|
||||
|
||||
/// True when the app may synthesize keyboard input. On macOS this is the
|
||||
/// Accessibility grant (System Settings → Privacy & Security → Accessibility);
|
||||
/// other OSes don't gate synthetic input behind a permission, so always true.
|
||||
#[tauri::command]
|
||||
pub fn check_accessibility() -> bool {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
accessibility_trusted()
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
{
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
/// Deep-link into the macOS Privacy → Accessibility pane so the widget can
|
||||
/// walk the user straight to the toggle an "a11y:" error asked for. No-op on
|
||||
/// other OSes (nothing to grant there).
|
||||
#[tauri::command]
|
||||
pub fn open_accessibility_settings() {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
let _ = std::process::Command::new("open")
|
||||
.arg("x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility")
|
||||
.spawn();
|
||||
}
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub fn simulate_paste(text: Option<String>) -> Result<(), String> {
|
||||
// macOS: fail loud BEFORE touching the clipboard if Accessibility isn't
|
||||
// granted — otherwise the ⌘V below silently goes nowhere and the caller
|
||||
// can't tell (the old fire-and-forget behavior).
|
||||
#[cfg(target_os = "macos")]
|
||||
if !accessibility_trusted() {
|
||||
return Err(kind_err("a11y", "accessibility permission not granted"));
|
||||
}
|
||||
|
||||
// Write the transcript to the clipboard natively first: the widget window
|
||||
// is intentionally unfocused on macOS (so the simulated ⌘V reaches the
|
||||
// target app), which makes the WebView clipboard APIs (navigator.clipboard
|
||||
// / execCommand('copy')) fail silently there (#287). `text` is optional so
|
||||
// call sites that already populated the clipboard keep working.
|
||||
//
|
||||
// Save what the user had there first (text only — restoring images/files
|
||||
// isn't worth the platform-specific surface) so dictation doesn't clobber
|
||||
// their clipboard.
|
||||
let mut saved: Option<String> = None;
|
||||
if let Some(t) = text {
|
||||
let mut cb = arboard::Clipboard::new()
|
||||
.map_err(|e| format!("clipboard init failed: {e}"))?;
|
||||
.map_err(|e| kind_err("clipboard", format!("init failed: {e}")))?;
|
||||
saved = cb.get_text().ok();
|
||||
cb.set_text(t)
|
||||
.map_err(|e| format!("clipboard write failed: {e}"))?;
|
||||
.map_err(|e| kind_err("clipboard", format!("write failed: {e}")))?;
|
||||
}
|
||||
|
||||
std::thread::sleep(Duration::from_millis(80));
|
||||
|
||||
let mut enigo = Enigo::new(&EnigoSettings::default())
|
||||
.map_err(|e| format!("Failed to init keyboard sim: {e}"))?;
|
||||
.map_err(|e| kind_err("paste", format!("failed to init keyboard sim: {e}")))?;
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
enigo.key(Key::Meta, Direction::Press)
|
||||
.map_err(|e| format!("key press failed: {e}"))?;
|
||||
.map_err(|e| kind_err("paste", format!("key press failed: {e}")))?;
|
||||
enigo.key(Key::Unicode('v'), Direction::Click)
|
||||
.map_err(|e| format!("key click failed: {e}"))?;
|
||||
.map_err(|e| kind_err("paste", format!("key click failed: {e}")))?;
|
||||
enigo.key(Key::Meta, Direction::Release)
|
||||
.map_err(|e| format!("key release failed: {e}"))?;
|
||||
.map_err(|e| kind_err("paste", format!("key release failed: {e}")))?;
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
{
|
||||
enigo.key(Key::Control, Direction::Press)
|
||||
.map_err(|e| format!("key press failed: {e}"))?;
|
||||
.map_err(|e| kind_err("paste", format!("key press failed: {e}")))?;
|
||||
enigo.key(Key::Unicode('v'), Direction::Click)
|
||||
.map_err(|e| format!("key click failed: {e}"))?;
|
||||
.map_err(|e| kind_err("paste", format!("key click failed: {e}")))?;
|
||||
enigo.key(Key::Control, Direction::Release)
|
||||
.map_err(|e| format!("key release failed: {e}"))?;
|
||||
.map_err(|e| kind_err("paste", format!("key release failed: {e}")))?;
|
||||
}
|
||||
|
||||
// Best-effort restore of the user's clipboard once the target app has
|
||||
// consumed the paste. Only on success — on a paste error the transcript
|
||||
// stays on the clipboard so the user can ⌘V it manually as a fallback.
|
||||
if let Some(prev) = saved {
|
||||
std::thread::spawn(move || {
|
||||
std::thread::sleep(CLIPBOARD_RESTORE_DELAY);
|
||||
if let Ok(mut cb) = arboard::Clipboard::new() {
|
||||
let _ = cb.set_text(prev);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -316,24 +394,32 @@ pub fn simulate_paste(text: Option<String>) -> Result<(), String> {
|
||||
///
|
||||
/// Returns `Err` if the input layer is unavailable (e.g. accessibility not
|
||||
/// granted) so the JS caller can fall back to the clipboard+paste path for
|
||||
/// that segment without double-inserting.
|
||||
/// that segment without double-inserting. Errors carry the same kind
|
||||
/// prefixes as `simulate_paste` ("a11y:" | "paste:").
|
||||
#[tauri::command]
|
||||
pub fn simulate_type(text: Option<String>, backspaces: Option<u32>) -> Result<(), String> {
|
||||
// Same a11y gate as simulate_paste — `.text()`/`.key()` go through the
|
||||
// identical CGEvent path on macOS and would silently no-op without it.
|
||||
#[cfg(target_os = "macos")]
|
||||
if !accessibility_trusted() {
|
||||
return Err(kind_err("a11y", "accessibility permission not granted"));
|
||||
}
|
||||
|
||||
let mut enigo = Enigo::new(&EnigoSettings::default())
|
||||
.map_err(|e| format!("Failed to init keyboard sim: {e}"))?;
|
||||
.map_err(|e| kind_err("paste", format!("failed to init keyboard sim: {e}")))?;
|
||||
|
||||
let n = backspaces.unwrap_or(0);
|
||||
for _ in 0..n {
|
||||
enigo
|
||||
.key(Key::Backspace, Direction::Click)
|
||||
.map_err(|e| format!("backspace failed: {e}"))?;
|
||||
.map_err(|e| kind_err("paste", format!("backspace failed: {e}")))?;
|
||||
}
|
||||
|
||||
if let Some(t) = text {
|
||||
if !t.is_empty() {
|
||||
enigo
|
||||
.text(&t)
|
||||
.map_err(|e| format!("type failed: {e}"))?;
|
||||
.map_err(|e| kind_err("paste", format!("type failed: {e}")))?;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -441,3 +527,36 @@ pub fn save_text_file(path: String, contents: String) -> Result<(), String> {
|
||||
}
|
||||
std::fs::write(p, contents).map_err(|e| format!("write: {e}"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod paste_error_tests {
|
||||
use super::{kind_err, CLIPBOARD_RESTORE_DELAY};
|
||||
|
||||
#[test]
|
||||
fn kind_err_prefixes_with_kind() {
|
||||
assert_eq!(kind_err("a11y", "not granted"), "a11y:not granted");
|
||||
assert_eq!(
|
||||
kind_err("clipboard", "write failed: busy"),
|
||||
"clipboard:write failed: busy"
|
||||
);
|
||||
assert_eq!(
|
||||
kind_err("paste", "key press failed"),
|
||||
"paste:key press failed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kind_survives_colons_in_detail() {
|
||||
// The widget does `err.split(':')[0]` — details containing ':' (OS
|
||||
// error strings usually do) must not corrupt the kind.
|
||||
let e = kind_err("clipboard", "init failed: os error 5");
|
||||
assert_eq!(e.split_once(':').map(|(k, _)| k), Some("clipboard"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restore_delay_is_about_300ms() {
|
||||
// Contract with the widget layer: previous clipboard comes back
|
||||
// ~300ms after the paste, long enough for slow paste consumers.
|
||||
assert_eq!(CLIPBOARD_RESTORE_DELAY.as_millis(), 300);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -256,6 +256,8 @@ pub fn run() {
|
||||
commands::hf_cache_scan,
|
||||
commands::simulate_paste,
|
||||
commands::simulate_type,
|
||||
commands::check_accessibility,
|
||||
commands::open_accessibility_settings,
|
||||
commands::set_tray_recording,
|
||||
commands::quit_app,
|
||||
commands::save_text_file,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,284 @@
|
||||
/**
|
||||
* CaptureWidget pill behaviour — mocked WS + Tauri invoke.
|
||||
*
|
||||
* Covers the truthfulness rebuild: model status frames render real
|
||||
* download/load progress, "Pasted" only appears after simulate_paste resolves
|
||||
* Ok, an "a11y:"-prefixed paste failure renders the actionable Accessibility
|
||||
* error, Esc aborts without pasting, live retract-retype is opt-in (default
|
||||
* sessions never call simulate_type), the missing-Accessibility setup state
|
||||
* shows on mount, and the waveform bars move from real mic frames.
|
||||
*/
|
||||
import React from 'react';
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
||||
import { render, screen, waitFor, fireEvent, act } from '@testing-library/react';
|
||||
import { I18nextProvider } from 'react-i18next';
|
||||
import i18n from '../i18n';
|
||||
|
||||
// ── Hoisted mock state (vi.mock factories may only reference vi.hoisted vars) ──
|
||||
const mocks = vi.hoisted(() => {
|
||||
const state = {
|
||||
dictationEnabled: true,
|
||||
dictationMode: 'toggle',
|
||||
dictationModelId: 'sherpa-parakeet-tdt-v3', // sherpa → raw-PCM live path
|
||||
aecEnabled: false,
|
||||
loadDictationPrefs: () => {},
|
||||
};
|
||||
const holder = {
|
||||
// Per-test knobs for the Tauri invoke mock.
|
||||
a11y: true,
|
||||
paste: async () => undefined,
|
||||
calls: [],
|
||||
// Captured micCapture frame callback (the worklet feed).
|
||||
onFrame: null,
|
||||
};
|
||||
return {
|
||||
state,
|
||||
holder,
|
||||
invoke: async (cmd, args) => {
|
||||
holder.calls.push([cmd, args]);
|
||||
if (cmd === 'check_accessibility') return holder.a11y;
|
||||
if (cmd === 'simulate_paste') return holder.paste();
|
||||
return undefined;
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('../store', () => ({
|
||||
useAppStore: Object.assign((sel) => sel(mocks.state), { getState: () => mocks.state }),
|
||||
}));
|
||||
vi.mock('../api/client', () => ({
|
||||
wsUrl: (p) => `ws://test${p}`,
|
||||
apiFetch: vi.fn(async () => ({ json: async () => ({}) })),
|
||||
}));
|
||||
vi.mock('../pages/Transcriptions', () => ({ addTranscription: vi.fn() }));
|
||||
vi.mock('../utils/copyText', () => ({ copyText: vi.fn(async () => {}) }));
|
||||
vi.mock('react-hot-toast', () => ({ toast: { error: vi.fn() } }));
|
||||
vi.mock('@tauri-apps/api/core', () => ({ invoke: mocks.invoke }));
|
||||
vi.mock('@tauri-apps/api/event', () => ({ listen: vi.fn(async () => () => {}) }));
|
||||
vi.mock('@tauri-apps/api/window', () => ({
|
||||
getCurrentWindow: () => ({ hide: vi.fn(async () => {}) }),
|
||||
}));
|
||||
vi.mock('../utils/aec/micCapture', () => ({
|
||||
startMicCapture: async (stream, onFrame) => {
|
||||
mocks.holder.onFrame = onFrame;
|
||||
return async () => {};
|
||||
},
|
||||
}));
|
||||
|
||||
import CaptureWidget from './CaptureWidget';
|
||||
|
||||
// ── Browser API fakes (jsdom has neither WebSocket use here nor MediaRecorder) ──
|
||||
class FakeWebSocket {
|
||||
static CONNECTING = 0;
|
||||
static OPEN = 1;
|
||||
static CLOSING = 2;
|
||||
static CLOSED = 3;
|
||||
static instances = [];
|
||||
constructor(url) {
|
||||
this.url = url;
|
||||
this.readyState = FakeWebSocket.OPEN; // pretend the connect is instant
|
||||
this.sent = [];
|
||||
this._listeners = {};
|
||||
FakeWebSocket.instances.push(this);
|
||||
}
|
||||
addEventListener(type, fn) {
|
||||
(this._listeners[type] ||= []).push(fn);
|
||||
}
|
||||
send(d) {
|
||||
this.sent.push(d);
|
||||
}
|
||||
close() {
|
||||
if (this.readyState === FakeWebSocket.CLOSED) return;
|
||||
this.readyState = FakeWebSocket.CLOSED;
|
||||
this.onclose?.();
|
||||
}
|
||||
/** Deliver a backend JSON frame. */
|
||||
msg(obj) {
|
||||
this.onmessage?.({ data: JSON.stringify(obj) });
|
||||
}
|
||||
}
|
||||
|
||||
class FakeMediaRecorder {
|
||||
static isTypeSupported() {
|
||||
return true;
|
||||
}
|
||||
constructor() {
|
||||
this.state = 'inactive';
|
||||
}
|
||||
start() {
|
||||
this.state = 'recording';
|
||||
}
|
||||
stop() {
|
||||
this.state = 'inactive';
|
||||
}
|
||||
}
|
||||
|
||||
function withI18n(node) {
|
||||
return <I18nextProvider i18n={i18n}>{node}</I18nextProvider>;
|
||||
}
|
||||
|
||||
// Start a session via the in-page shortcut and wait for the live socket.
|
||||
async function startSession() {
|
||||
fireEvent.keyDown(window, { code: 'Space', ctrlKey: true, shiftKey: true });
|
||||
await waitFor(() => expect(FakeWebSocket.instances.length).toBe(1));
|
||||
await screen.findByText(/Listening/);
|
||||
return FakeWebSocket.instances[0];
|
||||
}
|
||||
|
||||
describe('CaptureWidget', () => {
|
||||
beforeEach(() => {
|
||||
window.__TAURI_INTERNALS__ = {};
|
||||
mocks.holder.a11y = true;
|
||||
mocks.holder.paste = async () => undefined;
|
||||
mocks.holder.calls = [];
|
||||
mocks.holder.onFrame = null;
|
||||
FakeWebSocket.instances = [];
|
||||
global.WebSocket = FakeWebSocket;
|
||||
global.MediaRecorder = FakeMediaRecorder;
|
||||
Object.defineProperty(navigator, 'mediaDevices', {
|
||||
configurable: true,
|
||||
value: { getUserMedia: async () => ({ getTracks: () => [{ stop() {} }] }) },
|
||||
});
|
||||
localStorage.clear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete window.__TAURI_INTERNALS__;
|
||||
delete global.WebSocket;
|
||||
delete global.MediaRecorder;
|
||||
});
|
||||
|
||||
const pasteCalls = () => mocks.holder.calls.filter(([c]) => c === 'simulate_paste');
|
||||
const typeCalls = () => mocks.holder.calls.filter(([c]) => c === 'simulate_type');
|
||||
|
||||
it('renders truthful model status from {type:"status"} frames', async () => {
|
||||
render(withI18n(<CaptureWidget />));
|
||||
const ws = await startSession();
|
||||
|
||||
act(() => ws.msg({ type: 'status', stage: 'downloading', progress: 0.42 }));
|
||||
expect(screen.getByText(/Downloading voice model/)).toBeInTheDocument();
|
||||
expect(screen.getByText(/42%/)).toBeInTheDocument();
|
||||
|
||||
act(() => ws.msg({ type: 'status', stage: 'loading' }));
|
||||
expect(screen.getByText(/Loading model/)).toBeInTheDocument();
|
||||
|
||||
act(() => ws.msg({ type: 'status', stage: 'ready' }));
|
||||
expect(screen.getByText(/Listening/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows "Pasted" only after simulate_paste resolved Ok', async () => {
|
||||
render(withI18n(<CaptureWidget />));
|
||||
const ws = await startSession();
|
||||
|
||||
// Offline-model shape: one utterance final, then the EOF summary.
|
||||
act(() => ws.msg({ type: 'final', text: 'hello world' }));
|
||||
act(() => ws.msg({ type: 'final', text: 'hello world' }));
|
||||
|
||||
await screen.findByText(/Pasted/);
|
||||
expect(pasteCalls().length).toBeGreaterThan(0);
|
||||
expect(pasteCalls()[0][1]).toEqual({ text: 'hello world' });
|
||||
});
|
||||
|
||||
it('an "a11y:" paste rejection renders the actionable error, never "Pasted"', async () => {
|
||||
mocks.holder.paste = async () => {
|
||||
throw 'a11y: process is not trusted';
|
||||
};
|
||||
render(withI18n(<CaptureWidget />));
|
||||
const ws = await startSession();
|
||||
|
||||
act(() => ws.msg({ type: 'final', text: 'hello world' }));
|
||||
act(() => ws.msg({ type: 'final', text: 'hello world' }));
|
||||
|
||||
await screen.findByText(/Accessibility access needed/);
|
||||
expect(screen.queryByText(/Pasted/)).not.toBeInTheDocument();
|
||||
|
||||
// The action button opens the OS Accessibility pane.
|
||||
fireEvent.click(screen.getByText('Open Settings'));
|
||||
await waitFor(() =>
|
||||
expect(mocks.holder.calls.some(([c]) => c === 'open_accessibility_settings')).toBe(true),
|
||||
);
|
||||
});
|
||||
|
||||
it('Esc during recording aborts: socket closed, nothing pasted, pill gone', async () => {
|
||||
const { container } = render(withI18n(<CaptureWidget />));
|
||||
const ws = await startSession();
|
||||
|
||||
fireEvent.keyDown(window, { key: 'Escape' });
|
||||
await waitFor(() => expect(container.querySelector('.capture-pill')).toBeNull());
|
||||
expect(ws.readyState).toBe(FakeWebSocket.CLOSED);
|
||||
expect(pasteCalls()).toEqual([]);
|
||||
});
|
||||
|
||||
it('live retract-retype is OFF by default: partials never simulate_type', async () => {
|
||||
render(withI18n(<CaptureWidget />));
|
||||
const ws = await startSession();
|
||||
|
||||
act(() => ws.msg({ type: 'partial', text: 'hel' }));
|
||||
act(() => ws.msg({ type: 'partial', text: 'hello wor' }));
|
||||
act(() => ws.msg({ type: 'final', text: 'hello world' }));
|
||||
act(() => ws.msg({ type: 'final', text: 'hello world' }));
|
||||
|
||||
await screen.findByText(/Pasted/);
|
||||
// Committed final went through the paste path; no keystroke storms.
|
||||
expect(typeCalls()).toEqual([]);
|
||||
expect(pasteCalls().length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('the LS_LIVE_TYPING pref opts back into word-by-word typing', async () => {
|
||||
localStorage.setItem('omni_capture_live_typing', '1');
|
||||
render(withI18n(<CaptureWidget />));
|
||||
const ws = await startSession();
|
||||
|
||||
act(() => ws.msg({ type: 'partial', text: 'hello' }));
|
||||
await waitFor(() => expect(typeCalls().length).toBeGreaterThan(0));
|
||||
expect(typeCalls()[0][1]).toEqual({ text: 'hello', backspaces: 0 });
|
||||
});
|
||||
|
||||
it('a refined EOF summary is not re-pasted as a new utterance', async () => {
|
||||
render(withI18n(<CaptureWidget />));
|
||||
const ws = await startSession();
|
||||
|
||||
// Two per-utterance commits paste live…
|
||||
act(() => ws.msg({ type: 'final', text: 'Hello world.' }));
|
||||
act(() => ws.msg({ type: 'final', text: 'Second bit.' }));
|
||||
// …then the EOF summary arrives with an LLM-refined variant. Its raw
|
||||
// `text` equals the committed join, so it must finalise — never paste
|
||||
// the whole (refined) transcript a third time.
|
||||
act(() =>
|
||||
ws.msg({
|
||||
type: 'final',
|
||||
text: 'Hello world. Second bit.',
|
||||
refined_text: 'Hello world, second bit.',
|
||||
}),
|
||||
);
|
||||
|
||||
await screen.findByText(/Pasted/);
|
||||
expect(pasteCalls().map(([, a]) => a.text)).toEqual(['Hello world.', 'Second bit.']);
|
||||
});
|
||||
|
||||
it('renders the one-time Accessibility setup state when the mount probe fails', async () => {
|
||||
mocks.holder.a11y = false;
|
||||
render(withI18n(<CaptureWidget />));
|
||||
|
||||
await screen.findByText(/Allow Accessibility/);
|
||||
expect(screen.getByText('Open Settings')).toBeInTheDocument();
|
||||
// It does not pretend to record.
|
||||
expect(screen.queryByText(/Listening/)).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('waveform bars move from the worklet mic frames', async () => {
|
||||
const { container } = render(withI18n(<CaptureWidget />));
|
||||
await startSession();
|
||||
expect(mocks.holder.onFrame).toBeTypeOf('function');
|
||||
|
||||
// Feed ~5 frames of speech-level audio (≈100 ms at 20 ms/frame).
|
||||
for (let i = 0; i < 5; i++) mocks.holder.onFrame(new Float32Array(320).fill(0.5));
|
||||
|
||||
await waitFor(() => {
|
||||
const bars = container.querySelectorAll('.capture-pill__wave-bar');
|
||||
expect(bars.length).toBe(12);
|
||||
const heights = [...bars].map((b) => parseInt(b.style.height, 10));
|
||||
expect(Math.max(...heights)).toBeGreaterThan(12); // above the silence floor
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* captureWaveform — pure ring-buffer model behind the capture pill's live
|
||||
* waveform. No Web Audio, no DOM: the widget feeds it the Float32 frames the
|
||||
* EXISTING micCapture AudioWorklet already emits (~20 ms each at 16 kHz) and
|
||||
* polls `getBars(n)` to draw. Kept pure so the envelope math is unit-testable
|
||||
* without an AudioContext (mirrors utils/aec/pcm.js).
|
||||
*
|
||||
* const wave = createWaveform();
|
||||
* wave.push(frame); // per worklet frame (Float32Array | Int16Array | 0..1 number)
|
||||
* wave.getBars(12); // → 12 smoothed 0..1 bar heights, oldest → newest
|
||||
*/
|
||||
|
||||
/**
|
||||
* RMS level of one PCM frame, in [0, 1]. Accepts Float32 samples in [-1, 1]
|
||||
* or Int16Array samples (normalised by 32768). Empty/absent frames are silence.
|
||||
*/
|
||||
export function frameRms(frame) {
|
||||
if (!frame || !frame.length) return 0;
|
||||
const scale = frame instanceof Int16Array ? 1 / 32768 : 1;
|
||||
let sum = 0;
|
||||
for (let i = 0; i < frame.length; i++) {
|
||||
const s = frame[i] * scale;
|
||||
sum += s * s;
|
||||
}
|
||||
const rms = Math.sqrt(sum / frame.length);
|
||||
return rms > 1 ? 1 : rms;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a waveform ring buffer.
|
||||
*
|
||||
* `attack`/`release` are asymmetric EMA coefficients: a loud frame pulls the
|
||||
* envelope up fast (bars visibly move within a frame or two of speech — the
|
||||
* ~100 ms liveness budget), silence lets it fall smoothly instead of
|
||||
* flickering to zero between words.
|
||||
*
|
||||
* @param {{capacity?: number, attack?: number, release?: number}} opts
|
||||
* @returns {{push(frame: Float32Array|Int16Array|number): number,
|
||||
* getBars(n: number): number[],
|
||||
* reset(): void}}
|
||||
*/
|
||||
export function createWaveform({ capacity = 48, attack = 0.6, release = 0.25 } = {}) {
|
||||
const ring = new Float32Array(capacity);
|
||||
let head = 0; // next write index
|
||||
let count = 0; // total levels stored (caps at capacity)
|
||||
let level = 0; // smoothed envelope carried across pushes
|
||||
|
||||
return {
|
||||
/** Ingest one frame (or a precomputed 0..1 RMS); returns the new level. */
|
||||
push(frame) {
|
||||
const rms = typeof frame === 'number' ? Math.min(Math.max(frame, 0), 1) : frameRms(frame);
|
||||
level += (rms > level ? attack : release) * (rms - level);
|
||||
ring[head] = level;
|
||||
head = (head + 1) % capacity;
|
||||
if (count < capacity) count++;
|
||||
return level;
|
||||
},
|
||||
|
||||
/**
|
||||
* The most recent `n` levels as 0..1 bar heights, oldest → newest (newest
|
||||
* is the last entry, i.e. the right edge of the pill). Slots with no data
|
||||
* yet render as 0. Heights are sqrt-mapped so quiet-but-real speech
|
||||
* (RMS ≈ 0.05) still reads as movement.
|
||||
*/
|
||||
getBars(n) {
|
||||
const bars = Array.from({ length: n }, () => 0);
|
||||
const take = Math.min(n, count);
|
||||
for (let i = 0; i < take; i++) {
|
||||
const idx = (head - 1 - i + capacity * 2) % capacity;
|
||||
bars[n - 1 - i] = Math.sqrt(ring[idx]);
|
||||
}
|
||||
return bars;
|
||||
},
|
||||
|
||||
/** Clear all state (new recording session). */
|
||||
reset() {
|
||||
ring.fill(0);
|
||||
head = 0;
|
||||
count = 0;
|
||||
level = 0;
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -552,7 +552,16 @@
|
||||
"listening_label": "Listening…",
|
||||
"transcribing_label": "Transcribing…",
|
||||
"pasted": "Pasted",
|
||||
"copied": "Copied to clipboard",
|
||||
"no_speech": "No speech detected",
|
||||
"model_downloading": "Downloading voice model…",
|
||||
"model_downloading_pct": "Downloading voice model… {{percent}}%",
|
||||
"model_loading": "Loading model…",
|
||||
"a11y_setup": "Allow Accessibility so dictation can type for you",
|
||||
"a11y_error": "Can't paste — Accessibility access needed",
|
||||
"open_a11y_settings": "Open Settings",
|
||||
"clipboard_error": "Couldn't copy to the clipboard",
|
||||
"paste_error": "Couldn't paste into the active app",
|
||||
"mic_denied": "Mic access denied",
|
||||
"mic_denied_toast": "Microphone access denied. {{hint}}",
|
||||
"mic_hint_mac": "macOS: open System Settings → Privacy & Security → Microphone and enable OmniVoice.",
|
||||
|
||||
+35
-1
@@ -2808,7 +2808,9 @@ input[type="file"]::file-selector-button:hover {
|
||||
gap: 10px;
|
||||
padding: 10px 16px;
|
||||
min-width: 180px;
|
||||
max-width: 280px;
|
||||
/* Wide enough for the waveform + status labels ("Downloading voice
|
||||
model… 42%") and the Accessibility action button. */
|
||||
max-width: 340px;
|
||||
height: 48px;
|
||||
background: rgba(18, 18, 22, 0.88);
|
||||
backdrop-filter: blur(24px) saturate(180%);
|
||||
@@ -2855,6 +2857,10 @@ input[type="file"]::file-selector-button:hover {
|
||||
border-color: rgba(239, 68, 68, 0.3);
|
||||
}
|
||||
|
||||
.capture-pill--setup {
|
||||
border-color: rgba(245, 158, 11, 0.35);
|
||||
}
|
||||
|
||||
/* ── Animations ─────────────────────────────────────────────────────── */
|
||||
|
||||
@keyframes pill-slide-in {
|
||||
@@ -2901,11 +2907,38 @@ input[type="file"]::file-selector-button:hover {
|
||||
animation: none;
|
||||
}
|
||||
|
||||
.capture-pill--setup .capture-pill__dot {
|
||||
background: #f59e0b;
|
||||
animation: none;
|
||||
}
|
||||
|
||||
@keyframes dot-pulse {
|
||||
0%, 100% { opacity: 1; transform: scale(1); }
|
||||
50% { opacity: 0.4; transform: scale(0.6); }
|
||||
}
|
||||
|
||||
/* ── Live waveform (recording) ──────────────────────────────────────────
|
||||
Data-driven bars — heights come from the captureWaveform ring buffer
|
||||
(client-side RMS of the mic worklet frames), not a CSS animation, so the
|
||||
movement is the actual signal. */
|
||||
|
||||
.capture-pill__wave {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 2px;
|
||||
width: 44px;
|
||||
height: 20px;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.capture-pill__wave-bar {
|
||||
flex: 1 1 0;
|
||||
min-height: 2px;
|
||||
border-radius: 1px;
|
||||
background: #ef4444;
|
||||
transition: height 60ms linear;
|
||||
}
|
||||
|
||||
/* ── Content / timer / dismiss ──────────────────────────────────────────
|
||||
Layout + (theme-less, hardcoded-white) colors for the pill's content,
|
||||
label, timer, transcribing spinner, and dismiss button now live as
|
||||
@@ -2918,6 +2951,7 @@ input[type="file"]::file-selector-button:hover {
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.capture-pill { animation: none; opacity: 1; }
|
||||
.capture-pill__dot { animation: none !important; }
|
||||
.capture-pill__wave-bar { transition: none; }
|
||||
}
|
||||
|
||||
/* ── Widget-mode body (standalone Tauri window) ─────────────────────── */
|
||||
|
||||
@@ -7,7 +7,13 @@
|
||||
* — the heart of the "text appears as you speak" behaviour.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { isSherpaModel, classifySherpaFinal, computeTypeDelta } from '../components/CaptureWidget';
|
||||
import {
|
||||
isSherpaModel,
|
||||
classifySherpaFinal,
|
||||
computeTypeDelta,
|
||||
parsePasteError,
|
||||
} from '../components/CaptureWidget';
|
||||
import { frameRms, createWaveform } from '../components/captureWaveform';
|
||||
|
||||
describe('isSherpaModel', () => {
|
||||
it('matches the sherpa- dictation ids and nothing else', () => {
|
||||
@@ -134,3 +140,120 @@ describe('computeTypeDelta', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('parsePasteError', () => {
|
||||
it('splits the Rust kind prefixes off simulate_paste Err strings', () => {
|
||||
expect(parsePasteError('a11y: accessibility not granted')).toEqual({
|
||||
kind: 'a11y',
|
||||
message: 'accessibility not granted',
|
||||
});
|
||||
expect(parsePasteError('clipboard: could not restore')).toEqual({
|
||||
kind: 'clipboard',
|
||||
message: 'could not restore',
|
||||
});
|
||||
expect(parsePasteError('paste: key event failed')).toEqual({
|
||||
kind: 'paste',
|
||||
message: 'key event failed',
|
||||
});
|
||||
});
|
||||
|
||||
it('accepts Error objects (Tauri invoke may reject with either shape)', () => {
|
||||
expect(parsePasteError(new Error('a11y: nope'))).toEqual({ kind: 'a11y', message: 'nope' });
|
||||
});
|
||||
|
||||
it('falls back to a generic paste kind for unprefixed/unknown errors', () => {
|
||||
expect(parsePasteError('something exploded')).toEqual({
|
||||
kind: 'paste',
|
||||
message: 'something exploded',
|
||||
});
|
||||
expect(parsePasteError(undefined)).toEqual({ kind: 'paste', message: '' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('captureWaveform', () => {
|
||||
describe('frameRms', () => {
|
||||
it('is 0 for silence, empty frames, and missing frames', () => {
|
||||
expect(frameRms(new Float32Array(320))).toBe(0);
|
||||
expect(frameRms(new Float32Array(0))).toBe(0);
|
||||
expect(frameRms(undefined)).toBe(0);
|
||||
});
|
||||
|
||||
it('is 1 for a full-scale square wave and clamps beyond it', () => {
|
||||
expect(frameRms(Float32Array.from({ length: 32 }, (_, i) => (i % 2 ? 1 : -1)))).toBe(1);
|
||||
expect(frameRms(new Float32Array(8).fill(2))).toBe(1); // out-of-range input clamps
|
||||
});
|
||||
|
||||
it('normalises Int16Array frames like Float32 ones', () => {
|
||||
const f32 = new Float32Array(64).fill(0.5);
|
||||
const i16 = new Int16Array(64).fill(0.5 * 32768);
|
||||
expect(frameRms(i16)).toBeCloseTo(frameRms(f32), 5);
|
||||
});
|
||||
|
||||
it('computes RMS (a half-scale sine-ish constant is its amplitude)', () => {
|
||||
expect(frameRms(new Float32Array(128).fill(0.25))).toBeCloseTo(0.25, 6);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createWaveform', () => {
|
||||
it('returns all-zero bars before any audio arrives', () => {
|
||||
expect(createWaveform().getBars(8)).toEqual(Array.from({ length: 8 }, () => 0));
|
||||
});
|
||||
|
||||
it('reacts within the ~100 ms liveness budget (bars move after ≤2 frames)', () => {
|
||||
const wave = createWaveform();
|
||||
wave.push(new Float32Array(320).fill(0.5)); // one 20 ms frame of speech
|
||||
const bars = wave.getBars(4);
|
||||
expect(bars[3]).toBeGreaterThan(0.3); // newest bar clearly off the floor
|
||||
});
|
||||
|
||||
it('fills bars oldest→newest with the newest level last', () => {
|
||||
const wave = createWaveform();
|
||||
wave.push(0.1);
|
||||
wave.push(1.0);
|
||||
const bars = wave.getBars(4);
|
||||
expect(bars[0]).toBe(0); // unfilled slots pad the left
|
||||
expect(bars[1]).toBe(0);
|
||||
expect(bars[3]).toBeGreaterThan(bars[2]); // rising signal → rising bars
|
||||
});
|
||||
|
||||
it('smooths: attack is fast, release decays gradually (no flicker-to-zero)', () => {
|
||||
const wave = createWaveform();
|
||||
const loud = wave.push(1.0);
|
||||
const quiet1 = wave.push(0);
|
||||
const quiet2 = wave.push(0);
|
||||
expect(loud).toBeGreaterThan(0.5); // fast attack
|
||||
expect(quiet1).toBeGreaterThan(0); // does not slam to zero
|
||||
expect(quiet1).toBeLessThan(loud); // …but does fall
|
||||
expect(quiet2).toBeLessThan(quiet1); // monotonic decay
|
||||
});
|
||||
|
||||
it('keeps every bar in [0, 1] even for hot input', () => {
|
||||
const wave = createWaveform();
|
||||
for (let i = 0; i < 100; i++) wave.push(5); // out-of-range levels clamp
|
||||
for (const v of wave.getBars(12)) {
|
||||
expect(v).toBeGreaterThanOrEqual(0);
|
||||
expect(v).toBeLessThanOrEqual(1);
|
||||
}
|
||||
});
|
||||
|
||||
it('ring-wraps past capacity, keeping only the most recent levels', () => {
|
||||
const wave = createWaveform({ capacity: 4 });
|
||||
for (let i = 0; i < 3; i++) wave.push(1.0); // drive level up (peak ≈ 0.94)…
|
||||
for (let i = 0; i < 4; i++) wave.push(0); // …then overwrite the whole ring
|
||||
const bars = wave.getBars(4);
|
||||
expect(Math.max(...bars)).toBeLessThan(0.9); // the ≈0.97 peak bar rotated out
|
||||
// and what's left is the decaying tail, oldest→newest
|
||||
expect(bars[0]).toBeGreaterThan(bars[1]);
|
||||
expect(bars[2]).toBeGreaterThan(bars[3]);
|
||||
});
|
||||
|
||||
it('reset() clears levels and the smoothing envelope', () => {
|
||||
const wave = createWaveform();
|
||||
wave.push(1.0);
|
||||
wave.reset();
|
||||
expect(wave.getBars(4)).toEqual([0, 0, 0, 0]);
|
||||
// envelope reset too: the next quiet frame starts from silence
|
||||
expect(wave.push(0)).toBe(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -72,7 +72,9 @@ def test_eof_text_frame_triggers_final_without_disconnect(client):
|
||||
final = msg
|
||||
break
|
||||
assert final is not None, "server never delivered final after EOF"
|
||||
assert final["text"] == "hello world"
|
||||
# Finals are polished (dictation v2): leading capital + terminal
|
||||
# punctuation. The stub returns "hello world" raw.
|
||||
assert final["text"] == "Hello world."
|
||||
assert final["engine"] == "stub"
|
||||
|
||||
|
||||
@@ -100,3 +102,49 @@ def test_empty_binary_frame_acts_as_eof(client):
|
||||
break
|
||||
assert final is not None
|
||||
assert final["engine"] == "stub"
|
||||
|
||||
|
||||
# ── Capture-ASR background warm-up gating (dictation v2) ─────────────────────
|
||||
#
|
||||
# The dictation model warms in the background BY DEFAULT (~30s post-boot);
|
||||
# OMNIVOICE_PRELOAD_CAPTURE_ASR=0 opts out, and the warm-up is skipped when
|
||||
# the machine is under 4 GB of free RAM.
|
||||
|
||||
|
||||
def test_capture_preload_defaults_on(monkeypatch):
|
||||
import main
|
||||
monkeypatch.delenv("OMNIVOICE_PRELOAD_CAPTURE_ASR", raising=False)
|
||||
assert main._env_flag("OMNIVOICE_PRELOAD_CAPTURE_ASR", default=True)
|
||||
monkeypatch.setenv("OMNIVOICE_PRELOAD_CAPTURE_ASR", "0")
|
||||
assert not main._env_flag("OMNIVOICE_PRELOAD_CAPTURE_ASR", default=True)
|
||||
monkeypatch.setenv("OMNIVOICE_PRELOAD_CAPTURE_ASR", "1")
|
||||
assert main._env_flag("OMNIVOICE_PRELOAD_CAPTURE_ASR", default=True)
|
||||
|
||||
|
||||
def test_capture_preload_delay_default_and_override(monkeypatch):
|
||||
import main
|
||||
monkeypatch.delenv("OMNIVOICE_CAPTURE_PRELOAD_DELAY", raising=False)
|
||||
assert main._capture_preload_delay_s() == 30.0
|
||||
monkeypatch.setenv("OMNIVOICE_CAPTURE_PRELOAD_DELAY", "0")
|
||||
assert main._capture_preload_delay_s() == 0.0
|
||||
monkeypatch.setenv("OMNIVOICE_CAPTURE_PRELOAD_DELAY", "junk")
|
||||
assert main._capture_preload_delay_s() == 30.0
|
||||
|
||||
|
||||
def test_capture_preload_ram_guard(monkeypatch):
|
||||
import types
|
||||
import main
|
||||
import psutil
|
||||
|
||||
monkeypatch.setattr(psutil, "virtual_memory",
|
||||
lambda: types.SimpleNamespace(available=2 * 1024**3))
|
||||
assert not main._capture_preload_ram_ok()
|
||||
monkeypatch.setattr(psutil, "virtual_memory",
|
||||
lambda: types.SimpleNamespace(available=8 * 1024**3))
|
||||
assert main._capture_preload_ram_ok()
|
||||
|
||||
# Unmeasurable → warm anyway (the load path has its own error handling).
|
||||
def _boom():
|
||||
raise RuntimeError("no vm info")
|
||||
monkeypatch.setattr(psutil, "virtual_memory", _boom)
|
||||
assert main._capture_preload_ram_ok()
|
||||
|
||||
@@ -113,7 +113,8 @@ def test_partials_before_final(client):
|
||||
msgs.append(ws.receive_json())
|
||||
except Exception:
|
||||
break
|
||||
if msgs[-1].get("type") == "final" and msgs[-1].get("text") in ("a b c", ""):
|
||||
# Finals are polished (dictation v2) — "a b c" ships as "A b c."
|
||||
if msgs[-1].get("type") == "final" and msgs[-1].get("text") in ("A b c.", ""):
|
||||
# got the endpoint-final; keep draining for the EOF final too
|
||||
if len([m for m in msgs if m["type"] == "final"]) >= 1:
|
||||
# try one more receive for trailing final, then stop
|
||||
@@ -123,6 +124,8 @@ def test_partials_before_final(client):
|
||||
pass
|
||||
break
|
||||
|
||||
# Cold-start status frames don't count as results.
|
||||
msgs = [m for m in msgs if m.get("type") != "status"]
|
||||
types_seen = [m["type"] for m in msgs]
|
||||
partials = [m for m in msgs if m["type"] == "partial"]
|
||||
finals = [m for m in msgs if m["type"] == "final"]
|
||||
@@ -179,5 +182,167 @@ def test_non_streaming_model_uses_offline_handler(monkeypatch):
|
||||
final = m
|
||||
break
|
||||
assert final is not None
|
||||
assert final["text"] == "offline text"
|
||||
# Polished final (dictation v2): leading capital + terminal punctuation.
|
||||
assert final["text"] == "Offline text."
|
||||
assert final["engine"] == "sherpa-onnx-asr"
|
||||
|
||||
|
||||
# ── Utterance-windowed offline decoding (dictation v2) ───────────────────────
|
||||
|
||||
|
||||
def test_offline_silence_gate_commits_mid_session(monkeypatch):
|
||||
"""~0.6s of trailing silence must COMMIT the current utterance: a `final`
|
||||
flushes mid-session (not just at EOF) and the committed samples are
|
||||
dropped from the live buffer, so no decode ever spans more than one
|
||||
utterance (the O(n²) full-buffer re-decode fix)."""
|
||||
import time as _time
|
||||
|
||||
import numpy as np
|
||||
from fastapi.testclient import TestClient
|
||||
from api.routers import capture_ws as cw
|
||||
from services import sherpa_dictation as sd
|
||||
from services import asr_backend as ab
|
||||
|
||||
spec = sd.get_spec("sherpa-whisper-tiny") # offline kind
|
||||
monkeypatch.setattr(cw, "_select_sherpa_spec", lambda ws: spec)
|
||||
monkeypatch.setattr(ab.SherpaDictationBackend, "is_available",
|
||||
classmethod(lambda cls: (True, "ready")))
|
||||
monkeypatch.setattr(ab.SherpaDictationBackend, "ensure_loaded",
|
||||
lambda self: setattr(self, "_rec", object()))
|
||||
|
||||
decoded_lens = []
|
||||
|
||||
def fake_decode(self, samples, sr):
|
||||
decoded_lens.append(len(samples))
|
||||
return "utterance one"
|
||||
|
||||
monkeypatch.setattr(ab.SherpaDictationBackend, "_decode_offline", fake_decode)
|
||||
monkeypatch.setitem(sys.modules, "services.refinement",
|
||||
types.SimpleNamespace(maybe_refine=lambda t: None,
|
||||
collapse_repetitive_artifacts=lambda t: t))
|
||||
cw.SHERPA_OFFLINE_PARTIAL_S = 0.05 # fast ticks for the test
|
||||
|
||||
speech = np.full(4000, 3000, dtype=np.int16).tobytes() # 0.25s speech
|
||||
silence = b"\x00" * 22400 # 0.7s silence
|
||||
utt1_samples = (len(speech) + len(silence)) // 2 # 15200
|
||||
|
||||
from main import app
|
||||
client = TestClient(app, client=("127.0.0.1", 50000))
|
||||
with client.websocket_connect("/ws/transcribe?model=sherpa-whisper-tiny&sr=16000") as ws:
|
||||
ws.send_bytes(speech)
|
||||
ws.send_bytes(silence)
|
||||
# Give the gate a few ticks to commit utterance 1, then speak again.
|
||||
_time.sleep(0.4)
|
||||
ws.send_bytes(speech)
|
||||
ws.send_text("EOF")
|
||||
msgs = []
|
||||
for _ in range(40):
|
||||
try:
|
||||
m = ws.receive_json()
|
||||
except Exception:
|
||||
break
|
||||
msgs.append(m)
|
||||
|
||||
finals = [m for m in msgs if m.get("type") == "final"]
|
||||
# Two finals: the gate-committed utterance mid-session + the EOF trailing
|
||||
# final. The old behavior produced exactly one (everything at EOF).
|
||||
assert len(finals) >= 2, f"silence gate never committed mid-session: {msgs}"
|
||||
assert finals[0]["text"] == "Utterance one."
|
||||
# EOF final = committed pieces + the drained live tail (utterance 2).
|
||||
assert finals[-1]["text"] == "Utterance one. Utterance one."
|
||||
# O(n²) fix: every decode was bounded by ONE utterance window — never a
|
||||
# re-decode of already-committed audio (which would be >utt1_samples).
|
||||
assert decoded_lens, "decoder never ran"
|
||||
assert max(decoded_lens) <= utt1_samples
|
||||
|
||||
|
||||
def test_status_frames_precede_results(monkeypatch):
|
||||
"""A WS session whose model isn't cached yet narrates the cold start:
|
||||
status 'downloading' (or 'loading' when cached) then 'ready', before any
|
||||
partial/final."""
|
||||
from fastapi.testclient import TestClient
|
||||
from api.routers import capture_ws as cw
|
||||
from services import sherpa_dictation as sd
|
||||
from services import asr_backend as ab
|
||||
|
||||
spec = sd.get_spec("sherpa-whisper-tiny")
|
||||
monkeypatch.setattr(cw, "_select_sherpa_spec", lambda ws: spec)
|
||||
monkeypatch.setattr(ab.SherpaDictationBackend, "is_available",
|
||||
classmethod(lambda cls: (True, "ready")))
|
||||
monkeypatch.setattr(ab.SherpaDictationBackend, "ensure_loaded",
|
||||
lambda self: setattr(self, "_rec", object()))
|
||||
monkeypatch.setattr(ab.SherpaDictationBackend, "_decode_offline",
|
||||
lambda self, samples, sr: "hi")
|
||||
monkeypatch.setattr(sd, "is_installed", lambda spec: False) # cold cache
|
||||
monkeypatch.setitem(sys.modules, "services.refinement",
|
||||
types.SimpleNamespace(maybe_refine=lambda t: None,
|
||||
collapse_repetitive_artifacts=lambda t: t))
|
||||
|
||||
from main import app
|
||||
client = TestClient(app, client=("127.0.0.1", 50000))
|
||||
with client.websocket_connect("/ws/transcribe?model=sherpa-whisper-tiny&sr=16000") as ws:
|
||||
msgs = [ws.receive_json(), ws.receive_json()] # the two status frames
|
||||
ws.send_bytes(b"\x00" * 4000)
|
||||
ws.send_text("EOF")
|
||||
for _ in range(20):
|
||||
try:
|
||||
m = ws.receive_json()
|
||||
except Exception:
|
||||
break
|
||||
msgs.append(m)
|
||||
if m.get("type") == "final":
|
||||
break
|
||||
|
||||
assert msgs[0] == {"type": "status", "stage": "downloading"}
|
||||
assert msgs[1] == {"type": "status", "stage": "ready"}
|
||||
types_seen = [m["type"] for m in msgs]
|
||||
assert types_seen.index("status") < types_seen.index("final")
|
||||
|
||||
|
||||
# ── Endpoint-rule tuning (dictation v2) ──────────────────────────────────────
|
||||
|
||||
|
||||
class _KwargsOnlineRecognizer:
|
||||
last_kwargs = None
|
||||
|
||||
@classmethod
|
||||
def from_transducer(cls, **kw):
|
||||
cls.last_kwargs = kw
|
||||
return cls()
|
||||
|
||||
@classmethod
|
||||
def from_paraformer(cls, **kw):
|
||||
cls.last_kwargs = kw
|
||||
return cls()
|
||||
|
||||
|
||||
def test_endpoint_rules_fast_defaults_and_env_override(monkeypatch):
|
||||
"""Streaming endpoint rules commit at 1.0s/0.6s by default (was 2.4/1.2 —
|
||||
laggy) and honor OMNIVOICE_DICTATION_ENDPOINT_R1/R2."""
|
||||
from services import sherpa_dictation as sd
|
||||
|
||||
fake = types.ModuleType("sherpa_onnx")
|
||||
fake.OnlineRecognizer = _KwargsOnlineRecognizer
|
||||
monkeypatch.setitem(sys.modules, "sherpa_onnx", fake)
|
||||
monkeypatch.setattr(sd, "_resolve_model_dir",
|
||||
lambda spec, download=True: "/fake/dir")
|
||||
monkeypatch.delenv("OMNIVOICE_DICTATION_ENDPOINT_R1", raising=False)
|
||||
monkeypatch.delenv("OMNIVOICE_DICTATION_ENDPOINT_R2", raising=False)
|
||||
|
||||
sd.build_online_recognizer(sd.get_spec("sherpa-zipformer-en-20m"))
|
||||
kw = _KwargsOnlineRecognizer.last_kwargs
|
||||
assert kw["rule1_min_trailing_silence"] == 1.0
|
||||
assert kw["rule2_min_trailing_silence"] == 0.6
|
||||
assert kw["rule3_min_utterance_length"] == 20 # unchanged
|
||||
|
||||
monkeypatch.setenv("OMNIVOICE_DICTATION_ENDPOINT_R1", "2.4")
|
||||
monkeypatch.setenv("OMNIVOICE_DICTATION_ENDPOINT_R2", "1.2")
|
||||
sd.build_online_recognizer(sd.get_spec("sherpa-paraformer-bilingual-zh-en"))
|
||||
kw = _KwargsOnlineRecognizer.last_kwargs
|
||||
assert kw["rule1_min_trailing_silence"] == 2.4
|
||||
assert kw["rule2_min_trailing_silence"] == 1.2
|
||||
|
||||
# Garbage env falls back to the defaults rather than crashing dictation.
|
||||
monkeypatch.setenv("OMNIVOICE_DICTATION_ENDPOINT_R1", "fast")
|
||||
monkeypatch.setenv("OMNIVOICE_DICTATION_ENDPOINT_R2", "")
|
||||
assert sd._endpoint_rules() == (1.0, 0.6)
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
"""
|
||||
Unit tests for services/text_polish.py — the deterministic polish applied to
|
||||
every dictation `final` (dictation v2): leading capital (Latin scripts only),
|
||||
terminal punctuation (incl. CJK forms), whitespace cleanup. Pure function, no
|
||||
model — so these pin exact strings.
|
||||
"""
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("OMNIVOICE_MODEL", "test")
|
||||
os.environ.setdefault("OMNIVOICE_DISABLE_FILE_LOG", "1")
|
||||
|
||||
from services.text_polish import polish_text # noqa: E402
|
||||
|
||||
|
||||
# ── Latin: capitalization + terminal punctuation ─────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw,polished", [
|
||||
("hello world", "Hello world."),
|
||||
("Hello world", "Hello world."),
|
||||
("what time is it?", "What time is it?"), # already terminated
|
||||
("wow!", "Wow!"),
|
||||
("wait…", "Wait…"), # ellipsis is terminal
|
||||
("42 is the answer", "42 is the answer."), # non-letter start untouched
|
||||
("école ouverte", "École ouverte."), # Latin-1 accents capitalize
|
||||
("čapek wrote robots", "Čapek wrote robots."), # Latin Extended
|
||||
])
|
||||
def test_latin_basics(raw, polished):
|
||||
assert polish_text(raw) == polished
|
||||
|
||||
|
||||
def test_whitespace_collapsed_and_stripped():
|
||||
assert polish_text(" hello world ") == "Hello world."
|
||||
assert polish_text("a \t b") == "A b." # mixed space/tab run
|
||||
|
||||
|
||||
def test_dangling_separator_swapped_for_stop():
|
||||
# ASR often stops mid-breath on a comma — swap it, don't stack ",.".
|
||||
assert polish_text("see you tomorrow,") == "See you tomorrow."
|
||||
assert polish_text("first; second;") == "First; second."
|
||||
|
||||
|
||||
def test_terminal_behind_closing_quote_respected():
|
||||
assert polish_text('he said "stop."') == 'He said "stop."'
|
||||
assert polish_text("(done!)") == "(done!)"
|
||||
assert polish_text('he said "stop"') == 'He said "stop".'
|
||||
|
||||
|
||||
# ── CJK: passthrough (no capitalization) + fullwidth stop ────────────────────
|
||||
|
||||
|
||||
def test_cjk_already_punctuated_passes_through():
|
||||
assert polish_text("你好，世界。") == "你好，世界。"
|
||||
assert polish_text("すごい！") == "すごい！"
|
||||
assert polish_text("本当ですか？") == "本当ですか？"
|
||||
|
||||
|
||||
def test_cjk_gets_fullwidth_stop():
|
||||
assert polish_text("你好世界") == "你好世界。"
|
||||
assert polish_text("ありがとう") == "ありがとう。"
|
||||
|
||||
|
||||
def test_cjk_dangling_comma_swapped():
|
||||
assert polish_text("你好，") == "你好。"
|
||||
|
||||
|
||||
# ── Non-Latin alphabets: no capitalization, still get a stop ─────────────────
|
||||
|
||||
|
||||
def test_cyrillic_and_greek_not_capitalized():
|
||||
assert polish_text("привет мир") == "привет мир."
|
||||
assert polish_text("γεια σου") == "γεια σου."
|
||||
|
||||
|
||||
# ── Edge cases ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_empty_and_whitespace_only():
|
||||
assert polish_text("") == ""
|
||||
assert polish_text(" ") == ""
|
||||
assert polish_text("\t \t") == ""
|
||||
assert polish_text(",") == "" # dangling-only → empty
|
||||
|
||||
|
||||
def test_idempotent():
|
||||
for raw in ["hello world", "你好世界", "привет", "What?!", "a, b,"]:
|
||||
once = polish_text(raw)
|
||||
assert polish_text(once) == once
|
||||
Reference in New Issue
Block a user