mirror of
https://github.com/ggml-org/whisper.cpp.git
synced 2026-07-23 11:10:57 -05:00
b9d27b1358
* tests : add a new benchmark test for long-form audio
Based on "Earnings-21" corpus by Del Rio et al.
Earnings-21: A Practical Benchmark for ASR in the Wild (2021)
https://arxiv.org/abs/2104.11348
This dataset contains 39 hours of long-form speech, sourced from public
earning calls. Each recording contains roughly 50 minutes of English
dialogues between multiple speakers (2-20 persons).
This benchmark suite should allow us to evaluate the performance of
whisper.cpp on long-form audio data.
Signed-off-by: Fujimoto Seiji <fujimoto@ceptord.net>
* tests : apply PR feedback to 'earnings21/README.md'
Based on feedback from Daniel Bevenius.
- Simplify how to download & prepare a Silero VAD model.
- Fix typo: inferece -> inference
Signed-off-by: Fujimoto Seiji <fujimoto@ceptord.net>
* tests : avoid crashing on non-UTF-8 characters
Based on feedback from Daniel Bevenius.
Add 'errors' parameter to open() in order to avoid unhandled
exception on invalid UTF-8 bytes.
Signed-off-by: Fujimoto Seiji <fujimoto@ceptord.net>
* tests : try to interpret the hypothesis as Windows-1252
Based on the discussion in PR#3185.
Evidently Whisper.cpp can represent a quotation mark as '0x93', which
implifies Windows-1252 (Microsoft's ASCII excention), and cannot be
decoded by UTF-8.
Add an explicit decoding loop to address the issue.
Signed-off-by: Fujimoto Seiji <fujimoto@ceptord.net>
---------
Signed-off-by: Fujimoto Seiji <fujimoto@ceptord.net>
59 lines
1.9 KiB
Makefile
59 lines
1.9 KiB
Makefile
PYTHON = python
|
|
|
|
WHISPER_PREFIX = ../../
|
|
WHISPER_MODEL = tiny
|
|
|
|
WHISPER_CLI = $(WHISPER_PREFIX)build/bin/whisper-cli
|
|
WHISPER_FLAGS = --no-prints --language en --output-txt
|
|
|
|
# You can create eval.conf to override the WHISPER_* variables
|
|
# defined above.
|
|
-include eval.conf
|
|
|
|
# Add `EARNINGS21_EVAL10 = yes` to eval.conf to switch to a
|
|
# 10-hour subset. See "speech-datasets/earnings21/README.md" for
|
|
# more details about this subset.
|
|
ifdef EARNINGS21_EVAL10
|
|
METADATA_CSV = speech-datasets/earnings21/eval10-file-metadata.csv
|
|
AUDIO_SRCS = speech-datasets/earnings21/media/4320211.mp3 \
|
|
speech-datasets/earnings21/media/4341191.mp3 \
|
|
speech-datasets/earnings21/media/4346818.mp3 \
|
|
speech-datasets/earnings21/media/4359971.mp3 \
|
|
speech-datasets/earnings21/media/4365024.mp3 \
|
|
speech-datasets/earnings21/media/4366522.mp3 \
|
|
speech-datasets/earnings21/media/4366893.mp3 \
|
|
speech-datasets/earnings21/media/4367535.mp3 \
|
|
speech-datasets/earnings21/media/4383161.mp3 \
|
|
speech-datasets/earnings21/media/4384964.mp3 \
|
|
speech-datasets/earnings21/media/4387332.mp3
|
|
else
|
|
METADATA_CSV = speech-datasets/earnings21/earnings21-file-metadata.csv
|
|
AUDIO_SRCS = $(sort $(wildcard speech-datasets/earnings21/media/*.mp3))
|
|
endif
|
|
|
|
TRANS_TXTS = $(addsuffix .txt, $(AUDIO_SRCS))
|
|
|
|
# We output the evaluation result to this file.
|
|
DONE = $(WHISPER_MODEL).txt
|
|
|
|
all: $(DONE)
|
|
|
|
$(DONE): $(TRANS_TXTS)
|
|
$(PYTHON) eval.py $(METADATA_CSV) > $@.tmp
|
|
mv $@.tmp $@
|
|
|
|
# Note: This task writes to a temporary file first to
|
|
# create the target file atomically.
|
|
%.mp3.txt: %.mp3
|
|
$(WHISPER_CLI) $(WHISPER_FLAGS) --model $(WHISPER_PREFIX)models/ggml-$(WHISPER_MODEL).bin --file $^ --output-file $^.tmp
|
|
mv $^.tmp.txt $^.txt
|
|
|
|
archive:
|
|
tar -czf $(WHISPER_MODEL).tar.gz --exclude="*.mp3" speech-datasets/earnings21/media $(DONE)
|
|
|
|
clean:
|
|
@rm -f $(TRANS_TXTS)
|
|
@rm -f $(DONE)
|
|
|
|
.PHONY: all archive clean
|