1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
#!/usr/bin/env python3
"""Compare Cactus Whistle with the app's Whisper models on the same clips.
Usage:
python evaluate.py samples/jfk.wav en
python evaluate.py samples/*.wav # language guessed from a *.it.wav suffix
For every clip this runs:
* Whisper tiny (int8, multilingual) through sherpa-onnx
* Whisper base (int8, multilingual) through sherpa-onnx
* Cactus Whistle through its reference Node pipeline
and prints the transcript plus the word error rate (WER) against
`samples/<name>.txt` when that reference file exists.
Setup (see README.md):
npm install
node build-data.mjs
python3 -m venv .venv && .venv/bin/pip install sherpa-onnx numpy
"""
import argparse
import glob
import os
import re
import subprocess
import sys
import unicodedata
import wave
HERE = os.path.dirname(os.path.abspath(__file__))
VENV = os.environ.get("WHISTLE_EVAL_PYTHON", sys.executable)
def normalize(text: str) -> list:
text = unicodedata.normalize("NFKD", text.lower())
text = "".join(c for c in text if not unicodedata.combining(c))
text = re.sub(r"[^a-z0-9]+", " ", text)
return text.split()
def wer(reference: list, hypothesis: list) -> float:
if not reference:
return 0.0 if not hypothesis else 1.0
previous = list(range(len(hypothesis) + 1))
for i, ref in enumerate(reference, start=1):
current = [i]
for j, hyp in enumerate(hypothesis, start=1):
cost = 0 if ref == hyp else 1
current.append(min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + cost))
previous = current
return previous[-1] / len(reference)
def read_reference(audio: str):
base = os.path.splitext(audio)[0]
for candidate in (base + ".txt", audio + ".txt"):
if os.path.isfile(candidate):
with open(candidate, encoding="utf-8") as handle:
return handle.read().strip()
return None
def run_whistle(audio: str) -> str:
out = subprocess.run(
["node", os.path.join(HERE, "vendor", "js", "example.mjs"), audio],
cwd=HERE, capture_output=True, text=True, timeout=600)
if out.returncode != 0:
return f"<whistle failed: {out.stderr.strip().splitlines()[-1] if out.stderr else '?'}>"
first = out.stdout.strip().splitlines()[0] if out.stdout.strip() else ""
return re.sub(r"^\[[a-z]{2}\]\s*", "", first)
def parakeet_available() -> bool:
return os.path.isfile(os.path.join(
HERE, "models", "parakeet-v3-int8", "encoder.int8.onnx"))
def run_parakeet(audio: str) -> str:
script = os.path.join(HERE, "parakeet_baseline.py")
out = subprocess.run(
[VENV, script, audio], capture_output=True, text=True, timeout=1200)
if out.returncode != 0:
tail = out.stderr.strip().splitlines()
return f"<parakeet failed: {tail[-1] if tail else '?'}>"
return out.stdout.strip()
def run_whisper(audio: str, size: str, language: str) -> str:
script = os.path.join(HERE, "whisper_baseline.py")
out = subprocess.run(
[VENV, script, audio, size, language],
capture_output=True, text=True, timeout=900)
if out.returncode != 0:
tail = out.stderr.strip().splitlines()
return f"<whisper {size} failed: {tail[-1] if tail else '?'}>"
return out.stdout.strip()
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("clips", nargs="+")
parser.add_argument("language", nargs="?", default=None)
args = parser.parse_args()
clips = []
for pattern in args.clips:
clips.extend(sorted(glob.glob(pattern)) or [pattern])
print(f"{'clip':<16} {'model':<14} {'WER':>7} transcript")
print("-" * 100)
for audio in clips:
if not os.path.isfile(audio):
print(f"{audio}: not found")
continue
language = args.language
name = os.path.basename(audio).lower()
if language is None:
language = "it" if (name.startswith("it") or ".it." in name) else "en"
reference = read_reference(audio)
ref_words = normalize(reference) if reference else None
results = [
("whistle", run_whistle(audio)),
("whisper tiny", run_whisper(audio, "tiny", language)),
("whisper base", run_whisper(audio, "base", language)),
]
if parakeet_available():
results.append(("parakeet v3", run_parakeet(audio)))
for model, text in results:
if ref_words is None:
score = " n/a"
else:
score = f"{wer(ref_words, normalize(text)) * 100:6.1f}%"
print(f"{os.path.basename(audio):<16} {model:<14} {score} {text}")
if reference:
print(f"{'':<16} {'reference':<14} {'':>7} {reference}")
print()
return 0
if __name__ == "__main__":
raise SystemExit(main())
|
