Benchmarking

1# Example template for benchmark use:
2python src/habibi_tts/eval/0_benchmark.py -d MSA
3# --dialect DIALECT (MSA | SAU | UAE | ALG | IRQ | EGY | MAR)

1# Zero-shot TTS performance evaluation:
2accelerate launch src/habibi_tts/eval/1_infer_habibi.py -m Unified -d MAR
3# --model MODEL (Unified | Specialized)
4# --dialect DIALECT (MSA | SAU | UAE | ALG | IRQ | EGY | MAR)
5 
6# Use single prompt, to compare with 11Labs model:
7accelerate launch src/habibi_tts/eval/1_infer_habibi.py -m Specialized -d IRQ -s
8# --single (<- add this flag)
9 
10# Use single prompt, call ElevenLabs Eleven v3 (alpha) API:
11pip install elevenlabs
12python src/habibi_tts/eval/1_infer_11labs.py -a YOUR_API_KEY -d MSA
13# --api-key API_KEY (your 11labs account API key)
14# --dialect DIALECT (MSA | SAU | UAE | ALG | IRQ | EGY | MAR)

1# Evaluate WER-O with Meta Omnilingual-ASR-LLM-7B v1:
2pip install omnilingual-asr
3python src/habibi_tts/eval/2_cal_wer-o.py -w results/Habibi/IRQ_Specialized_single -d IRQ
4# --wav-dir WAV_DIR (the folder of generated samples)
5# --dialect DIALECT (MSA | SAU | UAE | ALG | IRQ | EGY | MAR)
6# --batch-size BATCH_SIZE (set smaller if OOM, default 64)
7 
8# Evaluate WER-S with dialect-specific ASR models:
9python src/habibi_tts/eval/2_cal_wer-s.py -w results/Habibi/MAR_Unified -d MAR
10# --wav-dir WAV_DIR (the folder of generated samples)
11# --dialect DIALECT (EGY | MAR)

1python src/habibi_tts/eval/3_cal_spksim.py -w results/Habibi/MAR_Unified -d MAR -c YOUR_WAVLM_PATH
2# --wav-dir WAV_DIR (the folder of generated samples)
3# --dialect DIALECT (MSA | SAU | UAE | ALG | IRQ | EGY | MAR)
4# --ckpt CKPT (the path of download WavLM model)
5 
6python src/habibi_tts/eval/3_cal_spksim.py -w results/Habibi/IRQ_Specialized_single -d IRQ -c YOUR_WAVLM_PATH -s
7# --single (if eval single prompt or 11labs results)

1python src/habibi_tts/eval/4_cal_utmos.py -w results/11Labs_3a/MSA
2# --wav-dir WAV_DIR (the folder of generated samples)