honey126/VoxAI

1# Evaluate with Seed-TTS testset2 3import sys4import os5 6sys.path.append(os.getcwd())7 8import multiprocessing as mp9import numpy as np10 11from model.utils import (12    get_seed_tts_test,13    run_asr_wer,14    run_sim,15)16 17 18eval_task = "wer"  # sim | wer19lang = "zh"  # zh | en20metalst = f"data/seedtts_testset/{lang}/meta.lst"  # seed-tts testset21# gen_wav_dir = f"data/seedtts_testset/{lang}/wavs"  # ground truth wavs22gen_wav_dir = "PATH_TO_GENERATED"  # generated wavs23 24 25# NOTE. paraformer-zh result will be slightly different according to the number of gpus, cuz batchsize is different26#       zh 1.254 seems a result of 4 workers wer_seed_tts27gpus = [0, 1, 2, 3, 4, 5, 6, 7]28test_set = get_seed_tts_test(metalst, gen_wav_dir, gpus)29 30local = False31if local:  # use local custom checkpoint dir32    if lang == "zh":33        asr_ckpt_dir = "../checkpoints/funasr"  # paraformer-zh dir under funasr34    elif lang == "en":35        asr_ckpt_dir = "../checkpoints/Systran/faster-whisper-large-v3"36else:37    asr_ckpt_dir = ""  # auto download to cache dir38 39wavlm_ckpt_dir = "../checkpoints/UniSpeech/wavlm_large_finetune.pth"40 41 42# --------------------------- WER ---------------------------43 44if eval_task == "wer":45    wers = []46 47    with mp.Pool(processes=len(gpus)) as pool:48        args = [(rank, lang, sub_test_set, asr_ckpt_dir) for (rank, sub_test_set) in test_set]49        results = pool.map(run_asr_wer, args)50        for wers_ in results:51            wers.extend(wers_)52 53    wer = round(np.mean(wers) * 100, 3)54    print(f"\nTotal {len(wers)} samples")55    print(f"WER      : {wer}%")56 57 58# --------------------------- SIM ---------------------------59 60if eval_task == "sim":61    sim_list = []62 63    with mp.Pool(processes=len(gpus)) as pool:64        args = [(rank, sub_test_set, wavlm_ckpt_dir) for (rank, sub_test_set) in test_set]65        results = pool.map(run_sim, args)66        for sim_ in results:67            sim_list.extend(sim_)68 69    sim = round(sum(sim_list) / len(sim_list), 3)70    print(f"\nTotal {len(sim_list)} samples")71    print(f"SIM      : {sim}")72