Julius8888/XiJingPing_Voice_Clone

1import json2from collections import defaultdict3from random import shuffle4from typing import Optional5import os6 7from tqdm import tqdm8import click9from text.cleaner import clean_text10from config import config11from infer import latest_version12 13preprocess_text_config = config.preprocess_text_config14 15 16@click.command()17@click.option(18    "--transcription-path",19    default=preprocess_text_config.transcription_path,20    type=click.Path(exists=True, file_okay=True, dir_okay=False),21)22@click.option("--cleaned-path", default=preprocess_text_config.cleaned_path)23@click.option("--train-path", default=preprocess_text_config.train_path)24@click.option("--val-path", default=preprocess_text_config.val_path)25@click.option(26    "--config-path",27    default=preprocess_text_config.config_path,28    type=click.Path(exists=True, file_okay=True, dir_okay=False),29)30@click.option("--val-per-lang", default=preprocess_text_config.val_per_lang)31@click.option("--max-val-total", default=preprocess_text_config.max_val_total)32@click.option("--clean/--no-clean", default=preprocess_text_config.clean)33@click.option("-y", "--yml_config")34def preprocess(35    transcription_path: str,36    cleaned_path: Optional[str],37    train_path: str,38    val_path: str,39    config_path: str,40    val_per_lang: int,41    max_val_total: int,42    clean: bool,43    yml_config: str,  # 这个不要删44):45    if cleaned_path == "" or cleaned_path is None:46        cleaned_path = transcription_path + ".cleaned"47 48    if clean:49        with open(cleaned_path, "w", encoding="utf-8") as out_file:50            with open(transcription_path, "r", encoding="utf-8") as trans_file:51                lines = trans_file.readlines()52                # print(lines, ' ', len(lines))53                if len(lines) != 0:54                    for line in tqdm(lines):55                        try:56                            utt, spk, language, text = line.strip().split("|")57                            norm_text, phones, tones, word2ph = clean_text(58                                text, language59                            )60                            out_file.write(61                                "{}|{}|{}|{}|{}|{}|{}\n".format(62                                    utt,63                                    spk,64                                    language,65                                    norm_text,66                                    " ".join(phones),67                                    " ".join([str(i) for i in tones]),68                                    " ".join([str(i) for i in word2ph]),69                                )70                            )71                        except Exception as e:72                            print(line)73                            print(f"生成训练集和验证集时发生错误！, 详细信息:\n{e}")74 75    transcription_path = cleaned_path76    spk_utt_map = defaultdict(list)77    spk_id_map = {}78    current_sid = 079 80    with open(transcription_path, "r", encoding="utf-8") as f:81        audioPaths = set()82        countSame = 083        countNotFound = 084        for line in f.readlines():85            utt, spk, language, text, phones, tones, word2ph = line.strip().split("|")86            if utt in audioPaths:87                # 过滤数据集错误：相同的音频匹配多个文本，导致后续bert出问题88                print(f"重复音频文本：{line}")89                countSame += 190                continue91            if not os.path.isfile(utt):92                # 过滤数据集错误：不存在对应音频93                print(f"没有找到对应的音频：{utt}")94                countNotFound += 195                continue96            audioPaths.add(utt)97            spk_utt_map[language].append(line)98            if spk not in spk_id_map.keys():99                spk_id_map[spk] = current_sid100                current_sid += 1101        print(f"总重复音频数：{countSame}，总未找到的音频数:{countNotFound}")102 103    train_list = []104    val_list = []105 106    for spk, utts in spk_utt_map.items():107        shuffle(utts)108        val_list += utts[:val_per_lang]109        train_list += utts[val_per_lang:]110 111    shuffle(val_list)112    if len(val_list) > max_val_total:113        train_list += val_list[max_val_total:]114        val_list = val_list[:max_val_total]115 116    with open(train_path, "w", encoding="utf-8") as f:117        for line in train_list:118            f.write(line)119 120    with open(val_path, "w", encoding="utf-8") as f:121        for line in val_list:122            f.write(line)123 124    json_config = json.load(open(config_path, encoding="utf-8"))125    json_config["data"]["spk2id"] = spk_id_map126    json_config["data"]["n_speakers"] = len(spk_id_map)127    # 新增写入：写入训练版本、数据集路径128    json_config["version"] = latest_version129    json_config["data"]["training_files"] = os.path.normpath(train_path).replace(130        "\\", "/"131    )132    json_config["data"]["validation_files"] = os.path.normpath(val_path).replace(133        "\\", "/"134    )135    with open(config_path, "w", encoding="utf-8") as f:136        json.dump(json_config, f, indent=2, ensure_ascii=False)137    print("训练集和验证集生成完成！")138 139 140if __name__ == "__main__":141    preprocess()142