lenML/ChatTTS-Forge

1import re2 3import zhon4 5from modules.models import get_tokenizer6from modules.utils.detect_lang import guess_lang7 8 9# 解析文本 并根据停止符号分割成句子10# 可以设置最大阈值，即如果分割片段小于这个阈值会与下一段合并11class SentenceSplitter:12    SEP_TOKEN = " "13 14    def __init__(self, threshold=100):15        assert (16            isinstance(threshold, int) and threshold > 017        ), "Threshold must be greater than 0."18 19        self.sentence_threshold = threshold20        self.tokenizer = get_tokenizer()21 22    def count_tokens(self, text: str):23        return len(self.tokenizer.tokenize(text))24 25    def parse(self, text: str):26        sentences = self.split_paragraph(text)27        sentences = self.merge_text_by_threshold(sentences)28 29        return sentences30 31    def merge_text_by_threshold(self, setences: list[str]):32        """33        Merge text by threshold.34 35        If the length of the text is less than the threshold, merge it with the next text.36        """37        merged_sentences: list[str] = []38        temp_sentence = ""39        for sentence in setences:40            if len(temp_sentence) + len(sentence) < self.sentence_threshold:41                temp_sentence += SentenceSplitter.SEP_TOKEN + sentence42            else:43                merged_sentences.append(temp_sentence)44                temp_sentence = sentence45 46        if temp_sentence:47            merged_sentences.append(temp_sentence)48        return merged_sentences49 50    def split_paragraph(self, text: str):51        """52        Split text into sentences.53        """54        lines = text.split("\n")55        sentences: list[str] = []56        for line in lines:57            if self.is_eng_sentence(line):58                sentences.extend(self.split_en_sentence(line))59            else:60                sentences.extend(self.split_zhon_sentence(line))61        return sentences62 63    def is_eng_sentence(self, text: str):64        return guess_lang(text) == "en"65 66    def split_en_sentence(self, text: str):67        """68        Split English text into sentences.69        """70        pattern = re.compile(r"(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?|\!)\s")71        sentences = pattern.split(text)72 73        sentences = [sentence.strip() for sentence in sentences if sentence.strip()]74 75        return sentences76 77    def split_zhon_sentence(self, text: str):78        """79        Split Chinese text into sentences.80        """81        sentences: list[str] = []82        pattern = re.compile(zhon.hanzi.sentence)83        start = 084        for match in pattern.finditer(text):85            end = match.end()86            sentences.append(text[start:end])87            start = end88 89        if start < len(text):90            sentences.append(text[start:])91 92        sentences = [t for t in sentences if t.strip()]93        return sentences94 95 96if __name__ == "__main__":97    max_threshold = 10098    parser = SentenceSplitter(max_threshold)99    text = """100中华美食，作为世界饮食文化的瑰宝，以其丰富的种类、独特的风味和精湛的烹饪技艺而闻名于世。中国地大物博，各地区的饮食习惯和烹饪方法各具特色，形成了独树一帜的美食体系。从北方的京鲁菜、东北菜，到南方的粤菜、闽菜，无不展现出中华美食的多样性。101 102在中华美食的世界里，五味调和，色香味俱全。无论是辣味浓郁的川菜，还是清淡鲜美的淮扬菜，都能够满足不同人的口味需求。除了味道上的独特，中华美食还注重色彩的搭配和形态的美感，让每一道菜品不仅是味觉的享受，更是一场视觉的盛宴。103 104中华美食不仅仅是食物，更是一种文化的传承。每一道菜背后都有着深厚的历史背景和文化故事。比如，北京的烤鸭，代表着皇家气派；而西安的羊肉泡馍，则体现了浓郁的地方风情。中华美食的精髓在于它追求的“天人合一”，讲究食材的自然性和烹饪过程中的和谐。105 106总之，中华美食博大精深，其丰富的口感和多样的烹饪技艺，构成了一个充满魅力和无限可能的美食世界。无论你来自哪里，都会被这独特的美食文化所吸引和感动。107    """108    result = parser.parse(text)109    for idx, sentence in enumerate(result):110        print(f"Sentence {idx + 1}: {sentence}")111