MapleGu/ChuanhuChatGPT

1from __future__ import annotations2import logging3 4from llama_index import Prompt5from typing import List, Tuple6import mdtex2html7 8from presets import *9from llama_func import *10 11 12def compact_text_chunks(self, prompt: Prompt, text_chunks: List[str]) -> List[str]:13    logging.debug("Compacting text chunks...🚀🚀🚀")14    combined_str = [c.strip() for c in text_chunks if c.strip()]15    combined_str = [f"[{index+1}] {c}" for index, c in enumerate(combined_str)]16    combined_str = "\n\n".join(combined_str)17    # resplit based on self.max_chunk_overlap18    text_splitter = self.get_text_splitter_given_prompt(prompt, 1, padding=1)19    return text_splitter.split_text(combined_str)20 21 22def postprocess(23    self, y: List[Tuple[str | None, str | None]]24) -> List[Tuple[str | None, str | None]]:25    """26    Parameters:27        y: List of tuples representing the message and response pairs. Each message and response should be a string, which may be in Markdown format.28    Returns:29        List of tuples representing the message and response. Each message and response will be a string of HTML.30    """31    if y is None or y == []:32        return []33    tag_regex = re.compile(r"^<\w+>[^<]+</\w+>")34    if tag_regex.search(y[-1][1]):35        y[-1] = (y[-1][0].replace("\n", "<br>"), y[-1][1])36    else:37        y[-1] = (y[-1][0].replace("\n", "<br>"), convert_mdtext(y[-1][1]))38    return y39