llmbb/LLMBB-Agent

1import re2 3from agent.utils.tokenization_qwen import count_tokens4 5 6def rm_newlines(text):7    text = re.sub(r'(?<=[^\.。:：])\n', ' ', text)8    return text9 10 11def rm_cid(text):12    text = re.sub(r'\(cid:\d+\)', '', text)13    return text14 15 16def rm_hexadecimal(text):17    text = re.sub(r'[0-9A-Fa-f]{21,}', '', text)18    return text19 20 21def deal(text):22    text = rm_newlines(text)23    text = rm_cid(text)24    text = rm_hexadecimal(text)25    return text26 27 28def parse_doc(path):29    if '.pdf' in path.lower():30        from langchain.document_loaders import PDFMinerLoader31        loader = PDFMinerLoader(path)32        pages = loader.load_and_split()33    elif '.docx' in path.lower():34        from langchain.document_loaders import Docx2txtLoader35        loader = Docx2txtLoader(path)36        pages = loader.load_and_split()37    elif '.pptx' in path.lower():38        from langchain.document_loaders import UnstructuredPowerPointLoader39        loader = UnstructuredPowerPointLoader(path)40        pages = loader.load_and_split()41    else:42        from langchain.document_loaders import UnstructuredFileLoader43        loader = UnstructuredFileLoader(path)44        pages = loader.load_and_split()45 46    res = []47    for page in pages:48        dealed_page_content = deal(page.page_content)49        res.append({50            'page_content': dealed_page_content,51            'token': count_tokens(dealed_page_content),52            'metadata': page.metadata53        })54 55    return res56 57 58def pre_process_html(s):59    # replace multiple newlines60    s = re.sub('\n+', '\n', s)61    # replace special string62    s = s.replace("Add to Qwen's Reading List", '')63    return s64 65 66def parse_html_bs(path):67    from langchain.document_loaders import BSHTMLLoader68 69    loader = BSHTMLLoader(path, open_encoding='utf-8')70    pages = loader.load_and_split()71    res = []72    for page in pages:73        dealed_page_content = pre_process_html(page.page_content)74        res.append({75            'page_content': dealed_page_content,76            'token': count_tokens(dealed_page_content),77            'metadata': page.metadata78        })79 80    return res81