elderman/ChatPaper
0
1import numpy as np2import os3import re4import datetime5import arxiv6import openai, tenacity7import base64, requests8import argparse9import configparser10import fitz, io, os11from PIL import Image12import gradio13import markdown14import json15import tiktoken16import concurrent.futures17from optimizeOpenAI import chatPaper18 19def parse_text(text):20 lines = text.split("\n")21 for i, line in enumerate(lines):22 if "```" in line:23 items = line.split('`')24 if items[-1]:25 lines[i] = f'<pre><code class="{items[-1]}">'26 else:27 lines[i] = f'</code></pre>'28 else:29 if i > 0:30 line = line.replace("<", "<")31 line = line.replace(">", ">")32 lines[i] = '<br/>' + line.replace(" ", " ")33 return "".join(lines)34 35 36# def get_response(system, context, myKey, raw = False):37# openai.api_key = myKey38# response = openai.ChatCompletion.create(39# model="gpt-3.5-turbo",40# messages=[system, *context],41# )42# openai.api_key = ""43# if raw:44# return response45# else:46# message = response["choices"][0]["message"]["content"]47# message_with_stats = f'{message}'48# return message, parse_text(message_with_stats)49 50valid_api_keys = []51 52 53def api_key_check(api_key):54 try:55 chat = chatPaper([api_key])56 if chat.check_api_available():57 return api_key58 else:59 return None60 except:61 return None62 63 64def valid_apikey(api_keys):65 api_keys = api_keys.replace(' ', '')66 api_key_list = api_keys.split(',')67 print(api_key_list)68 global valid_api_keys69 with concurrent.futures.ThreadPoolExecutor() as executor:70 future_results = {71 executor.submit(api_key_check, api_key): api_key72 for api_key in api_key_list73 }74 for future in concurrent.futures.as_completed(future_results):75 result = future.result()76 if result:77 valid_api_keys.append(result)78 if len(valid_api_keys) > 0:79 return "有效的api-key一共有{}个，分别是：{}, 现在可以提交你的paper".format(80 len(valid_api_keys), valid_api_keys)81 return "无效的api-key"82 83 84class Paper:85 86 def __init__(self, path, title='', url='', abs='', authers=[], sl=[]):87 # 初始化函数，根据pdf路径初始化Paper对象88 self.url = url # 文章链接89 self.path = path # pdf路径90 self.sl = sl91 self.section_names = [] # 段落标题92 self.section_texts = {} # 段落内容93 self.abs = abs94 self.title_page = 095 if title == '':96 self.pdf = fitz.open(self.path) # pdf文档97 self.title = self.get_title()98 self.parse_pdf()99 else:100 self.title = title101 self.authers = authers102 self.roman_num = [103 "I", "II", 'III', "IV", "V", "VI", "VII", "VIII", "IIX", "IX", "X"104 ]105 self.digit_num = [str(d + 1) for d in range(10)]106 self.first_image = ''107 108 def parse_pdf(self):109 self.pdf = fitz.open(self.path) # pdf文档110 self.text_list = [page.get_text() for page in self.pdf]111 self.all_text = ' '.join(self.text_list)112 self.section_page_dict = self._get_all_page_index() # 段落与页码的对应字典113 print("section_page_dict", self.section_page_dict)114 self.section_text_dict = self._get_all_page() # 段落与内容的对应字典115 self.section_text_dict.update({"title": self.title})116 self.section_text_dict.update({"paper_info": self.get_paper_info()})117 self.pdf.close()118 119 def get_paper_info(self):120 first_page_text = self.pdf[self.title_page].get_text()121 if "Abstract" in self.section_text_dict.keys():122 abstract_text = self.section_text_dict['Abstract']123 else:124 abstract_text = self.abs125 introduction_text = self.section_text_dict['Introduction']126 first_page_text = first_page_text.replace(abstract_text, "").replace(127 introduction_text, "")128 return first_page_text129 130 def get_image_path(self, image_path=''):131 """132 将PDF中的第一张图保存到image.png里面，存到本地目录，返回文件名称，供gitee读取133 :param filename: 图片所在路径，"C:\\Users\\Administrator\\Desktop\\nwd.pdf"134 :param image_path: 图片提取后的保存路径135 :return:136 """137 # open file138 max_size = 0139 image_list = []140 with fitz.Document(self.path) as my_pdf_file:141 # 遍历所有页面142 for page_number in range(1, len(my_pdf_file) + 1):143 # 查看独立页面144 page = my_pdf_file[page_number - 1]145 # 查看当前页所有图片146 images = page.get_images()147 # 遍历当前页面所有图片148 for image_number, image in enumerate(page.get_images(),149 start=1):150 # 访问图片xref151 xref_value = image[0]152 # 提取图片信息153 base_image = my_pdf_file.extract_image(xref_value)154 # 访问图片155 image_bytes = base_image["image"]156 # 获取图片扩展名157 ext = base_image["ext"]158 # 加载图片159 image = Image.open(io.BytesIO(image_bytes))160 image_size = image.size[0] * image.size[1]161 if image_size > max_size:162 max_size = image_size163 image_list.append(image)164 for image in image_list:165 image_size = image.size[0] * image.size[1]166 if image_size == max_size:167 image_name = f"image.{ext}"168 im_path = os.path.join(image_path, image_name)169 print("im_path:", im_path)170 171 max_pix = 480172 origin_min_pix = min(image.size[0], image.size[1])173 174 if image.size[0] > image.size[1]:175 min_pix = int(image.size[1] * (max_pix / image.size[0]))176 newsize = (max_pix, min_pix)177 else:178 min_pix = int(image.size[0] * (max_pix / image.size[1]))179 newsize = (min_pix, max_pix)180 image = image.resize(newsize)181 182 image.save(open(im_path, "wb"))183 return im_path, ext184 return None, None185 186 # 定义一个函数，根据字体的大小，识别每个章节名称，并返回一个列表187 def get_chapter_names(self, ):188 # # 打开一个pdf文件189 doc = fitz.open(self.path) # pdf文档190 text_list = [page.get_text() for page in doc]191 all_text = ''192 for text in text_list:193 all_text += text194 # # 创建一个空列表，用于存储章节名称195 chapter_names = []196 for line in all_text.split('\n'):197 line_list = line.split(' ')198 if '.' in line:199 point_split_list = line.split('.')200 space_split_list = line.split(' ')201 if 1 < len(space_split_list) < 5:202 if 1 < len(point_split_list) < 5 and (203 point_split_list[0] in self.roman_num204 or point_split_list[0] in self.digit_num):205 print("line:", line)206 chapter_names.append(line)207 208 return chapter_names209 210 def get_title(self):211 doc = self.pdf # 打开pdf文件212 max_font_size = 0 # 初始化最大字体大小为0213 max_string = "" # 初始化最大字体大小对应的字符串为空214 max_font_sizes = [0]215 for page_index, page in enumerate(doc): # 遍历每一页216 text = page.get_text("dict") # 获取页面上的文本信息217 blocks = text["blocks"] # 获取文本块列表218 for block in blocks: # 遍历每个文本块219 if block["type"] == 0 and len(block['lines']): # 如果是文字类型220 if len(block["lines"][0]["spans"]):221 font_size = block["lines"][0]["spans"][0][222 "size"] # 获取第一行第一段文字的字体大小223 max_font_sizes.append(font_size)224 if font_size > max_font_size: # 如果字体大小大于当前最大值225 max_font_size = font_size # 更新最大值226 max_string = block["lines"][0]["spans"][0][227 "text"] # 更新最大值对应的字符串228 max_font_sizes.sort()229 print("max_font_sizes", max_font_sizes[-10:])230 cur_title = ''231 for page_index, page in enumerate(doc): # 遍历每一页232 text = page.get_text("dict") # 获取页面上的文本信息233 blocks = text["blocks"] # 获取文本块列表234 for block in blocks: # 遍历每个文本块235 if block["type"] == 0 and len(block['lines']): # 如果是文字类型236 if len(block["lines"][0]["spans"]):237 cur_string = block["lines"][0]["spans"][0][238 "text"] # 更新最大值对应的字符串239 font_flags = block["lines"][0]["spans"][0][240 "flags"] # 获取第一行第一段文字的字体特征241 font_size = block["lines"][0]["spans"][0][242 "size"] # 获取第一行第一段文字的字体大小243 # print(font_size)244 if abs(font_size - max_font_sizes[-1]) < 0.3 or abs(245 font_size - max_font_sizes[-2]) < 0.3:246 # print("The string is bold.", max_string, "font_size:", font_size, "font_flags:", font_flags)247 if len(cur_string248 ) > 4 and "arXiv" not in cur_string:249 # print("The string is bold.", max_string, "font_size:", font_size, "font_flags:", font_flags)250 if cur_title == '':251 cur_title += cur_string252 else:253 cur_title += ' ' + cur_string254 self.title_page = page_index255 256 title = cur_title.replace('\n', ' ')257 return title258 259 def _get_all_page_index(self):260 # 定义需要寻找的章节名称列表261 section_list = self.sl262 # 初始化一个字典来存储找到的章节和它们在文档中出现的页码263 section_page_dict = {}264 # 遍历每一页文档265 for page_index, page in enumerate(self.pdf):266 # 获取当前页面的文本内容267 cur_text = page.get_text()268 # 遍历需要寻找的章节名称列表269 for section_name in section_list:270 # 将章节名称转换成大写形式271 section_name_upper = section_name.upper()272 # 如果当前页面包含"Abstract"这个关键词273 if "Abstract" == section_name and section_name in cur_text:274 # 将"Abstract"和它所在的页码加入字典中275 section_page_dict[section_name] = page_index276 # 如果当前页面包含章节名称，则将章节名称和它所在的页码加入字典中277 else:278 if section_name + '\n' in cur_text:279 section_page_dict[section_name] = page_index280 elif section_name_upper + '\n' in cur_text:281 section_page_dict[section_name] = page_index282 # 返回所有找到的章节名称及它们在文档中出现的页码283 return section_page_dict284 285 def _get_all_page(self):286 """287 获取PDF文件中每个页面的文本信息，并将文本信息按照章节组织成字典返回。288 Returns:289 section_dict (dict): 每个章节的文本信息字典，key为章节名，value为章节文本。290 """291 text = ''292 text_list = []293 section_dict = {}294 295 # 再处理其他章节：296 text_list = [page.get_text() for page in self.pdf]297 for sec_index, sec_name in enumerate(self.section_page_dict):298 print(sec_index, sec_name, self.section_page_dict[sec_name])299 if sec_index <= 0 and self.abs:300 continue301 else:302 # 直接考虑后面的内容：303 start_page = self.section_page_dict[sec_name]304 if sec_index < len(list(self.section_page_dict.keys())) - 1:305 end_page = self.section_page_dict[list(306 self.section_page_dict.keys())[sec_index + 1]]307 else:308 end_page = len(text_list)309 print("start_page, end_page:", start_page, end_page)310 cur_sec_text = ''311 if end_page - start_page == 0:312 if sec_index < len(list(313 self.section_page_dict.keys())) - 1:314 next_sec = list(315 self.section_page_dict.keys())[sec_index + 1]316 if text_list[start_page].find(sec_name) == -1:317 start_i = text_list[start_page].find(318 sec_name.upper())319 else:320 start_i = text_list[start_page].find(sec_name)321 if text_list[start_page].find(next_sec) == -1:322 end_i = text_list[start_page].find(323 next_sec.upper())324 else:325 end_i = text_list[start_page].find(next_sec)326 cur_sec_text += text_list[start_page][start_i:end_i]327 else:328 for page_i in range(start_page, end_page):329 # print("page_i:", page_i)330 if page_i == start_page:331 if text_list[start_page].find(sec_name) == -1:332 start_i = text_list[start_page].find(333 sec_name.upper())334 else:335 start_i = text_list[start_page].find(sec_name)336 cur_sec_text += text_list[page_i][start_i:]337 elif page_i < end_page:338 cur_sec_text += text_list[page_i]339 elif page_i == end_page:340 if sec_index < len(341 list(self.section_page_dict.keys())) - 1:342 next_sec = list(343 self.section_page_dict.keys())[sec_index +344 1]345 if text_list[start_page].find(next_sec) == -1:346 end_i = text_list[start_page].find(347 next_sec.upper())348 else:349 end_i = text_list[start_page].find(350 next_sec)351 cur_sec_text += text_list[page_i][:end_i]352 section_dict[sec_name] = cur_sec_text.replace('-\n',353 '').replace(354 '\n', ' ')355 return section_dict356 357 358# 定义Reader类359class Reader:360 # 初始化方法，设置属性361 def __init__(self,362 key_word='',363 query='',364 filter_keys='',365 root_path='./',366 gitee_key='',367 sort=arxiv.SortCriterion.SubmittedDate,368 user_name='defualt',369 language='cn',370 api_keys: list = [],371 model_name="gpt-3.5-turbo",372 p=1.0,373 temperature=1.0):374 self.api_keys = api_keys375 self.chatPaper = chatPaper(api_keys=self.api_keys,376 apiTimeInterval=10,377 temperature=temperature,378 top_p=p,379 model_name=model_name) #openAI api封装380 self.user_name = user_name # 读者姓名381 self.key_word = key_word # 读者感兴趣的关键词382 self.query = query # 读者输入的搜索查询383 self.sort = sort # 读者选择的排序方式384 self.language = language # 读者选择的语言385 self.filter_keys = filter_keys # 用于在摘要中筛选的关键词386 self.root_path = root_path387 self.file_format = 'md' # or 'txt'，如果为图片，则必须为'md'388 self.save_image = False389 if self.save_image:390 self.gitee_key = self.config.get('Gitee', 'api')391 else:392 self.gitee_key = ''393 self.max_token_num = 4096394 self.encoding = tiktoken.get_encoding("gpt2")395 396 def get_arxiv(self, max_results=30):397 search = arxiv.Search(398 query=self.query,399 max_results=max_results,400 sort_by=self.sort,401 sort_order=arxiv.SortOrder.Descending,402 )403 return search404 405 def filter_arxiv(self, max_results=30):406 search = self.get_arxiv(max_results=max_results)407 print("all search:")408 for index, result in enumerate(search.results()):409 print(index, result.title, result.updated)410 411 filter_results = []412 filter_keys = self.filter_keys413 414 print("filter_keys:", self.filter_keys)415 # 确保每个关键词都能在摘要中找到，才算是目标论文416 for index, result in enumerate(search.results()):417 abs_text = result.summary.replace('-\n', '-').replace('\n', ' ')418 meet_num = 0419 for f_key in filter_keys.split(" "):420 if f_key.lower() in abs_text.lower():421 meet_num += 1422 if meet_num == len(filter_keys.split(" ")):423 filter_results.append(result)424 # break425 print("filter_results:", len(filter_results))426 print("filter_papers:")427 for index, result in enumerate(filter_results):428 print(index, result.title, result.updated)429 return filter_results430 431 def validateTitle(self, title):432 # 将论文的乱七八糟的路径格式修正433 rstr = r"[\/\\\:\*\?\"\<\>\|]" # '/ \ : * ? " < > |'434 new_title = re.sub(rstr, "_", title) # 替换为下划线435 return new_title436 437 def download_pdf(self, filter_results):438 # 先创建文件夹439 date_str = str(datetime.datetime.now())[:13].replace(' ', '-')440 key_word = str(self.key_word.replace(':', ' '))441 path = self.root_path + 'pdf_files/' + self.query.replace(442 'au: ', '').replace('title: ', '').replace('ti: ', '').replace(443 ':', ' ')[:25] + '-' + date_str444 try:445 os.makedirs(path)446 except:447 pass448 print("All_paper:", len(filter_results))449 # 开始下载：450 paper_list = []451 for r_index, result in enumerate(filter_results):452 try:453 title_str = self.validateTitle(result.title)454 pdf_name = title_str + '.pdf'455 # result.download_pdf(path, filename=pdf_name)456 self.try_download_pdf(result, path, pdf_name)457 paper_path = os.path.join(path, pdf_name)458 print("paper_path:", paper_path)459 paper = Paper(460 path=paper_path,461 url=result.entry_id,462 title=result.title,463 abs=result.summary.replace('-\n', '-').replace('\n', ' '),464 authers=[str(aut) for aut in result.authors],465 )466 # 下载完毕，开始解析：467 paper.parse_pdf()468 paper_list.append(paper)469 except Exception as e:470 print("download_error:", e)471 pass472 return paper_list473 474 @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=4,475 max=10),476 stop=tenacity.stop_after_attempt(5),477 reraise=True)478 def try_download_pdf(self, result, path, pdf_name):479 result.download_pdf(path, filename=pdf_name)480 481 @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=4,482 max=10),483 stop=tenacity.stop_after_attempt(5),484 reraise=True)485 def upload_gitee(self, image_path, image_name='', ext='png'):486 """487 上传到码云488 :return:489 """490 with open(image_path, 'rb') as f:491 base64_data = base64.b64encode(f.read())492 base64_content = base64_data.decode()493 494 date_str = str(datetime.datetime.now())[:19].replace(':', '-').replace(495 ' ', '-') + '.' + ext496 path = image_name + '-' + date_str497 498 payload = {499 "access_token": self.gitee_key,500 "owner": self.config.get('Gitee', 'owner'),501 "repo": self.config.get('Gitee', 'repo'),502 "path": self.config.get('Gitee', 'path'),503 "content": base64_content,504 "message": "upload image"505 }506 # 这里需要修改成你的gitee的账户和仓库名，以及文件夹的名字：507 url = f'https://gitee.com/api/v5/repos/' + self.config.get(508 'Gitee', 'owner') + '/' + self.config.get(509 'Gitee', 'repo') + '/contents/' + self.config.get(510 'Gitee', 'path') + '/' + path511 rep = requests.post(url, json=payload).json()512 print("rep:", rep)513 if 'content' in rep.keys():514 image_url = rep['content']['download_url']515 else:516 image_url = r"https://gitee.com/api/v5/repos/" + self.config.get(517 'Gitee', 'owner') + '/' + self.config.get(518 'Gitee', 'repo') + '/contents/' + self.config.get(519 'Gitee', 'path') + '/' + path520 521 return image_url522 523 524 def summary_with_chat(self, paper_list):525 htmls = []526 utoken = 0527 ctoken = 0528 ttoken = 0529 for paper_index, paper in enumerate(paper_list):530 # 第一步先用title，abs，和introduction进行总结。531 text = ''532 text += 'Title:' + paper.title533 text += 'Url:' + paper.url534 text += 'Abstrat:' + paper.abs535 text += 'Paper_info:' + paper.section_text_dict['paper_info']536 # intro537 text += list(paper.section_text_dict.values())[0]538 #max_token = 2500 * 4539 #text = text[:max_token]540 chat_summary_text, utoken1, ctoken1, ttoken1 = self.chat_summary(541 text=text)542 htmls.append(chat_summary_text)543 544 # TODO 往md文档中插入论文里的像素最大的一张图片，这个方案可以弄的更加智能一些：545 method_key = ''546 for parse_key in paper.section_text_dict.keys():547 if 'method' in parse_key.lower(548 ) or 'approach' in parse_key.lower():549 method_key = parse_key550 break551 552 if method_key != '':553 text = ''554 method_text = ''555 summary_text = ''556 summary_text += "<summary>" + chat_summary_text557 # methods558 method_text += paper.section_text_dict[method_key]559 text = summary_text + "\n<Methods>:\n" + method_text560 chat_method_text, utoken2, ctoken2, ttoken2 = self.chat_method(561 text=text)562 else:563 chat_method_text = ''564 htmls.append(chat_method_text)565 htmls.append("\n")566 567 # 第三步总结全文，并打分：568 conclusion_key = ''569 for parse_key in paper.section_text_dict.keys():570 if 'conclu' in parse_key.lower():571 conclusion_key = parse_key572 break573 574 text = ''575 conclusion_text = ''576 summary_text = ''577 summary_text += "<summary>" + chat_summary_text + "\n <Method summary>:\n" + chat_method_text578 if conclusion_key != '':579 # conclusion580 conclusion_text += paper.section_text_dict[conclusion_key]581 text = summary_text + "\n <Conclusion>:\n" + conclusion_text582 else:583 text = summary_text584 chat_conclusion_text, utoken3, ctoken3, ttoken3 = self.chat_conclusion(585 text=text)586 htmls.append(chat_conclusion_text)587 htmls.append("\n")588 # token统计589 utoken = utoken + utoken1 + utoken2 + utoken3590 ctoken = ctoken + ctoken1 + ctoken2 + ctoken3591 ttoken = ttoken + ttoken1 + ttoken2 + ttoken3592 cost = (ttoken / 1000) * 0.002593 pos_count = {594 "usage_token_used": str(utoken),595 "completion_token_used": str(ctoken),596 "total_token_used": str(ttoken),597 "cost": str(cost),598 }599 md_text = "\n".join(htmls)600 601 #with open(os.path.join('./', 'output.md'), "w", encoding="utf8") as f:602 # f.write(md_text)603 604 return markdown.markdown(md_text), pos_count # , os.path.join('./', 'output.md')605 606 @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=4,607 max=10),608 stop=tenacity.stop_after_attempt(5),609 reraise=True)610 def chat_conclusion(self, text):611 conclusion_prompt_token = 650612 text_token = len(self.encoding.encode(text))613 clip_text_index = int(614 len(text) * (self.max_token_num - conclusion_prompt_token) /615 text_token)616 clip_text = text[:clip_text_index]617 self.chatPaper.reset(618 convo_id="chatConclusion",619 system_prompt="You are a reviewer in the field of [" +620 self.key_word + "] and you need to critically review this article")621 self.chatPaper.add_to_conversation(622 convo_id="chatConclusion",623 role="assistant",624 message=625 "This is the <summary> and <conclusion> part of an English literature, where <summary> you have already summarized, but <conclusion> part, I need your help to summarize the following questions:"626 + clip_text) # 背景知识，可以参考OpenReview的审稿流程627 content = """ 628 8. Make the following summary.Be sure to use Chinese answers (proper nouns need to be marked in English).629 - (1):What is the significance of this piece of work?630 - (2):Summarize the strengths and weaknesses of this article in three dimensions: innovation point, performance, and workload. 631 .......632 Follow the format of the output later: 633 8. Conclusion: \n\n634 - (1):xxx;\n 635 - (2):Innovation point: xxx; Performance: xxx; Workload: xxx;\n 636 637 Be sure to use Chinese answers (proper nouns need to be marked in English), statements as concise and academic as possible, do not repeat the content of the previous <summary>, the value of the use of the original numbers, be sure to strictly follow the format, the corresponding content output to xxx, in accordance with \n line feed, ....... means fill in according to the actual requirements, if not, you can not write. 638 """639 result = self.chatPaper.ask(640 prompt=content,641 role="user",642 convo_id="chatConclusion",643 )644 print(result)645 return result[0], result[1], result[2], result[3]646 647 @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=4,648 max=10),649 stop=tenacity.stop_after_attempt(5),650 reraise=True)651 def chat_method(self, text):652 method_prompt_token = 650653 text_token = len(self.encoding.encode(text))654 clip_text_index = int(655 len(text) * (self.max_token_num - method_prompt_token) /656 text_token)657 clip_text = text[:clip_text_index]658 self.chatPaper.reset(659 convo_id="chatMethod",660 system_prompt="You are a researcher in the field of [" +661 self.key_word +662 "] who is good at summarizing papers using concise statements"663 ) # chatgpt 角色664 self.chatPaper.add_to_conversation(665 convo_id="chatMethod",666 role="assistant",667 message=str(668 "This is the <summary> and <Method> part of an English document, where <summary> you have summarized, but the <Methods> part, I need your help to read and summarize the following questions."669 + clip_text))670 content = """ 671 7. Describe in detail the methodological idea of this article. Be sure to use Chinese answers (proper nouns need to be marked in English). For example, its steps are.672 - (1):...673 - (2):...674 - (3):...675 - .......676 Follow the format of the output that follows: 677 7. Methods: \n\n678 - (1):xxx;\n 679 - (2):xxx;\n 680 - (3):xxx;\n 681 ....... \n\n 682 683 Be sure to use Chinese answers (proper nouns need to be marked in English), statements as concise and academic as possible, do not repeat the content of the previous <summary>, the value of the use of the original numbers, be sure to strictly follow the format, the corresponding content output to xxx, in accordance with \n line feed, ....... means fill in according to the actual requirements, if not, you can not write. 684 """685 result = self.chatPaper.ask(686 prompt=content,687 role="user",688 convo_id="chatMethod",689 )690 print(result)691 return result[0], result[1], result[2], result[3]692 693 @tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, min=4,694 max=10),695 stop=tenacity.stop_after_attempt(5),696 reraise=True)697 def chat_summary(self, text):698 summary_prompt_token = 1000699 text_token = len(self.encoding.encode(text))700 clip_text_index = int(701 len(text) * (self.max_token_num - summary_prompt_token) /702 text_token)703 clip_text = text[:clip_text_index]704 self.chatPaper.reset(705 convo_id="chatSummary",706 system_prompt="You are a researcher in the field of [" +707 self.key_word +708 "] who is good at summarizing papers using concise statements")709 self.chatPaper.add_to_conversation(710 convo_id="chatSummary",711 role="assistant",712 message=str(713 "This is the title, author, link, abstract and introduction of an English document. I need your help to read and summarize the following questions: "714 + clip_text))715 content = """ 716 1. Mark the title of the paper (with Chinese translation)717 2. list all the authors' names (use English)718 3. mark the first author's affiliation (output Chinese translation only) 719 4. mark the keywords of this article (use English)720 5. link to the paper, Github code link (if available, fill in Github:None if not)721 6. summarize according to the following four points.Be sure to use Chinese answers (proper nouns need to be marked in English)722 - (1):What is the research background of this article?723 - (2):What are the past methods? What are the problems with them? Is the approach well motivated?724 - (3):What is the research methodology proposed in this paper?725 - (4):On what task and what performance is achieved by the methods in this paper? Can the performance support their goals?726 Follow the format of the output that follows: 727 1. Title: xxx\n\n728 2. Authors: xxx\n\n729 3. Affiliation: xxx\n\n 730 4. Keywords: xxx\n\n 731 5. Urls: xxx or xxx , xxx \n\n 732 6. Summary: \n\n733 - (1):xxx;\n 734 - (2):xxx;\n 735 - (3):xxx;\n 736 - (4):xxx.\n\n 737 738 Be sure to use Chinese answers (proper nouns need to be marked in English), statements as concise and academic as possible, do not have too much repetitive information, numerical values using the original numbers, be sure to strictly follow the format, the corresponding content output to xxx, in accordance with \n line feed. 739 """740 result = self.chatPaper.ask(741 prompt=content,742 role="user",743 convo_id="chatSummary",744 )745 print(result)746 return result[0], result[1], result[2], result[3]747 748 def export_to_markdown(self, text, file_name, mode='w'):749 # 使用markdown模块的convert方法，将文本转换为html格式750 # html = markdown.markdown(text)751 # 打开一个文件，以写入模式752 with open(file_name, mode, encoding="utf-8") as f:753 # 将html格式的内容写入文件754 f.write(text)755 756 # 定义一个方法，打印出读者信息757 def show_info(self):758 print(f"Key word: {self.key_word}")759 print(f"Query: {self.query}")760 print(f"Sort: {self.sort}")761 762 763def upload_pdf(api_keys, text, model_name, p, temperature, file):764 # 检查两个输入都不为空765 api_key_list = None766 if api_keys:767 api_key_list = api_keys.split(',')768 elif not api_keys and valid_api_keys != []:769 api_key_list = valid_api_keys770 if not text or not file or not api_key_list:771 return "两个输入都不能为空，请输入字符并上传 PDF 文件！"772 773 # 判断PDF文件774 #if file and file.name.split(".")[-1].lower() != "pdf":775 # return '请勿上传非 PDF 文件！'776 else:777 section_list = text.split(',')778 paper_list = [Paper(path=file, sl=section_list)]779 # 创建一个Reader对象780 print(api_key_list)781 reader = Reader(api_keys=api_key_list,782 model_name=model_name,783 p=p,784 temperature=temperature)785 sum_info, cost = reader.summary_with_chat(786 paper_list=paper_list) # type: ignore787 return cost, sum_info788 789 790api_title = "api-key可用验证"791api_description = '''<div align='left'>792 793<img src='https://visitor-badge.laobi.icu/badge?page_id=https://huggingface.co/spaces/wangrongsheng/ChatPaper'>794 795<img align='right' src='https://i.328888.xyz/2023/03/12/vH9dU.png' width="150">796 797使用卡顿？请Fork到自己的Space，轻松使用：<a href="https://huggingface.co/spaces/wangrongsheng/ChatPaper?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a>798 799💥💥💥<strong>面向全球，服务万千科研人的ChatPaper在线版正式上线：<a href="https://chatpaper.org/">https://chatpaper.org/</a> </strong>💥💥💥800 801Use ChatGPT to summary the papers.Star our Github [🌟ChatPaper](https://github.com/kaixindelele/ChatPaper) .802 803💗如果您觉得我们的项目对您有帮助，还请您给我们一些鼓励！💗804 805🔴请注意：千万不要用于严肃的学术场景，只能用于论文阅读前的初筛！806 807使用卡顿？请点击右上角<strong>Duplicate this Space</strong> 项目！808 809</div>810'''811 812api_input = [813 gradio.inputs.Textbox(label="请输入你的API-key(必填, 多个API-key请用英文逗号隔开)",814 default="",815 # default="",816 type='password')817]818api_gui = gradio.Interface(fn=valid_apikey,819 inputs=api_input,820 outputs="text",821 title=api_title,822 description=api_description)823 824# 标题825title = "ChatPaper"826# 描述827description = '''<div align='left'>828 829<img src='https://visitor-badge.laobi.icu/badge?page_id=https://huggingface.co/spaces/wangrongsheng/ChatPaper'>830 831<img align='right' src='https://i.328888.xyz/2023/03/12/vH9dU.png' width="150">832 833使用卡顿？请Fork到自己的Space，轻松使用：<a href="https://huggingface.co/spaces/wangrongsheng/ChatPaper?duplicate=true"><img src="https://bit.ly/3gLdBN6" alt="Duplicate Space"></a>834 835💥💥💥<strong>面向全球，服务万千科研人的ChatPaper在线版正式上线：<a href="https://chatpaper.org/">https://chatpaper.org/</a> </strong>💥💥💥836 837Use ChatGPT to summary the papers.Star our Github [🌟ChatPaper](https://github.com/kaixindelele/ChatPaper) .838 839💗如果您觉得我们的项目对您有帮助，还请您给我们一些鼓励！💗840 841🔴请注意：千万不要用于严肃的学术场景，只能用于论文阅读前的初筛！842 843使用卡顿？请点击右上角<strong>Duplicate this Space</strong> 项目！844 845</div>846'''847# 创建Gradio界面848ip = [849 gradio.inputs.Textbox(label="请输入你的API-key(必填, 多个API-key请用英文逗号隔开),不需要空格",850 default="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",851 type='password'),852 gradio.inputs.Textbox(853 label="请输入论文大标题索引(用英文逗号隔开,必填)",854 default=855 "'Abstract,Introduction,Related Work,Background,Preliminary,Problem Formulation,Methods,Methodology,Method,Approach,Approaches,Materials and Methods,Experiment Settings,Experiment,Experimental Results,Evaluation,Experiments,Results,Findings,Data Analysis,Discussion,Results and Discussion,Conclusion,References'"856 ),857 gradio.inputs.Radio(choices=["gpt-3.5-turbo", "gpt-3.5-turbo-0301"],858 default="gpt-3.5-turbo",859 label="Select model"),860 gradio.inputs.Slider(minimum=-0,861 maximum=1.0,862 default=1.0,863 step=0.05,864 label="Top-p (nucleus sampling)"),865 gradio.inputs.Slider(minimum=-0,866 maximum=5.0,867 default=0.5,868 step=0.5,869 label="Temperature"),870 gradio.inputs.File(label="请上传论文PDF(必填)")871]872 873chatpaper_gui = gradio.Interface(fn=upload_pdf,874 inputs=ip,875 outputs=["json", "html"],876 title=title,877 description=description)878 879# Start server880gui = gradio.TabbedInterface(interface_list=[api_gui, chatpaper_gui],881 tab_names=["API-key", "ChatPaper"])882gui.launch(quiet=True, show_api=False)883