fl399/deplot_plus_llm
47
1import os 2import torch3import openai4import requests5import gradio as gr6import transformers7from transformers import Pix2StructForConditionalGeneration, Pix2StructProcessor8#from peft import PeftModel9 10 11if torch.cuda.is_available():12 device = "cuda"13else:14 device = "cpu"15 16try:17 if torch.backends.mps.is_available():18 device = "mps"19except:20 pass21 22## CoT prompts23 24def _add_markup(table):25 try:26 parts = [p.strip() for p in table.splitlines(keepends=False)]27 if parts[0].startswith('TITLE'):28 result = f"Title: {parts[0].split(' | ')[1].strip()}\n"29 rows = parts[1:]30 else:31 result = ''32 rows = parts33 prefixes = ['Header: '] + [f'Row {i+1}: ' for i in range(len(rows) - 1)]34 return result + '\n'.join(prefix + row for prefix, row in zip(prefixes, rows))35 except:36 # just use the raw table if parsing fails37 return table38 39_TABLE = """Year | Democrats | Republicans | Independents402004 | 68.1% | 45.0% | 53.0%412006 | 58.0% | 42.0% | 53.0%422007 | 59.0% | 38.0% | 45.0%432009 | 72.0% | 49.0% | 60.0%442011 | 71.0% | 51.2% | 58.0%452012 | 70.0% | 48.0% | 53.0%462013 | 72.0% | 41.0% | 60.0%"""47 48_INSTRUCTION = 'Read the table below to answer the following questions.'49 50_TEMPLATE = f"""First read an example then the complete question for the second table.51------------52{_INSTRUCTION}53{_add_markup(_TABLE)}54Q: In which year republicans have the lowest favor rate?55A: Let's find the column of republicans. Then let's extract the favor rates, they [45.0, 42.0, 38.0, 49.0, 51.2, 48.0, 41.0]. The smallest number is 38.0, that's Row 3. Row 3 is year 2007. The answer is 2007.56Q: What is the sum of Democrats' favor rates of 2004, 2012, and 2013?57A: Let's find the rows of years 2004, 2012, and 2013. We find Row 1, 6, 7. The favor dates of Demoncrats on that 3 rows are 68.1, 70.0, and 72.0. 68.1+70.0+72=210.1. The answer is 210.1.58Q: By how many points do Independents surpass Republicans in the year of 2011?59A: Let's find the row with year = 2011. We find Row 5. We extract Independents and Republicans' numbers. They are 58.0 and 51.2. 58.0-51.2=6.8. The answer is 6.8.60Q: Which group has the overall worst performance?61A: Let's sample a couple of years. In Row 1, year 2004, we find Republicans having the lowest favor rate 45.0 (since 45.0<68.1, 45.0<53.0). In year 2006, Row 2, we find Republicans having the lowest favor rate 42.0 (42.0<58.0, 42.0<53.0). The trend continues to other years. The answer is Republicans.62Q: Which party has the second highest favor rates in 2007?63A: Let's find the row of year 2007, that's Row 3. Let's extract the numbers on Row 3: [59.0, 38.0, 45.0]. 45.0 is the second highest. 45.0 is the number of Independents. The answer is Independents.64{_INSTRUCTION}"""65 66 67## alpaca-lora68 69# assert (70# "LlamaTokenizer" in transformers._import_structure["models.llama"]71# ), "LLaMA is now in HuggingFace's main branch.\nPlease reinstall it: pip uninstall transformers && pip install git+https://github.com/huggingface/transformers.git"72# from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig73 74# tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-7b-hf")75 76# BASE_MODEL = "decapoda-research/llama-7b-hf"77# LORA_WEIGHTS = "tloen/alpaca-lora-7b"78 79# if device == "cuda":80# model = LlamaForCausalLM.from_pretrained(81# BASE_MODEL,82# load_in_8bit=False,83# torch_dtype=torch.float16,84# device_map="auto",85# )86# model = PeftModel.from_pretrained(87# model, LORA_WEIGHTS, torch_dtype=torch.float16, force_download=True88# )89# elif device == "mps":90# model = LlamaForCausalLM.from_pretrained(91# BASE_MODEL,92# device_map={"": device},93# torch_dtype=torch.float16,94# )95# model = PeftModel.from_pretrained(96# model,97# LORA_WEIGHTS,98# device_map={"": device},99# torch_dtype=torch.float16,100# )101# else:102# model = LlamaForCausalLM.from_pretrained(103# BASE_MODEL, device_map={"": device}, low_cpu_mem_usage=True104# )105# model = PeftModel.from_pretrained(106# model,107# LORA_WEIGHTS,108# device_map={"": device},109# )110 111 112# if device != "cpu":113# model.half()114# model.eval()115# if torch.__version__ >= "2":116# model = torch.compile(model)117 118 119## FLAN-UL2120HF_TOKEN = os.environ.get("API_TOKEN", None)121API_URL = "https://api-inference.huggingface.co/models/google/flan-ul2"122headers = {"Authorization": f"Bearer {HF_TOKEN}"}123def query(payload):124 response = requests.post(API_URL, headers=headers, json=payload)125 return response.json()126 127## OpenAI models128openai.api_key = os.environ.get("OPENAI_TOKEN", None) 129def set_openai_api_key(api_key):130 if api_key and api_key.startswith("sk-") and len(api_key) > 50:131 openai.api_key = api_key132 133def get_response_from_openai(prompt, model="gpt-3.5-turbo", max_output_tokens=256):134 messages = [{"role": "assistant", "content": prompt}]135 response = openai.ChatCompletion.create(136 model=model,137 messages=messages,138 temperature=0.7,139 max_tokens=max_output_tokens,140 top_p=1,141 frequency_penalty=0,142 presence_penalty=0,143 )144 ret = response.choices[0].message['content']145 return ret146 147## deplot models148model_deplot = Pix2StructForConditionalGeneration.from_pretrained("google/deplot", torch_dtype=torch.bfloat16)149if device == "cuda":150 model_deplot = model_deplot.to(0)151processor_deplot = Pix2StructProcessor.from_pretrained("google/deplot")152 153def evaluate(154 table,155 question,156 llm="alpaca-lora",157 input=None,158 temperature=0.1,159 top_p=0.75,160 top_k=40,161 num_beams=4,162 max_new_tokens=128,163 **kwargs,164):165 prompt_0shot = _INSTRUCTION + "\n" + _add_markup(table) + "\n" + "Q: " + question + "\n" + "A:"166 prompt = _TEMPLATE + "\n" + _add_markup(table) + "\n" + "Q: " + question + "\n" + "A:"167 if llm == "alpaca-lora":168 inputs = tokenizer(prompt, return_tensors="pt")169 input_ids = inputs["input_ids"].to(device)170 generation_config = GenerationConfig(171 temperature=temperature,172 top_p=top_p,173 top_k=top_k,174 num_beams=num_beams,175 **kwargs,176 )177 with torch.no_grad():178 generation_output = model.generate(179 input_ids=input_ids,180 generation_config=generation_config,181 return_dict_in_generate=True,182 output_scores=True,183 max_new_tokens=max_new_tokens,184 )185 s = generation_output.sequences[0]186 output = tokenizer.decode(s)187 elif llm == "flan-ul2":188 try:189 output = query({"inputs": prompt_0shot})[0]["generated_text"]190 except:191 output = "<flan-ul2 inference API error - try later>"192 elif llm == "gpt-3.5-turbo":193 try:194 output = get_response_from_openai(prompt_0shot)195 except:196 output = "<Remember to input your OpenAI API key ☺>"197 else:198 RuntimeError(f"No such LLM: {llm}")199 200 return output201 202 203def process_document(image, question, llm):204 # image = Image.open(image)205 inputs = processor_deplot(images=image, text="Generate the underlying data table for the figure below:", return_tensors="pt").to(torch.bfloat16)206 if device == "cuda":207 inputs = inputs.to(0)208 predictions = model_deplot.generate(**inputs, max_new_tokens=512)209 table = processor_deplot.decode(predictions[0], skip_special_tokens=True).replace("<0x0A>", "\n")210 211 # send prompt+table to LLM212 res = evaluate(table, question, llm=llm)213 if llm == "alpaca-lora":214 return [table, res.split("A:")[-1]]215 else:216 return [table, res]217 218# theme = gr.themes.Monochrome(219# primary_hue="indigo",220# secondary_hue="blue",221# neutral_hue="slate",222# radius_size=gr.themes.sizes.radius_sm,223# font=[gr.themes.GoogleFont("Open Sans"), "ui-sans-serif", "system-ui", "sans-serif"],224# )225 226with gr.Blocks(theme="gradio/soft") as demo:227 with gr.Column():228 # gr.Markdown(229 # """<h1><center>DePlot+LLM: Multimodal chain-of-thought reasoning on plots</center></h1>230 # <p>231 # This is a demo of DePlot+LLM for QA and summarisation. <a href='https://arxiv.org/abs/2212.10505' target='_blank'>DePlot</a> is an image-to-text model that converts plots and charts into a textual sequence. The sequence then is used to prompt LLM for chain-of-thought reasoning. The current underlying LLMs are <a href='https://huggingface.co/spaces/tloen/alpaca-lora' target='_blank'>alpaca-lora</a>, <a href='https://huggingface.co/google/flan-ul2' target='_blank'>flan-ul2</a>, and <a href='https://openai.com/blog/chatgpt' target='_blank'>gpt-3.5-turbo</a>. To use it, simply upload your image and type a question or instruction and click 'submit', or click one of the examples to load them. Read more at the links below.232 # </p>233 # """234 # )235 gr.Markdown(236 """<h1><center>DePlot+LLM: Multimodal chain-of-thought reasoning on plot📊</center></h1>237 <h3>238 <center>239 <a href='https://arxiv.org/abs/2212.09662' target='_blank'>[paper]</a> <a href='https://ai.googleblog.com/2023/05/foundation-models-for-reasoning-on.html' target='_blank'>[google-ai blog]</a> <a href='https://github.com/google-research/google-research/tree/master/deplot' target='_blank'>[code]</a>240 </center>241 </h3>242 <p>243 This is a demo of DePlot+LLM for QA and summarisation. <a href='https://arxiv.org/abs/2212.10505' target='_blank'>DePlot</a> is an image-to-text model that converts plots and charts into a textual sequence. The sequence then is used to prompt LLM for chain-of-thought reasoning. The current underlying LLMs are <a href='https://huggingface.co/google/flan-ul2' target='_blank'>flan-ul2</a> and <a href='https://openai.com/blog/chatgpt' target='_blank'>gpt-3.5-turbo</a>. To use it, simply upload your image and type a question or instruction and click 'submit', or click one of the examples to load them. 244 </p>245 """246 )247 248 with gr.Row():249 with gr.Column(scale=2):250 input_image = gr.Image(label="Input Image", type="pil", interactive=True)251 #input_image.style(height=512, width=512)252 instruction = gr.Textbox(placeholder="Enter your instruction/question...", label="Question/Instruction")253 #llm = gr.Dropdown(["alpaca-lora", "flan-ul2", "gpt-3.5-turbo"], label="LLM")254 llm = gr.Dropdown(["flan-ul2", "gpt-3.5-turbo"], label="LLM")255 openai_api_key_textbox = gr.Textbox(value='', 256 placeholder="Paste your OpenAI API key (sk-...) and hit Enter (if using OpenAI models, otherwise leave empty)",257 show_label=False, lines=1, type='password') 258 submit = gr.Button("Submit", variant="primary")259 260 with gr.Column(scale=2): 261 with gr.Accordion("Show intermediate table", open=False):262 output_table = gr.Textbox(lines=8, label="Intermediate Table")263 output_text = gr.Textbox(lines=8, label="Output")264 265 gr.Examples(266 examples=[267 ["deplot_case_study_6.png", "Rank the four methods according to average model performances. By how much does deplot outperform the second strongest approach on average across the two sets? Show the computation.", "gpt-3.5-turbo"], # ex 1268 ["deplot_case_study_4.png", "What are the acceptance rates? And how does the acceptance change over the years?", "gpt-3.5-turbo"], # ex 2269 ["deplot_case_study_m1.png", "Summarise the chart for me please.", "gpt-3.5-turbo"], # ex 3270 #["deplot_case_study_m1.png", "What is the sum of numbers of Indonesia and Ireland? Remember to think step by step.", "alpaca-lora"],271 #["deplot_case_study_3.png", "By how much did China's growth rate drop? Think step by step.", "alpaca-lora"],272 #["deplot_case_study_4.png", "How many papers are submitted in 2020?", "flan-ul2"],273 ["deplot_case_study_5.png", "Which sales channel has the second highest portion?", "flan-ul2"], # ex 4274 #["deplot_case_study_x2.png", "Summarise the chart for me please.", "alpaca-lora"],275 #["deplot_case_study_4.png", "How many papers are submitted in 2020?", "alpaca-lora"],276 #["deplot_case_study_m1.png", "Summarise the chart for me please.", "alpaca-lora"],277 #["deplot_case_study_4.png", "acceptance rate = # accepted / #submitted . What is the acceptance rate of 2010?", "flan-ul2"],278 #["deplot_case_study_m1.png", "Summarise the chart for me please.", "flan-ul2"],279 ],280 cache_examples=True,281 inputs=[input_image, instruction, llm],282 outputs=[output_table, output_text],283 fn=process_document284 )285 286 gr.Markdown(287 """<p style='text-align: center'><a href='https://arxiv.org/abs/2212.10505' target='_blank'>DePlot: One-shot visual language reasoning by plot-to-table translation</a></p>"""288 )289 openai.api_key = ""290 openai_api_key_textbox.change(set_openai_api_key,291 inputs=[openai_api_key_textbox],292 outputs=[])293 openai_api_key_textbox.submit(set_openai_api_key,294 inputs=[openai_api_key_textbox],295 outputs=[])296 submit.click(process_document, inputs=[input_image, instruction, llm], outputs=[output_table, output_text])297 instruction.submit(298 process_document, inputs=[input_image, instruction, llm], outputs=[output_table, output_text]299 )300 301demo.queue().launch(share=True)