Ankit2802/phi3_vision_128k

1 2 3from PIL import Image4import requests5import torch6from transformers import AutoModelForCausalLM7from transformers import AutoProcessor8model_path = "./"9 10kwargs = {}11kwargs['torch_dtype'] = torch.bfloat1612 13processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)14model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True, torch_dtype="auto").cuda()15 16user_prompt = '<|user|>\n'17assistant_prompt = '<|assistant|>\n'18prompt_suffix = "<|end|>\n"19 20#################################################### text-only ####################################################21prompt = f"{user_prompt}what is the answer for 1+1? Explain it.{prompt_suffix}{assistant_prompt}"22print(f">>> Prompt\n{prompt}")23inputs = processor(prompt, images=None, return_tensors="pt").to("cuda:0")24generate_ids = model.generate(**inputs, 25                              max_new_tokens=1000,26                              eos_token_id=processor.tokenizer.eos_token_id,27                              )28generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]29response = processor.batch_decode(generate_ids, 30                                  skip_special_tokens=True, 31                                  clean_up_tokenization_spaces=False)[0]32print(f'>>> Response\n{response}')33 34#################################################### text-only 2 ####################################################35prompt = f"{user_prompt}Give me the code for sloving two-sum problem.{prompt_suffix}{assistant_prompt}"36print(f">>> Prompt\n{prompt}")37inputs = processor(prompt, images=None, return_tensors="pt").to("cuda:0")38generate_ids = model.generate(**inputs, 39                              max_new_tokens=1000,40                              eos_token_id=processor.tokenizer.eos_token_id,41                              )42generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]43response = processor.batch_decode(generate_ids, 44                                  skip_special_tokens=True, 45                                  clean_up_tokenization_spaces=False)[0]46print(f'>>> Response\n{response}')47 48 49#################################################### EXAMPLE 1 ####################################################50# single-image prompt51prompt = f"{user_prompt}<|image_1|>\nWhat is shown in this image?{prompt_suffix}{assistant_prompt}"52url = "https://www.ilankelman.org/stopsigns/australia.jpg"53print(f">>> Prompt\n{prompt}")54image = Image.open(requests.get(url, stream=True).raw)55inputs = processor(prompt, image, return_tensors="pt").to("cuda:0")56generate_ids = model.generate(**inputs, 57                              max_new_tokens=1000,58                              eos_token_id=processor.tokenizer.eos_token_id,59                              )60generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]61response = processor.batch_decode(generate_ids, 62                                  skip_special_tokens=True, 63                                  clean_up_tokenization_spaces=False)[0]64print(f'>>> Response\n{response}')65 66#################################################### EXAMPLE 2 ####################################################67# chat template68chat = [69    {"role": "user", "content": "<|image_1|>\nWhat is shown in this image?"},70    {"role": "assistant", "content": "The image depicts a street scene with a prominent red stop sign in the foreground. The background showcases a building with traditional Chinese architecture, characterized by its red roof and ornate decorations. There are also several statues of lions, which are common in Chinese culture, positioned in front of the building. The street is lined with various shops and businesses, and there's a car passing by."},71    {"role": "user", "content": "What is so special about this image"}72]73url = "https://www.ilankelman.org/stopsigns/australia.jpg"74image = Image.open(requests.get(url, stream=True).raw)75prompt = processor.tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)76# need to remove last <|endoftext|> if it is there, which is used for training, not inference. For training, make sure to add <|endoftext|> in the end.77if prompt.endswith("<|endoftext|>"):78    prompt = prompt.rstrip("<|endoftext|>")79 80print(f">>> Prompt\n{prompt}")81 82inputs = processor(prompt, [image], return_tensors="pt").to("cuda:0")83generate_ids = model.generate(**inputs, 84                              max_new_tokens=1000,85                              eos_token_id=processor.tokenizer.eos_token_id,86                              )87generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]88response = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]89print(f'>>> Response\n{response}')90 91 92############################# to markdown #############################93# single-image prompt94prompt = f"{user_prompt}<|image_1|>\nCan you convert the table to markdown format?{prompt_suffix}{assistant_prompt}"95url = "https://support.content.office.net/en-us/media/3dd2b79b-9160-403d-9967-af893d17b580.png"96image = Image.open(requests.get(url, stream=True).raw)97inputs = processor(prompt, image, return_tensors="pt").to("cuda:0")98 99print(f">>> Prompt\n{prompt}")100generate_ids = model.generate(**inputs, 101                              max_new_tokens=1000,102                              eos_token_id=processor.tokenizer.eos_token_id,103                              )104generate_ids = generate_ids[:, inputs['input_ids'].shape[1]:]105response = processor.batch_decode(generate_ids, 106                                  skip_special_tokens=False, 107                                  clean_up_tokenization_spaces=False)[0]108print(f'>>> Response\n{response}')109