HuggingFaceM4/AI_Meme_Generator
111
1import ast2import copy3import glob4import hashlib5import logging6import os7import re8import time9from pathlib import Path10from typing import List, Optional, Tuple11from urllib.parse import urlparse12from PIL import Image, ImageDraw, ImageFont13from io import BytesIO14 15import requests16import concurrent.futures17import random18import gradio as gr19import PIL20from gradio import processing_utils21from gradio_client.client import DEFAULT_TEMP_DIR22from text_generation import Client23from transformers import AutoProcessor24 25 26MODELS = [27 # "HuggingFaceM4/idefics-9b-instruct",28 "HuggingFaceM4/idefics-80b-instruct",29]30 31API_PATHS = {32 "HuggingFaceM4/idefics-9b-instruct": (33 "https://api-inference.huggingface.co/models/HuggingFaceM4/idefics-9b-instruct"34 ),35 "HuggingFaceM4/idefics-80b-instruct": (36 "https://api-inference.huggingface.co/models/HuggingFaceM4/idefics-80b-instruct"37 ),38}39 40SYSTEM_PROMPT = [41 """The following is a conversation between a highly knowledgeable and intelligent visual AI assistant, called Assistant, and a human user, called User.42In the following interactions, User and Assistant will converse in natural language, and Assistant will answer in a sassy way.43Assistant's main purpose is to create funny meme texts from the images User provides.44Assistant should be funny, sassy, and impertinent, and sometimes Assistant roasts people.45Assistant should not be mean. It should not say toxic, homophobic, sexist, racist, things or any demeaning things that can make people uncomfortable.46Assistant was created by Hugging Face.47 48Here's a conversation example:""",49 """\nUser:""",50 "https://ichef.bbci.co.uk/news/976/cpsprodpb/7727/production/_103330503_musk3.jpg",51 "Write a meme for that image.<end_of_utterance>",52 """\nAssistant: When you're trying to quit smoking but the cravings are too strong.<end_of_utterance>""",53 "\nUser:How about this image?",54 "https://www.boredpanda.com/blog/wp-content/uploads/2017/01/image-copy-copy-587d0e7918b57-png__700.jpg",55 "Write something funny about this image.<end_of_utterance>",56 """\nAssistant: Eggcellent service!<end_of_utterance>""",57 "\nUser: Roast this person",58 "https://i.pinimg.com/564x/98/34/4b/98344b2483bd7c8b71a5c0fed6fe20b6.jpg",59 "<end_of_utterance>",60 """\nAssistant: Damn your handwritting is pretty awful. But I suppose it must be pretty hard to hold a pen, considering you are a hammerhead shark.<end_of_utterance>""",61]62 63BAN_TOKENS = ( # For documentation puporse. We are not using this list, it is hardcoded inside `idefics_causal_lm.py` inside TGI.64 "<image>;<fake_token_around_image>"65)66EOS_STRINGS = ["<end_of_utterance>", "\nUser:"]67STOP_SUSPECT_LIST = []68 69API_TOKEN = os.getenv("HF_AUTH_TOKEN")70IDEFICS_LOGO = "https://huggingface.co/spaces/HuggingFaceM4/idefics_playground/resolve/main/IDEFICS_logo.png"71 72PROCESSOR = AutoProcessor.from_pretrained(73 "HuggingFaceM4/idefics-9b-instruct",74 token=API_TOKEN,75)76 77BOT_AVATAR = "IDEFICS_logo.png"78IMAGE_GALLERY_PATHS = [79 f"example_images/{image_dir}/{ex_image}"80 for image_dir in os.listdir("example_images")81 for ex_image in os.listdir(f"example_images/{image_dir}")82]83random.shuffle(IMAGE_GALLERY_PATHS)84logging.basicConfig(level=logging.INFO)85logger = logging.getLogger()86 87 88# Monkey patch adapted from gradio.components.image.Image - mostly to make the `save` step optional in `pil_to_temp_file`89def hash_bytes(bytes: bytes):90 sha1 = hashlib.sha1()91 sha1.update(bytes)92 return sha1.hexdigest()93 94 95def pil_to_temp_file(96 img: PIL.Image.Image,97 dir: str = DEFAULT_TEMP_DIR,98 format: str = "png",99 resize: bool = False,100) -> str:101 """Save a PIL image into a temp file"""102 if resize:103 img = img.resize((224, 224), Image.LANCZOS)104 bytes_data = processing_utils.encode_pil_to_bytes(img, format)105 temp_dir = Path(dir) / hash_bytes(bytes_data)106 temp_dir.mkdir(exist_ok=True, parents=True)107 filename = str(temp_dir / f"image.{format}")108 if not os.path.exists(filename):109 img.save(filename, pnginfo=processing_utils.get_pil_metadata(img))110 return filename111 112 113def pil_to_base64(114 img: PIL.Image.Image,115 resize: bool = False,116) -> str:117 """Save a PIL image into a temp file"""118 if resize:119 img = img.resize((224, 224), Image.LANCZOS)120 base64_img = processing_utils.encode_pil_to_base64(img)121 return base64_img122 123 124def add_file_gallery(selected_state: gr.SelectData, gallery_list: List[str]):125 return (126 "Write a meme about this image.",127 gallery_list[selected_state.index]["name"],128 "",129 )130 131 132def choose_gallery(gallery_type: str):133 if gallery_type == "Meme templates":134 image_gallery_list = [135 f"example_images/meme_templates/{ex_image}"136 for ex_image in os.listdir("example_images/meme_templates")137 ]138 elif gallery_type == "Funny images":139 image_gallery_list = [140 f"example_images/funny_images/{ex_image}"141 for ex_image in os.listdir("example_images/funny_images")142 ]143 elif gallery_type == "Politics":144 image_gallery_list = [145 f"example_images/politics_memes/{ex_image}"146 for ex_image in os.listdir("example_images/politics_memes")147 ]148 else:149 image_gallery_list = [150 f"example_images/{image_dir}/{ex_image}"151 for image_dir in os.listdir("example_images")152 for ex_image in os.listdir(f"example_images/{image_dir}")153 ]154 random.shuffle(image_gallery_list)155 return image_gallery_list156 157 158# Utils to handle the image markdown display logic159def split_str_on_im_markdown(string: str) -> List[str]:160 """161 Extract from a string (typically the user prompt string) the potential images from markdown162 Examples:163 - `User:Describe this image.` would become `["User:", "https://favurl.com/chicken_on_money.png", "Describe this image."]`164 - `User:Describe this image.` would become `["User:", "/my_temp/chicken_on_money.png", "Describe this image."]`165 """166 IMAGES_PATTERN = re.compile(r"!\[[^\]]*\]\((.*?)\s*(\"(?:.*[^\"])\")?\s*\)")167 parts = []168 cursor = 0169 for pattern in IMAGES_PATTERN.finditer(string):170 start = pattern.start()171 if start != cursor:172 parts.append(string[cursor:start])173 image_url = pattern.group(1)174 if image_url.startswith("/file="):175 image_url = image_url[6:] # Remove the 'file=' prefix176 parts.append(image_url)177 cursor = pattern.end()178 if cursor != len(string):179 parts.append(string[cursor:])180 return parts181 182 183def is_image(string: str) -> bool:184 """185 There are two ways for images: local image path or url.186 """187 return is_url(string) or string.startswith(DEFAULT_TEMP_DIR)188 189 190def is_url(string: str) -> bool:191 """192 Checks if the passed string contains a valid url and nothing else. e.g. if space is included it's immediately193 invalidated the url194 """195 if " " in string:196 return False197 result = urlparse(string)198 return all([result.scheme, result.netloc])199 200 201def isolate_images_urls(prompt_list: List) -> List:202 """203 Convert a full string prompt to the list format expected by the processor.204 In particular, image urls (as delimited by <fake_token_around_image>) should be their own elements.205 From:206 ```207 [208 "bonjour<fake_token_around_image><image:IMG_URL><fake_token_around_image>hello",209 PIL.Image.Image,210 "Aurevoir",211 ]212 ```213 to:214 ```215 [216 "bonjour",217 IMG_URL,218 "hello",219 PIL.Image.Image,220 "Aurevoir",221 ]222 ```223 """224 linearized_list = []225 for prompt in prompt_list:226 # Prompt can be either a string, or a PIL image227 if isinstance(prompt, PIL.Image.Image):228 linearized_list.append(prompt)229 elif isinstance(prompt, str):230 if "<fake_token_around_image>" not in prompt:231 linearized_list.append(prompt)232 else:233 prompt_splitted = prompt.split("<fake_token_around_image>")234 for ps in prompt_splitted:235 if ps == "":236 continue237 if ps.startswith("<image:"):238 linearized_list.append(ps[7:-1])239 else:240 linearized_list.append(ps)241 else:242 raise TypeError(243 f"Unrecognized type for `prompt`. Got {type(type(prompt))}. Was expecting something in [`str`,"244 " `PIL.Image.Image`]"245 )246 return linearized_list247 248 249def fetch_images(url_list: str) -> PIL.Image.Image:250 """Fetching images"""251 return PROCESSOR.image_processor.fetch_images(url_list)252 253 254def handle_manual_images_in_user_prompt(user_prompt: str) -> List[str]:255 """256 Handle the case of textually manually inputted images (i.e. the `<fake_token_around_image><image:IMG_URL><fake_token_around_image>`) in the user prompt257 by fetching them, saving them locally and replacing the whole sub-sequence the image local path.258 """259 if "<fake_token_around_image>" in user_prompt:260 splitted_user_prompt = isolate_images_urls([user_prompt])261 resulting_user_prompt = []262 for u_p in splitted_user_prompt:263 if is_url(u_p):264 img = fetch_images([u_p])[0]265 tmp_file = pil_to_temp_file(img)266 resulting_user_prompt.append(tmp_file)267 else:268 resulting_user_prompt.append(u_p)269 return resulting_user_prompt270 else:271 return [user_prompt]272 273 274def prompt_list_to_markdown(prompt_list: List[str]) -> str:275 """276 Convert a user prompt in the list format (i.e. elements are either a PIL image or a string) into277 the markdown format that is used for the chatbot history and rendering.278 """279 resulting_string = ""280 for elem in prompt_list:281 if is_image(elem):282 if is_url(elem):283 resulting_string += f""284 else:285 resulting_string += f""286 else:287 resulting_string += elem288 return resulting_string289 290 291def prompt_list_to_tgi_input(prompt_list: List[str]) -> str:292 """293 TGI expects a string that contains both text and images in the image markdown format (i.e. the `![]()` ).294 The images links are parsed on TGI side295 """296 result_string_input = ""297 for elem in prompt_list:298 if is_image(elem):299 try:300 if is_url(elem):301 response = requests.get(elem)302 if response.status_code == 200:303 elem_pil = Image.open(BytesIO(response.content))304 else:305 elem_pil = Image.open(elem)306 base64_img = pil_to_base64(elem_pil, resize=True)307 result_string_input += f""308 except Exception as e:309 logger.error(f"Image can't be loaded because of exception {e}")310 else:311 result_string_input += elem312 return result_string_input313 314 315def remove_spaces_around_token(text: str) -> str:316 pattern = r"\s*(<fake_token_around_image>)\s*"317 replacement = r"\1"318 result = re.sub(pattern, replacement, text)319 return result320 321 322# Chatbot utils323def insert_backslash(string, max_length=50):324 # Check if the string length is less than or equal to the max_length325 if len(string) <= max_length:326 return string327 328 # Start from the max_length character and search for the last space character before it329 for i in range(max_length - 1, -1, -1):330 if string[i] == " ":331 # Insert a backslash before the last space character332 return string[:i] + "\n" + string[i:]333 334 # If no space character is found, just insert a backslash at the max_length character335 return string[:max_length] + "\n" + string[max_length:]336 337 338def resize_with_ratio(image: PIL.Image.Image, fixed_width: int) -> PIL.Image.Image:339 # Get the current width and height340 width, height = image.size341 342 # Calculate the new width while maintaining the aspect ratio up to 2:3 ratio343 new_width = fixed_width344 new_height = min(int(height * (new_width / width)), int(1.5 * new_width))345 346 # Resize the image347 resized_img = image.resize((new_width, new_height), Image.LANCZOS)348 349 return resized_img350 351 352def make_new_lines(draw, image, font, text_is_too_long, lines, num_lines, num_loops):353 max_len_increment = 0354 while text_is_too_long and max_len_increment < 10:355 new_lines = lines.copy()356 last_line_with_backslash = insert_backslash(357 new_lines[-1],358 max_length=(len(new_lines[-1]) + max_len_increment)359 // (num_lines - num_loops),360 )361 penultimate_line, last_line = (362 last_line_with_backslash.split("\n")[0],363 last_line_with_backslash.split("\n")[1],364 )365 new_lines.pop(-1)366 new_lines.append(penultimate_line)367 new_lines.append(last_line)368 # If the we haven't reached the last line, we split it again369 if len(new_lines) < num_lines:370 new_lines, text_width, text_is_too_long = make_new_lines(371 draw=draw,372 image=image,373 font=font,374 text_is_too_long=text_is_too_long,375 lines=new_lines,376 num_lines=num_lines,377 num_loops=num_loops + 1,378 )379 text_width = max([draw.textlength(line, font) for line in new_lines])380 text_is_too_long = text_width > image.width381 max_len_increment += 1382 if not text_is_too_long:383 lines = new_lines384 return lines, text_width, text_is_too_long385 386 387def test_font_size(388 draw,389 image,390 text,391 font,392 font_meme_text,393 num_lines=1,394 min_font=35,395 font_size_reduction=5,396):397 text_width = draw.textlength(text, font)398 text_is_too_long = True399 lines = [text]400 while font.size > min_font and text_is_too_long:401 font = ImageFont.truetype(402 f"fonts/{font_meme_text}.ttf", size=font.size - font_size_reduction403 )404 if num_lines == 1:405 text_width = draw.textlength(text, font)406 text_is_too_long = text_width > image.width407 else:408 lines, text_width, text_is_too_long = make_new_lines(409 draw=draw,410 image=image,411 font=font,412 text_is_too_long=text_is_too_long,413 lines=lines,414 num_lines=num_lines,415 num_loops=0,416 )417 temp_text = "\n".join(lines)418 419 if not text_is_too_long and num_lines > 1:420 text = temp_text421 return text, font, text_width, text_is_too_long422 423 424def make_meme_image(425 image: str,426 text: str,427 font_meme_text: str,428 all_caps_meme_text: bool = False,429 text_at_the_top: bool = False,430) -> PIL.Image.Image:431 """432 Takes an image and a text and returns a meme image.433 """434 text = text.replace("\nUser", " ").replace("\n", " ").strip().rstrip(".")435 if all_caps_meme_text:436 text = text.upper()437 # Resize image438 fixed_width = 700439 image = Image.open(image)440 image = resize_with_ratio(image, fixed_width)441 image_width, image_height = image.size442 height_width_ratio = image_height / image_width443 444 draw = ImageDraw.Draw(image)445 min_font = 35446 initial_font_size = 60447 if height_width_ratio >= 1:448 min_font = 45449 initial_font_size = 80450 text_is_too_long = True451 num_lines = 0452 while text_is_too_long and num_lines < 8:453 num_lines += 1454 font = ImageFont.truetype(f"fonts/{font_meme_text}.ttf", size=initial_font_size)455 text, font, text_width, text_is_too_long = test_font_size(456 draw,457 image,458 text,459 font,460 font_meme_text,461 num_lines=num_lines,462 min_font=min_font,463 font_size_reduction=5,464 )465 466 if text_is_too_long:467 text = f"Text is too long to fit the image"468 if all_caps_meme_text:469 text = text.upper()470 font = ImageFont.truetype(f"fonts/{font_meme_text}.ttf", size=font.size)471 text_width = draw.textlength(text, font)472 473 outline_width = 2474 text_x = (image_width - text_width) / 2475 text_y = image_height - num_lines * font.size - 10 - 2 * num_lines476 if text_at_the_top:477 text_y = 0478 479 for i in range(-outline_width, outline_width + 1):480 for j in range(-outline_width, outline_width + 1):481 draw.multiline_text(482 (text_x + i, text_y + j), text, fill="black", align="center", font=font483 )484 draw.multiline_text((text_x, text_y), text, fill="white", align="center", font=font)485 486 return image487 488 489def format_user_prompt_with_im_history_and_system_conditioning(490 system_prompt: List[str],491 current_user_prompt_str: str,492 current_image: Optional[str],493 history: List[Tuple[str, str]],494) -> Tuple[List[str], List[str]]:495 """496 Produces the resulting list that needs to go inside the processor.497 It handles the potential image box input, the history and the system conditionning.498 """499 # resulting_list = copy.deepcopy(SYSTEM_PROMPT)500 resulting_list = system_prompt501 502 # Format history503 for turn in history:504 user_utterance, assistant_utterance = turn505 splitted_user_utterance = split_str_on_im_markdown(user_utterance)506 507 optional_space = ""508 if not is_image(splitted_user_utterance[0]):509 optional_space = " "510 resulting_list.append(f"\nUser:{optional_space}")511 resulting_list.extend(splitted_user_utterance)512 resulting_list.append(f"<end_of_utterance>\nAssistant: {assistant_utterance}")513 514 # Format current input515 current_user_prompt_str = remove_spaces_around_token(current_user_prompt_str)516 if current_image is None:517 if "![](" in current_user_prompt_str:518 current_user_prompt_list = split_str_on_im_markdown(current_user_prompt_str)519 else:520 current_user_prompt_list = handle_manual_images_in_user_prompt(521 current_user_prompt_str522 )523 524 optional_space = ""525 if not is_image(current_user_prompt_list[0]):526 # Check if the first element is an image (and more precisely a path to an image)527 optional_space = " "528 resulting_list.append(f"\nUser:{optional_space}")529 resulting_list.extend(current_user_prompt_list)530 resulting_list.append("<end_of_utterance>\nAssistant:")531 else:532 # Choosing to put the image first when the image is inputted through the UI, but this is an arbiratrary choice.533 resulting_list.extend(534 [535 "\nUser:",536 current_image,537 f"{current_user_prompt_str}<end_of_utterance>\nAssistant:",538 ]539 )540 current_user_prompt_list = [current_user_prompt_str]541 542 return resulting_list, current_user_prompt_list543 544 545def expand_layout():546 return gr.Column(scale=2), gr.Gallery(height=682)547 548 549def generate_meme(550 client,551 query,552 image,553 font_meme_text,554 all_caps_meme_text,555 text_at_the_top,556 generation_args,557):558 try:559 text = client.generate(prompt=query, **generation_args).generated_text560 except Exception as e:561 logger.error(f"Error {e} while generating meme text")562 text = ""563 if image is not None and text != "":564 meme_image = make_meme_image(565 image=image,566 text=text,567 font_meme_text=font_meme_text,568 all_caps_meme_text=all_caps_meme_text,569 text_at_the_top=text_at_the_top,570 )571 return meme_image572 else:573 return None574 575 576def model_inference(577 model_selector,578 system_prompt,579 user_prompt_str,580 chat_history,581 image,582 decoding_strategy,583 temperature,584 max_new_tokens,585 repetition_penalty,586 top_p,587 all_caps_meme_text,588 text_at_the_top,589 font_meme_text,590):591 chat_history = []592 if user_prompt_str.strip() == "" and image is None:593 return "", None, chat_history594 595 system_prompt = ast.literal_eval(system_prompt)596 (597 formated_prompt_list,598 user_prompt_list,599 ) = format_user_prompt_with_im_history_and_system_conditioning(600 system_prompt=system_prompt,601 current_user_prompt_str=user_prompt_str.strip(),602 current_image=image,603 history=chat_history,604 )605 606 client_endpoint = API_PATHS[model_selector]607 client = Client(608 base_url=client_endpoint,609 headers={"x-use-cache": "0", "Authorization": f"Bearer {API_TOKEN}"},610 timeout=45,611 )612 613 # Common parameters to all decoding strategies614 # This documentation is useful to read: https://huggingface.co/docs/transformers/main/en/generation_strategies615 generation_args = {616 "max_new_tokens": max_new_tokens,617 "repetition_penalty": repetition_penalty,618 "stop_sequences": EOS_STRINGS,619 }620 621 assert decoding_strategy in [622 "Greedy",623 "Top P Sampling",624 ]625 if decoding_strategy == "Greedy":626 generation_args["do_sample"] = False627 elif decoding_strategy == "Top P Sampling":628 generation_args["temperature"] = temperature629 generation_args["do_sample"] = True630 generation_args["top_p"] = top_p631 632 chat_history.append([prompt_list_to_markdown(user_prompt_list), ""])633 634 query = prompt_list_to_tgi_input(formated_prompt_list)635 all_meme_images = []636 with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:637 all_meme_images = list(638 executor.map(639 generate_meme,640 [client for _ in range(4)],641 [query for _ in range(4)],642 [image for _ in range(4)],643 [font_meme_text for _ in range(4)],644 [all_caps_meme_text for _ in range(4)],645 [text_at_the_top for _ in range(4)],646 [generation_args for _ in range(4)],647 )648 )649 all_meme_images = [meme for meme in all_meme_images if meme is not None]650 return user_prompt_str, all_meme_images, chat_history651 652 653def remove_last_turn(chat_history):654 if len(chat_history) == 0:655 return chat_history, "", ""656 last_interaction = chat_history[-1]657 chat_history = chat_history[:-1]658 chat_update = chat_history659 text_update = last_interaction[0]660 return chat_update, text_update, ""661 662 663textbox = gr.Textbox(664 placeholder="Upload an image and ask the AI to create a meme!",665 show_label=False,666 value="Write a meme about this image.",667 visible=True,668 container=False,669 label="Text input",670 scale=8,671 max_lines=5,672)673chatbot = gr.Chatbot(674 elem_id="chatbot",675 label="AI Meme Generator Chatbot",676 visible=False,677 avatar_images=[None, BOT_AVATAR],678)679css = """680.gradio-container{max-width: 1000px!important}681h1{display: flex;align-items: center;justify-content: center;gap: .25em}682*{transition: width 0.5s ease, flex-grow 0.5s ease}683"""684with gr.Blocks(title="AI Meme Generator", theme=gr.themes.Base(), css=css) as demo:685 with gr.Row(scale=0.5):686 gr.HTML(687 """<h1 align="center">AI Meme Generator <span style="font-size: 13px;">powered by <a href="https://huggingface.co/blog/idefics">IDEFICS</a></span><img width=40 height=40 src="https://cdn-uploads.huggingface.co/production/uploads/624bebf604abc7ebb01789af/v770xGti5vH1SYLBgyOO_.png" /></h1>"""688 )689 690 with gr.Row(elem_id="model_selector_row"):691 model_selector = gr.Dropdown(692 choices=MODELS,693 value="HuggingFaceM4/idefics-80b-instruct",694 interactive=True,695 show_label=False,696 container=False,697 label="Model",698 visible=False,699 )700 with gr.Row(equal_height=True):701 # scale=2 when expanded702 with gr.Column(scale=4, min_width=250) as upload_area:703 imagebox = gr.Image(704 type="filepath", label="Image to meme", height=272, visible=True705 )706 with gr.Group():707 with gr.Row():708 textbox.render()709 with gr.Row():710 submit_btn = gr.Button(711 value="▶️ Submit", visible=True, min_width=120712 )713 clear_btn = gr.ClearButton(714 [textbox, imagebox, chatbot], value="🧹 Clear", min_width=120715 )716 regenerate_btn = gr.Button(717 value="🔄 Regenerate", visible=True, min_width=120718 )719 with gr.Accordion(720 "Advanced settings", open=False, visible=True721 ) as parameter_row:722 with gr.Row():723 with gr.Column():724 all_caps_meme_text = gr.Checkbox(725 value=True,726 label="All Caps",727 interactive=True,728 info="",729 )730 text_at_the_top = gr.Checkbox(731 value=False,732 label="Text at the top",733 interactive=True,734 info="",735 )736 with gr.Column():737 font_meme_text = gr.Radio(738 [739 "impact",740 "Roboto-Regular",741 ],742 value="impact",743 label="Font",744 interactive=True,745 info="",746 )747 system_prompt = gr.Textbox(748 value=SYSTEM_PROMPT,749 visible=False,750 lines=20,751 max_lines=50,752 interactive=True,753 )754 max_new_tokens = gr.Slider(755 minimum=8,756 maximum=150,757 value=90,758 step=1,759 interactive=True,760 label="Maximum number of new tokens to generate",761 )762 repetition_penalty = gr.Slider(763 minimum=0.0,764 maximum=5.0,765 value=1.2,766 step=0.01,767 interactive=True,768 label="Repetition penalty",769 info="1.0 is equivalent to no penalty",770 )771 decoding_strategy = gr.Radio(772 [773 "Greedy",774 "Top P Sampling",775 ],776 value="Top P Sampling",777 label="Decoding strategy",778 interactive=True,779 info="Higher values is equivalent to sampling more low-probability tokens.",780 )781 temperature = gr.Slider(782 minimum=0.0,783 maximum=5.0,784 value=0.6,785 step=0.1,786 interactive=True,787 visible=True,788 label="Sampling temperature",789 info="Higher values will produce more diverse outputs.",790 )791 decoding_strategy.change(792 fn=lambda selection: gr.Slider.update(793 visible=(794 selection795 in [796 "contrastive_sampling",797 "beam_sampling",798 "Top P Sampling",799 "sampling_top_k",800 ]801 )802 ),803 inputs=decoding_strategy,804 outputs=temperature,805 )806 top_p = gr.Slider(807 minimum=0.01,808 maximum=0.99,809 value=0.8,810 step=0.01,811 interactive=True,812 visible=True,813 label="Top P",814 info="Higher values is equivalent to sampling more low-probability tokens.",815 )816 decoding_strategy.change(817 fn=lambda selection: gr.Slider.update(818 visible=(selection in ["Top P Sampling"])819 ),820 inputs=decoding_strategy,821 outputs=top_p,822 )823 with gr.Column(scale=5) as result_area:824 generated_memes_gallery = gr.Gallery(825 # value="Images generated will appear here",826 label="IDEFICS Generated Memes",827 allow_preview=True,828 elem_id="generated_memes_gallery",829 show_download_button=True,830 show_share_button=True,831 columns=[2],832 object_fit="contain",833 height=428,834 ) # height 600 when expanded835 with gr.Row(equal_height=True):836 with gr.Box(elem_id="gallery_box"):837 gallery_type_choice = gr.Radio(838 [839 "All",840 "Meme templates",841 "Funny images",842 "Politics",843 ],844 value="All",845 label="Gallery Type",846 interactive=True,847 visible=False,848 info="Choose the type of gallery you want to see.",849 )850 template_gallery = gr.Gallery(851 value=IMAGE_GALLERY_PATHS,852 label="Templates Gallery",853 allow_preview=False,854 columns=6,855 elem_id="gallery",856 show_share_button=False,857 height=400,858 )859 with gr.Row(variant="panel"):860 with gr.Column(scale=1):861 gr.Image(862 IDEFICS_LOGO,863 elem_id="banner-image",864 show_label=False,865 show_download_button=False,866 height=200,867 width=250,868 )869 with gr.Column(scale=5):870 gr.HTML(871 """872 <p><strong>AI Meme Generator</strong> is an AI system that writes humorous content inspired by images, allowing you to make the funniest memes with little effort. Upload your image and ask the Idefics chatbot to make a tailored meme.</p>873 <p>AI Meme Generator is a space inspired from <a href="https://huggingface.co/spaces/HuggingFaceM4/ai_dad_jokes">AI Dad Jokes</a> and powered by <a href="https://huggingface.co/blog/idefics">IDEFICS</a>, an open-access large visual language model developped by Hugging Face. Like GPT-4, the multimodal model accepts arbitrary sequences of image and text inputs and produces text outputs. IDEFICS can answer questions about images, describe visual content, create stories grounded in multiple images, etc.</p>874 875 <p>⛔️ <strong>Intended uses and limitations:</strong> This demo is provided as research artifact to the community showcasing IDEFICS'capabilities. We detail misuses and out-of-scope uses <a href="https://huggingface.co/HuggingFaceM4/idefics-80b#misuse-and-out-of-scope-use">here</a>. In particular, the system should not be used to engage in harassment, abuse and bullying. The model can produce factually incorrect texts, hallucinate facts (with or without an image) and will struggle with small details in images. While the system will tend to refuse answering questionable user requests, it can produce problematic outputs (including racist, stereotypical, and disrespectful texts), in particular when prompted to do so.</p>876 """877 )878 with gr.Row():879 chatbot.render()880 881 gr.on(882 triggers=[883 textbox.submit,884 imagebox.upload,885 submit_btn.click,886 template_gallery.select,887 regenerate_btn.click,888 ],889 fn=expand_layout,890 outputs=[upload_area, generated_memes_gallery],891 queue=False,892 ).success(893 fn=lambda: "", inputs=[], outputs=[generated_memes_gallery], queue=False894 ).success(895 fn=model_inference,896 inputs=[897 model_selector,898 system_prompt,899 textbox,900 chatbot,901 imagebox,902 decoding_strategy,903 temperature,904 max_new_tokens,905 repetition_penalty,906 top_p,907 all_caps_meme_text,908 text_at_the_top,909 font_meme_text,910 ],911 outputs=[textbox, generated_memes_gallery, chatbot],912 )913 914 regenerate_btn.click(915 fn=remove_last_turn,916 inputs=chatbot,917 outputs=[chatbot, textbox, generated_memes_gallery],918 queue=False,919 )920 921 # gallery_type_choice.change(922 # fn=choose_gallery,923 # inputs=[gallery_type_choice],924 # outputs=[template_gallery],925 # queue=False,926 # )927 template_gallery.select(928 fn=add_file_gallery,929 inputs=[template_gallery],930 outputs=[textbox, imagebox, generated_memes_gallery],931 queue=False,932 )933 demo.load(934 # fn=choose_gallery,935 # inputs=[gallery_type_choice],936 # outputs=[template_gallery],937 queue=False,938 )939demo.queue(concurrency_count=8, max_size=40, api_open=False)940demo.launch(max_threads=400)941