adityaardak/Describe_Image_using_FastVLM
0
1import gradio as gr2import torch3from transformers import AutoTokenizer, AutoModelForCausalLM4 5# =========================================================6# Model setup7# =========================================================8MID = "apple/FastVLM-0.5B"9IMAGE_TOKEN_INDEX = -20010 11tok = None12model = None13 14def load_model():15 global tok, model16 if tok is None or model is None:17 print("Loading FastVLM on CPU...")18 tok = AutoTokenizer.from_pretrained(MID, trust_remote_code=True)19 model = AutoModelForCausalLM.from_pretrained(20 MID,21 torch_dtype=torch.float32,22 device_map="cpu",23 trust_remote_code=True,24 )25 print("Model loaded successfully on CPU.")26 return tok, model27 28 29def run_fastvlm(image, prompt, max_new_tokens=180):30 if image is None:31 return "Please upload an image first."32 33 try:34 tok, model = load_model()35 36 if image.mode != "RGB":37 image = image.convert("RGB")38 39 messages = [{"role": "user", "content": f"<image>\n{prompt}"}]40 rendered = tok.apply_chat_template(41 messages,42 add_generation_prompt=True,43 tokenize=False44 )45 46 pre, post = rendered.split("<image>", 1)47 48 pre_ids = tok(pre, return_tensors="pt", add_special_tokens=False).input_ids49 post_ids = tok(post, return_tensors="pt", add_special_tokens=False).input_ids50 51 model_device = next(model.parameters()).device52 model_dtype = next(model.parameters()).dtype53 54 img_tok = torch.tensor([[IMAGE_TOKEN_INDEX]], dtype=pre_ids.dtype, device=model_device)55 56 input_ids = torch.cat(57 [pre_ids.to(model_device), img_tok, post_ids.to(model_device)],58 dim=159 )60 61 attention_mask = torch.ones_like(input_ids, device=model_device)62 63 pixel_values = model.get_vision_tower().image_processor(64 images=image,65 return_tensors="pt"66 )["pixel_values"].to(device=model_device, dtype=model_dtype)67 68 with torch.no_grad():69 out = model.generate(70 inputs=input_ids,71 attention_mask=attention_mask,72 images=pixel_values,73 max_new_tokens=max_new_tokens,74 do_sample=False75 )76 77 generated_text = tok.decode(out[0], skip_special_tokens=True)78 79 if "Assistant:" in generated_text:80 response = generated_text.split("Assistant:", 1)[-1].strip()81 elif "assistant" in generated_text:82 response = generated_text.split("assistant", 1)[-1].strip()83 else:84 response = generated_text.strip()85 86 return response87 88 except Exception as e:89 return f"Error generating response: {str(e)}"90 91 92# =========================================================93# Use case knowledge cards94# =========================================================95USE_CASE_INFO = {96 "Accessibility Assistant": {97 "problem": "A visually impaired user may need quick scene understanding and help identifying objects or obstacles.",98 "beneficiaries": "Visually impaired users, caregivers, accessibility NGOs, smart assistive-tech teams.",99 "proof": "The app describes the scene, highlights key items, and gives practical guidance.",100 "judge_angle": "Shows AI for inclusion and social impact."101 },102 "Safety Checker": {103 "problem": "People often miss visible risks in busy roads, stairways, cluttered spaces, or public areas.",104 "beneficiaries": "Schools, public-space monitoring teams, safety awareness projects.",105 "proof": "The app flags possible risks, risky zones, and next-safe-action ideas.",106 "judge_angle": "Shows preventive AI and practical awareness."107 },108 "Museum / Exhibit Guide": {109 "problem": "Visitors want engaging explanations, not just raw object names.",110 "beneficiaries": "Museums, exhibitions, tourism projects, learning spaces.",111 "proof": "The app turns the same image into a friendly guide-like explanation.",112 "judge_angle": "Shows storytelling plus education."113 },114 "Retail Shelf Helper": {115 "problem": "Customers and staff need quick item understanding, arrangement insight, and shelf-level interpretation.",116 "beneficiaries": "Retail stores, FMCG demos, smart shopping assistants.",117 "proof": "The app summarizes visible products, arrangement, and shopper-facing insights.",118 "judge_angle": "Shows business and commercial use."119 },120 "Classroom Explainer": {121 "problem": "Students often understand better when images are explained in simple, structured language.",122 "beneficiaries": "Teachers, students, EdTech demos, smart classrooms.",123 "proof": "The app explains the image like a teacher using easy language and teaching points.",124 "judge_angle": "Shows educational value."125 },126 "Travel Interpreter": {127 "problem": "Travelers want quick understanding of landmarks, scenes, crowd conditions, and surroundings.",128 "beneficiaries": "Travel apps, tourism assistance, city experience projects.",129 "proof": "The app explains what the place appears to be, what stands out, and what a visitor should notice.",130 "judge_angle": "Shows lifestyle and tourism use."131 }132}133 134 135def get_use_case_card(use_case):136 info = USE_CASE_INFO[use_case]137 return f"""138### {use_case}139 140**Problem Solved** 141{info['problem']}142 143**Who Benefits** 144{info['beneficiaries']}145 146**What This Demo Proves** 147{info['proof']}148 149**Why Judges Usually Like It** 150{info['judge_angle']}151"""152 153 154# =========================================================155# Prompt builders156# =========================================================157def build_use_case_prompt(use_case, user_context):158 context = user_context.strip() if user_context else "No extra context provided."159 160 prompts = {161 "Accessibility Assistant": f"""162You are an assistive AI helping a visually impaired user.163 164Analyze the uploaded image and return your answer in this format:1651. Quick Scene Summary1662. Main Objects and Their Positions1673. Anything Important to Notice1684. Helpful Guidance for the User169 170Use simple, natural, practical language.171Mention uncertainty when needed.172 173Context: {context}174""",175 176 "Safety Checker": f"""177You are an AI safety observer.178 179Analyze the uploaded image and return your answer in this format:1801. What the Scene Appears to Show1812. Possible Hazards or Risky Elements1823. Risk Level: Low / Medium / High1834. Best Next Safe Action184 185Be cautious, grounded, and practical.186Do not invent invisible hazards.187Mention uncertainty when needed.188 189Context: {context}190""",191 192 "Museum / Exhibit Guide": f"""193You are a smart museum guide.194 195Analyze the uploaded image and return:1961. What Visitors Are Looking At1972. Interesting Visual Details1983. Why It Could Matter / Be Memorable1994. A Friendly 2-line Visitor Guide200 201Make it warm, engaging, and exhibition-friendly.202Context: {context}203""",204 205 "Retail Shelf Helper": f"""206You are an AI retail assistant.207 208Analyze the uploaded image and return:2091. What Products / Objects Are Visible2102. Arrangement or Display Observations2113. Shopper-Friendly Insights2124. Staff / Store Improvement Suggestion213 214Be concise, business-relevant, and practical.215Context: {context}216""",217 218 "Classroom Explainer": f"""219You are a teacher explaining the image to students.220 221Return:2221. What We See2232. Main Concepts / Objects2243. Easy Explanation for Students2254. One Learning Question226 227Use clear, beginner-friendly language.228Context: {context}229""",230 231 "Travel Interpreter": f"""232You are an AI travel companion.233 234Analyze the uploaded image and return:2351. What This Place / Scene Looks Like2362. What a Visitor Would Notice First2373. Interesting or Useful Observations2384. One Practical Travel Tip239 240Stay grounded in the visible scene.241Context: {context}242"""243 }244 245 return prompts[use_case]246 247 248def build_persona_prompt(persona, tone, goal):249 goal_text = goal.strip() if goal else "Explain the image in your role."250 return f"""251You are analyzing the image as this role: {persona}252Tone: {tone}253Goal: {goal_text}254 255Return your answer in this format:2561. Role Introduction2572. What I Notice First2583. What Matters Most From My Perspective2594. My Advice / Commentary2605. One Memorable Closing Line261 262Stay grounded in the image.263Do not pretend to know hidden facts.264"""265 266 267def build_mission_prompt(mission, mission_context):268 context = mission_context.strip() if mission_context else "No extra context."269 270 mission_prompts = {271 "Hidden Detail Hunt": f"""272Study the image carefully.273 274Return:2751. 5 specific details that are easy to miss2762. Why each detail matters2773. What those details suggest about the scene278 279Stay grounded in the visible image only.280Context: {context}281""",282 283 "Exhibit Quiz Maker": f"""284Create a mini exhibition quiz from the image.285 286Return:2871. Five quiz questions2882. Correct answer under each question2893. One final bonus question290 291Make the quiz engaging and image-based.292Context: {context}293""",294 295 "Pitch From the Picture": f"""296Look at the image and imagine a useful product, service, or startup idea inspired by it.297 298Return:2991. Problem Seen in the Image3002. Product / Service Idea3013. Target Users3024. One-line Pitch303 304Keep it smart, creative, but still linked to the image.305Context: {context}306""",307 308 "Evidence Board": f"""309Analyze the image critically.310 311Return:3121. Things that are clearly visible3132. Things that are likely but not certain3143. Things that should NOT be assumed3154. Why careful interpretation matters316 317This mission is for teaching responsible AI reasoning.318Context: {context}319""",320 321 "Story Spark": f"""322Create a short story inspired by the image.323 324Return:3251. Title3262. Story in under 120 words3273. What visual details inspired the story328 329Keep it imaginative but tied to the scene.330Context: {context}331""",332 333 "Accessibility Voiceover": f"""334Create a voiceover-style narration for a visually impaired user.335 336Return:3371. Calm spoken scene narration3382. Important objects3393. Immediate practical note3404. Final short reassurance341 342Make it audio-friendly and natural.343Context: {context}344"""345 }346 347 return mission_prompts[mission]348 349 350def build_question_prompt(question):351 user_q = question.strip() if question else "What is happening in this image?"352 return f"""353Answer the user's question about the image.354 355Question: {user_q}356 357Return:3581. Direct Answer3592. Evidence From the Image3603. Uncertainty Note if Needed361 362Keep it short and reliable.363"""364 365 366# =========================================================367# App functions368# =========================================================369def analyze_use_case(image, use_case, user_context):370 prompt = build_use_case_prompt(use_case, user_context)371 return run_fastvlm(image, prompt, max_new_tokens=200)372 373 374def persona_playground(image, persona, tone, goal):375 prompt = build_persona_prompt(persona, tone, goal)376 return run_fastvlm(image, prompt, max_new_tokens=190)377 378 379def mission_lab(image, mission, mission_context):380 prompt = build_mission_prompt(mission, mission_context)381 return run_fastvlm(image, prompt, max_new_tokens=220)382 383 384def ask_image(image, question):385 prompt = build_question_prompt(question)386 return run_fastvlm(image, prompt, max_new_tokens=160)387 388 389def compare_booth(image, compare_context):390 context = compare_context.strip() if compare_context else "No extra context."391 392 prompt_1 = f"""393Explain this image as an Accessibility Assistant.394Return:3951. Scene Summary3962. Important Objects3973. Helpful Guidance398Context: {context}399"""400 prompt_2 = f"""401Explain this image as a Safety Checker.402Return:4031. Visible Risks4042. Risk Level4053. Safe Next Step406Context: {context}407"""408 prompt_3 = f"""409Explain this image as a Classroom Teacher.410Return:4111. What Students See4122. Main Idea4133. One Learning Question414Context: {context}415"""416 417 out1 = run_fastvlm(image, prompt_1, max_new_tokens=140)418 out2 = run_fastvlm(image, prompt_2, max_new_tokens=140)419 out3 = run_fastvlm(image, prompt_3, max_new_tokens=140)420 421 return out1, out2, out3422 423 424def generate_exhibit_script(use_case):425 scripts = {426 "Accessibility Assistant": """427### 30-Second Pitch428 429This project turns image understanding into an accessibility helper. 430A user uploads a scene, and the system explains what is visible, what matters most, and what practical guidance may help. 431This shows how multimodal AI can support inclusion, independence, and human-centered design.432 433**Best line for judges:** 434"We are not just describing pictures. We are translating visual space into usable understanding."435""",436 437 "Safety Checker": """438### 30-Second Pitch439 440This project uses visual AI to inspect scenes for visible risk signals such as clutter, unsafe movement zones, or attention-worthy areas. 441It is useful as an awareness tool for schools, public demonstrations, and smart safety education. 442The value is not only detection, but guidance.443 444**Best line for judges:** 445"This app turns passive vision into preventive awareness."446""",447 448 "Museum / Exhibit Guide": """449### 30-Second Pitch450 451This project acts like an AI guide that explains images in a visitor-friendly way. 452Instead of only naming objects, it creates interpretation, context, and memorable observations. 453It can be adapted for museums, campus exhibitions, tourism booths, and educational spaces.454 455**Best line for judges:** 456"We changed image captioning into an interactive guide experience."457""",458 459 "Retail Shelf Helper": """460### 30-Second Pitch461 462This project interprets shelf images and converts them into shopper and business insights. 463It can help summarize visible products, arrangement cues, and display observations. 464This shows how the same AI model can serve a commercial use case without retraining.465 466**Best line for judges:** 467"One image can become both a customer insight and an operational insight."468""",469 470 "Classroom Explainer": """471### 30-Second Pitch472 473This project uses image understanding to support teaching. 474It explains the same visual in simple educational language and even creates learning prompts. 475That makes it useful for smart classrooms, EdTech projects, and visual learning tools.476 477**Best line for judges:** 478"This app helps students look at an image and actually learn from it."479""",480 481 "Travel Interpreter": """482### 30-Second Pitch483 484This project behaves like a visual travel companion. 485It interprets scenes, highlights what visitors may notice, and gives useful context or practical tips. 486That makes it relevant for tourism, smart city experiences, and visitor support.487 488**Best line for judges:** 489"We turned one uploaded image into a mini travel briefing."490"""491 }492 return scripts[use_case]493 494 495# =========================================================496# UI text497# =========================================================498HERO = """499# VisionVerse AI500## Exhibition Studio for Real-World Image Intelligence501 502Upload one image and explore many use cases:503- accessibility504- safety505- teaching506- tourism507- retail508- storytelling509- evidence checking510- interactive Q&A511 512### What makes this exhibition-ready?513This is not a one-button caption demo. 514It is a **multi-use visual intelligence studio** designed to prove that a single AI vision engine can serve many real-world situations.515"""516 517INFO_PAGE = """518# Project Info519 520## 1) What this project is521VisionVerse AI is an exhibition-ready visual intelligence app built on top of a multimodal image-language model. 522Instead of using the model for just one generic caption, the app wraps it in multiple roles, scenarios, and interaction modes.523 524## 2) Core idea525One uploaded image can be interpreted in many ways:526- as an accessibility helper527- as a safety observer528- as a teacher529- as a museum guide530- as a retail assistant531- as a travel companion532- as a critical evidence checker533 534## 3) Why this matters535In many student projects, the model is good but the demonstration feels narrow. 536This app proves flexibility, purpose, and user-centered design.537 538## 4) Architecture539- Gradio front-end540- FastVLM multimodal model541- CPU-only inference542- Prompt engineering for role adaptation543- Tab-based interaction design544 545## 5) Strengths546- many real-world uses from one model547- strong exhibition storytelling548- easy demo with any uploaded image549- playful interaction modes550- educational and social impact angles551 552## 6) Limitations553- runs on CPU, so response can be slower554- not a certified medical or safety device555- may miss fine details or make uncertain interpretations556- should be used as assistive AI, not final authority557 558## 7) Responsible AI note559The Evidence Board mission is included to show that good AI systems should separate:560- what is clearly visible561- what is likely562- what should not be assumed563 564## 8) Suggested evaluation ideas565- response usefulness566- clarity of explanation567- consistency across different scenes568- user satisfaction by use case569- educational / accessibility impact570 571## 9) Best demo images572- road or traffic scene573- classroom or laboratory574- store shelf575- museum object576- crowded public place577- home kitchen or hallway578 579## 10) Best exhibition closing line580"This project is not about generating text from images. It is about generating the right kind of help for the right kind of user."581"""582 583CSS = """584.gradio-container {585 max-width: 1400px !important;586}587.card-note {588 border-radius: 16px;589 padding: 14px;590 background: #f6f8ff;591}592"""593 594 595# =========================================================596# Gradio UI597# =========================================================598with gr.Blocks(title="VisionVerse AI", css=CSS, theme=gr.themes.Soft()) as demo:599 gr.Markdown(HERO)600 601 with gr.Row():602 with gr.Column(scale=1):603 shared_image = gr.Image(type="pil", label="Upload Image for All Tabs")604 clear_all = gr.ClearButton([shared_image], value="Clear Image")605 with gr.Column(scale=1):606 gr.Markdown("""607### Quick Demo Route6081. Upload one image 6092. Open **Use Case Studio** 6103. Open **Persona Playground** 6114. Open **Mission Lab** 6125. Open **Compare Booth** 6136. End with **Live Exhibit Script**614 615This flow makes the demo feel layered, interactive, and purposeful.616""")617 618 with gr.Tabs():619 with gr.Tab("Use Case Studio"):620 with gr.Row():621 with gr.Column():622 use_case = gr.Dropdown(623 choices=list(USE_CASE_INFO.keys()),624 value="Accessibility Assistant",625 label="Choose Real-World Use Case"626 )627 use_case_context = gr.Textbox(628 label="Optional Context",629 placeholder="Example: school corridor / grocery shelf / street crossing / museum object",630 lines=2631 )632 use_case_btn = gr.Button("Run Use Case Analysis", variant="primary")633 with gr.Column():634 use_case_card = gr.Markdown(get_use_case_card("Accessibility Assistant"))635 use_case_output = gr.Textbox(636 label="Use Case Output",637 lines=16,638 max_lines=24,639 show_copy_button=True640 )641 642 use_case.change(fn=get_use_case_card, inputs=use_case, outputs=use_case_card)643 use_case_btn.click(644 fn=analyze_use_case,645 inputs=[shared_image, use_case, use_case_context],646 outputs=use_case_output647 )648 649 with gr.Tab("Persona Playground"):650 gr.Markdown("Make the same image speak through different roles. This is great for grabbing attention at an exhibition.")651 652 with gr.Row():653 with gr.Column():654 persona = gr.Dropdown(655 choices=[656 "Teacher",657 "Tour Guide",658 "Safety Officer",659 "Journalist",660 "Retail Manager",661 "Emergency Responder",662 "Storyteller",663 "Accessibility Coach"664 ],665 value="Teacher",666 label="Choose Persona"667 )668 tone = gr.Dropdown(669 choices=["Friendly", "Professional", "Calm", "Excited", "Analytical", "Simple"],670 value="Friendly",671 label="Tone"672 )673 persona_goal = gr.Textbox(674 label="Goal",675 placeholder="Example: explain to children / brief judges / guide a visitor",676 lines=2677 )678 persona_btn = gr.Button("Transform Through Persona", variant="primary")679 680 with gr.Column():681 persona_output = gr.Textbox(682 label="Persona Response",683 lines=18,684 max_lines=26,685 show_copy_button=True686 )687 688 persona_btn.click(689 fn=persona_playground,690 inputs=[shared_image, persona, tone, persona_goal],691 outputs=persona_output692 )693 694 with gr.Tab("Mission Lab"):695 gr.Markdown("This tab gives the app unusual interaction playgrounds. These are excellent for proving flexibility, creativity, and responsible reasoning.")696 697 with gr.Row():698 with gr.Column():699 mission = gr.Radio(700 choices=[701 "Hidden Detail Hunt",702 "Exhibit Quiz Maker",703 "Pitch From the Picture",704 "Evidence Board",705 "Story Spark",706 "Accessibility Voiceover"707 ],708 value="Hidden Detail Hunt",709 label="Choose Mission"710 )711 mission_context = gr.Textbox(712 label="Mission Context",713 placeholder="Example: target audience is school students / judges / visually impaired users",714 lines=2715 )716 mission_btn = gr.Button("Run Mission", variant="primary")717 with gr.Column():718 mission_output = gr.Textbox(719 label="Mission Output",720 lines=18,721 max_lines=28,722 show_copy_button=True723 )724 725 mission_btn.click(726 fn=mission_lab,727 inputs=[shared_image, mission, mission_context],728 outputs=mission_output729 )730 731 with gr.Tab("Ask the Image"):732 gr.Markdown("Ask anything about the uploaded image. This makes the demo feel conversational rather than static.")733 734 with gr.Row():735 with gr.Column():736 user_question = gr.Textbox(737 label="Ask a Question About the Image",738 placeholder="What is the most important object here? / Does this look crowded? / What should a student learn from this?",739 lines=2740 )741 ask_btn = gr.Button("Ask", variant="primary")742 with gr.Column():743 ask_output = gr.Textbox(744 label="Answer",745 lines=12,746 max_lines=20,747 show_copy_button=True748 )749 750 ask_btn.click(751 fn=ask_image,752 inputs=[shared_image, user_question],753 outputs=ask_output754 )755 756 with gr.Tab("Compare Booth"):757 gr.Markdown("One image, three minds. This tab is strong for proving that the same model can support different goals.")758 759 compare_context = gr.Textbox(760 label="Optional Compare Context",761 placeholder="Example: public road / classroom / tourist spot",762 lines=2763 )764 compare_btn = gr.Button("Run 3-Way Compare", variant="primary")765 766 with gr.Row():767 compare_out_1 = gr.Textbox(label="Accessibility Lens", lines=14, show_copy_button=True)768 compare_out_2 = gr.Textbox(label="Safety Lens", lines=14, show_copy_button=True)769 compare_out_3 = gr.Textbox(label="Teaching Lens", lines=14, show_copy_button=True)770 771 compare_btn.click(772 fn=compare_booth,773 inputs=[shared_image, compare_context],774 outputs=[compare_out_1, compare_out_2, compare_out_3]775 )776 777 with gr.Tab("Live Exhibit Script"):778 gr.Markdown("Use this tab at the end of your demo. It gives you clean lines to say in front of judges.")779 780 script_use_case = gr.Dropdown(781 choices=list(USE_CASE_INFO.keys()),782 value="Accessibility Assistant",783 label="Choose Your Main Showcase Angle"784 )785 script_btn = gr.Button("Generate Pitch Script", variant="primary")786 script_output = gr.Markdown()787 788 script_btn.click(789 fn=generate_exhibit_script,790 inputs=script_use_case,791 outputs=script_output792 )793 794 with gr.Tab("Project Info"):795 gr.Markdown(INFO_PAGE)796 797 gr.Markdown("""798---799### Extra Exhibition Tips800 801**Best live flow**802- start with Accessibility Assistant 803- switch to Persona Playground 804- show Evidence Board in Mission Lab 805- finish with Compare Booth 806- close using Live Exhibit Script807 808**Why that works**809You show usefulness, creativity, responsibility, and communication in one go.810 811**Note**812The Compare Booth runs the model three times, so it can be slower on CPU.813""")814 815 816if __name__ == "__main__":817 demo.launch(818 share=False,819 show_error=True,820 server_name="0.0.0.0",821 server_port=7860822 )