JuanMa360/vision_intelligence

1from PIL import Image2import json3import gradio as gr4import requests5from transformers import CLIPProcessor, CLIPModel, pipeline, BlipProcessor, BlipForConditionalGeneration6 7model = CLIPModel.from_pretrained("model")8processor = CLIPProcessor.from_pretrained("tokenizer")9vqa_pipeline = pipeline("visual-question-answering",model="vqa")10 11space_type_labels = ["living room", "bedroom", "kitchen", "terrace", "closet","bathroom", "dining room", "office", "garage", "garden",12    "balcony", "attic", "hallway","gym", "playroom", "storage room", "studio","is_exterior","swimming pool","others"]13 14equipment_questions = [15    "Does the image show outdoor furniture?",16    "Does the image show a parasol?",17    "Does the image show a pergola?",18    "Does the image show a grill?",19    "Does the image show a heater?",20    "Does the image show outdoor lighting?",21    "Does the image show planters?",22    "Does the image show water features?",23    "Does the image show floor coverings?",24    "Does the image show decorative items?",25    "Does the image show entertainment equipment?",26    "Does the image show protective materials?"27]28 29weights = {30    "Does the image show outdoor furniture?": 0.15,31    "Does the image show a parasol?": 0.05,32    "Does the image show a pergola?": 0.1,33    "Does the image show a grill?": 0.15,34    "Does the image show a heater?": 0.1,35    "Does the image show outdoor lighting?": 0.1,36    "Does the image show planters?": 0.05,37    "Does the image show water features?": 0.1,38    "Does the image show floor coverings?": 0.05,39    "Does the image show decorative items?": 0.05,40    "Does the image show entertainment equipment?": 0.05,41    "Does the image show protective materials?": 0.0542}43 44luminosity_classes = [45    'A well-lit room with abundant natural light, showcasing windows or a balcony through which sunlight passes unobstructed.',46    'A room depicted in darkness, where there is minimal or no visible light source.',47    'A room illuminated by artificial light sources such as lamps or ceiling lights.'48]49 50#luminosity_classes = [51#    "A room filled with natural daylight.",52#    "A room lit by artificial lights.",53#    "A dark room with no lights."54#]55 56luminosity_labels = ['natural_light', 'no_light', 'artificial_light']57 58#view_questions = [59    #"Is this a panoramic view?",60#    "Is this a city view?",61#    "Is this a view of greenery?",62#    "Is this a mountain view?",63#    "Is this a view of the sea?"64#]65 66view_questions = [67   # "This is a panoramic view, showing a wide expanse of the surroundings.",68    "This is a city view, showing buildings, streets, and urban areas.",69    "This is a view of greenery, including trees, parks, or gardens.",70    "This is a mountain view, showing mountains and hilly landscapes.",71    "This is a view of the sea"72]73 74view_labels = ['city', 'greenery', 'mountain', 'sea']75 76certainty_classes = [77    'Windows, balconies, or terraces with an unobstructed outward view',78    'exterior view of a building or appearance of a house or apartment',79    'Artificial or fake view of any city or sea',80    'View obstructed by objects such as buildings, trees, or other structures',81    'Hallway or interior view with no outdoor visibility'82]83 84#certainty_classes = ['Windows, balconies, or terraces with an unobstructed outward view','Exterior view appearance of a house or apartment','unreal picture or fake of any city or sea view','view unfree from any obstructive objects such as buildings, trees, or other structures, and ideally seen through windows, balconies, or terraces','hallway']85 86render_classes = [87    "This is a realistic photo of an interior.",88    "This is a computer-generated render of an interior.",89    "This is a realistic photo of an exterior.",90    "This is a computer-generated render of an exterior."91]92 93threshold = 094 95def calculate_equipment_score(image_results, weights):96    score = sum(weights[question] for question, present in image_results.items() if present)97    return score98 99def calculate_luminosity_score(processed_image):100    inputs = processor(text=luminosity_classes, images=processed_image, return_tensors="pt", padding=True)101    outputs = model(**inputs)102    logits_per_image = outputs.logits_per_image103    probs = logits_per_image.softmax(dim=1)104    probabilities_list = probs.squeeze().tolist()105    luminosity_score = {class_name: probability for class_name, probability in zip(luminosity_labels, probabilities_list)}106    return luminosity_score107 108def calculate_space_type(processed_image):109    inputs = processor(text=space_type_labels, images=processed_image, return_tensors="pt", padding=True)110    outputs = model(**inputs)111    logits_per_image = outputs.logits_per_image112    probs = logits_per_image.softmax(dim=1)113    probabilities_list = probs.squeeze().tolist()114    space_type_score = {class_name: probability for class_name, probability in zip(space_type_labels, probabilities_list)}115    return space_type_score116 117def certainty(processed_image):118    inputs = processor(text=certainty_classes, images=processed_image, return_tensors="pt", padding=True)119    outputs = model(**inputs)120    logits_per_image = outputs.logits_per_image121    probs = logits_per_image.softmax(dim=1)122    probabilities_list = probs.squeeze().tolist()123    is_fake_score = {class_name: probability for class_name, probability in zip(certainty_classes, probabilities_list)}124    return is_fake_score125 126def views(processed_image):127    inputs = processor(text=view_questions, images=processed_image, return_tensors="pt", padding=True)128    outputs = model(**inputs)129    logits_per_image = outputs.logits_per_image130    probs = logits_per_image.softmax(dim=1)131    probabilities_list = probs.squeeze().tolist()132    views_score = {class_name: probability for class_name, probability in zip(view_labels, probabilities_list)}133    return views_score134 135def calculate_is_render(processed_image):136    render_inputs = processor(text=render_classes, images=processed_image, return_tensors="pt", padding=True)137    render_outputs = model(**render_inputs)138    render_logits = render_outputs.logits_per_image139    render_probs = render_logits.softmax(dim=1)140    render_probabilities_list = render_probs.squeeze().tolist()141    render_score = {class_name: probability for class_name, probability in zip(render_classes, render_probabilities_list)}142    is_render_prob = render_score["This is a realistic photo of an interior."]+render_score["This is a realistic photo of an exterior."]143    return is_render_prob144 145def generate_answer(image):146 147    processed_image = image148 149    image_data = {150        "image_context": None,151        "validation": None,152        "equipment_score": None,153        "luminosity_score": {"score": None},154        "view_type": {"views": None, "certainty_score": None}155    }156 157    space_type_score = calculate_space_type(processed_image)158    max_space_type = max(space_type_score, key=space_type_score.get)159    if space_type_score[max_space_type] >= 0:160        space_type = max_space_type.lower()161        if space_type == "patio":162            space_type = "terrace"163    image_data["image_context"] = space_type_score164 165    image_results = {}166    if max_space_type == "terrace":167      for question in equipment_questions:168          result = vqa_pipeline(processed_image, question, top_k=1)169          answer = result[0]['answer'].lower() == "yes"170          image_results[question] = answer171      equipment_score = calculate_equipment_score(image_results, weights)172      image_data["equipment_score"] = equipment_score173 174    result = vqa_pipeline(processed_image, "Is there a real window?", top_k=1)175    has_window = result[0]176    image_data["validation"] = "pass validation" if has_window['score'] > 0.9 else "No candidate"177 178    window_exists = has_window["answer"].lower() == "yes" and has_window["score"] > 0.9179 180    if max_space_type in ["bedroom", "living room", "kitchen"] and window_exists:181      luminosity_score = calculate_luminosity_score(processed_image)182      image_data["luminosity_score"]['score'] = luminosity_score['natural_light']183 184      view = views(processed_image)185      image_data["view_type"]["views"] = view186 187      certainty_score = certainty(processed_image)188      certainty_score = list(certainty_score.values())[0]189      image_data["view_type"]["certainty_score"] = certainty_score190 191    #is_render = calculate_is_render(processed_image)192    #image_data["is_render"] = is_render193 194    return json.dumps(image_data, indent=4)195 196 197image_input = gr.Image(type="pil", label="Upload Image")198 199iface = gr.Interface(200    fn=generate_answer, 201    inputs=[image_input], 202    outputs="text",203    title="Vision intelligence",204    description="Upload an image"205)206 207iface.launch()