coco-gelamay/missing-items

1from transformers import SegformerFeatureExtractor, SegformerForSemanticSegmentation2from transformers import pipeline3from PIL import Image4import gradio as gr5import torch6 7#this converts text to speech8fastspeech = gr.Interface.load("huggingface/facebook/fastspeech2-en-ljspeech")9 10#this function detects the objects in the room11def object_classify(img1,img2):12 13  feature_extractor = SegformerFeatureExtractor.from_pretrained("nvidia/segformer-b2-finetuned-ade-512-512")14  model = SegformerForSemanticSegmentation.from_pretrained("nvidia/segformer-b2-finetuned-ade-512-512")15  16  object_detector = pipeline(task="image-segmentation", model = model, feature_extractor = feature_extractor)17  18  #list of dictionaries19  dict_obj1 = object_detector(img1) 20  dict_obj2 = object_detector(img2)21  22  #list of object labels present in the image23  objects_1=[]24  objects_2=[]25  26  #this is will read by the fastspeech27  tts_words=['The missing items are']28  29  #gets the label from each dictionary30  for i in dict_obj1:31    objects_1.append(i['label'])32  33  for j in dict_obj2:34    objects_2.append(j['label'])35   36  #gets the uncommon elements from the 2 lists37  missing_objects= list(set(objects_1)-set(objects_2))38  39  if len(missing_objects)==0:40    tts_words.append('None')41  elif len(missing_objects)==1:42    tts_words[0]='The missing item is a'43    tts_words.extend(missing_objects)44  else:45    tts_words.extend(missing_objects)46    47  gonna_process=' '.join(tts_words)48 49 50  return missing_objects, fastspeech(gonna_process)51 52  53TITLE = 'Missing Items using Nvidia Segformer'54DESCRIPTION = 'Input two pictures. First image being the original and second is the one with the missing item/s. This will output a list of items that are missing and an audio version of it'55EXAMPLES = [['Bedroom_1.jpg'],['Bedroom_2.jpg']]56 57INPUTS=[gr.inputs.Image(type = 'pil', label='Original Image'),gr.inputs.Image(type = 'pil', label='Second Image')]58OUTPUTS=[gr.outputs.Textbox(label='Missing Item/s is/are'),gr.outputs.Audio(type="auto", label="Missing Items Audio")]59 60 61interface=gr.Interface(object_classify,62                       INPUTS,63                       OUTPUTS,64                       examples = EXAMPLES,65                       title = TITLE, 66                       description=DESCRIPTION, allow_flagging="never")67 68 69interface.launch()