divyanshu1807gupta/caption_api

1from flask import Flask,request2import google.generativeai as palm3import re4import pickle5import numpy as np6import requests7from PIL import Image8from io import BytesIO9from tensorflow.keras.applications.vgg16 import VGG16, preprocess_input10from tensorflow.keras.preprocessing.image import load_img, img_to_array11from tensorflow.keras.preprocessing.text import Tokenizer12from tensorflow.keras.preprocessing.sequence import pad_sequences13from tensorflow.keras.models import Model14from tensorflow.keras.utils import to_categorical, plot_model15from tensorflow.keras.layers import Input, Dense, LSTM, Embedding, Dropout, add16from tensorflow.keras.models import load_model17 18 19#tokenizer=pickle.load(open('tokenizer.pkl','rb'))20#vgg_model = load_model('vgg_model.h5')21model = load_model('best_model.h5')22max_len=3523 24 25with open('captions.txt','r') as f:26    next(f)27    caption_file=f.read()28 29captions={}30for line in caption_file.split('\n'):31    values=line.split(",")32    if(len(line)<2):33        continue34    #get image_id35    image_id=values[0]36    image_id=image_id.split('.')[0]37    #get caption38    caption=values[1:]39    caption=" ".join(caption)40    #mapping caption41    if image_id not in captions:42        captions[image_id]=[]43    captions[image_id].append(caption)44 45def clean(captions):46    for key,caption_ in captions.items():47        for i in range(len(caption_)):48            caption=caption_[i]49            #process caption50            caption=caption.lower()51            caption = re.sub('[^a-zA-Z]', ' ', caption)52            caption = re.sub('\s+', ' ', caption)53            caption=" ".join([word for word in caption.split() if len(word)>1])54            caption="startseq "+caption+" endseq"55            caption_[i]=caption56 57clean(captions)58 59all_captions=[]60for key,caption_ in captions.items():61        for i in range(len(caption_)):62            all_captions.append(caption_[i])63 64tokenizer=Tokenizer()65tokenizer.fit_on_texts(all_captions)66 67# load vgg16 model68vgg_model = VGG16()69# restructure the model70vgg_model = Model(inputs=vgg_model.inputs, outputs=vgg_model.layers[-2].output)71 72def index_to_word(indx,tokenizer):73  for word,index in tokenizer.word_index.items():74    if index == indx:75      return word76  return None77 78def predict_captions(model,image,tokenizer,max_len):79  in_text='startseq'80  for i in range(max_len):81    seq=tokenizer.texts_to_sequences([in_text])[0]82    seq=pad_sequences([seq],max_len)[0]83    if len(image.shape) == 3:84            image = np.expand_dims(image, axis=0)85    y_pred=model.predict([image, np.expand_dims(seq, axis=0)],verbose=0)86    y_pred=np.argmax(y_pred)87 88    word=index_to_word(y_pred,tokenizer)89    if word == None:90      break91    in_text += " " + word92    if word == 'endseq':93      break94  return in_text95 96def caption_generator(url):97    #load image98    response = requests.get(url)99    image= Image.open(BytesIO(response.content))100    image = image.resize((224,224))101    #convert image into numpy array102    image=img_to_array(image)103    #reshape image104    image=image.reshape((1,image.shape[0],image.shape[1],image.shape[2]))105    #preprrocess image for vgg16106    image=preprocess_input(image)107    #extract features108    feature=vgg_model.predict(image,verbose=0)109    y_pred = predict_captions(model, feature, tokenizer, max_len)110    #plt.imshow(image_pic)111    return y_pred112 113app=Flask(__name__)114 115@app.route('/')116def home():117    return "HELLO WORLD"118 119@app.route('/predict',methods=['POST'])120def predict():121    url=request.get_json()122    print(url)123    result=caption_generator(url['url'])124    palm.configure(api_key='AIzaSyDDXOjF1BBgJM6g1tMV-6tcI7xh9-ctvQU')125    #models = [m for m in palm.list_models() if 'generateText' in m.supported_generation_methods]126    #model = models[0].name127    model="models/text-bison-001"128    prompt = "Generate a creative & attractive instagram caption of 10-30 words words for" + str(result)129    completion = palm.generate_text(130        model=model,131        prompt=prompt,132        temperature=0,133        # The maximum length of the response134        max_output_tokens=100,135    )136    return completion.result137    #return {'caption':str(result)}138 139if __name__ == '__main__':140    app.run(debug=True)