vinid/webplip

1import streamlit as st2import pandas as pd3import numpy as np4from PIL import Image5import requests6import tokenizers7import os8from io import BytesIO9import pickle10import base6411import datetime12 13import torch14from transformers import (15    VisionTextDualEncoderModel,16    AutoFeatureExtractor,17    AutoTokenizer,18    CLIPModel,19    AutoProcessor20)21import streamlit.components.v1 as components22from st_clickable_images import clickable_images #pip install st-clickable-images23 24 25@st.cache(26    hash_funcs={27        torch.nn.parameter.Parameter: lambda _: None,28        tokenizers.Tokenizer: lambda _: None,29        tokenizers.AddedToken: lambda _: None30    }31)32def load_path_clip():33    model = CLIPModel.from_pretrained("vinid/plip")34    processor = AutoProcessor.from_pretrained("vinid/plip")35    return model, processor36 37@st.cache38def init():39    with open('data/twitter.asset', 'rb') as f:40        data = pickle.load(f)41    meta = data['meta'].reset_index(drop=True)42    image_embedding = data['image_embedding']43    text_embedding = data['text_embedding']44    print(meta.shape, image_embedding.shape)45    validation_subset_index = meta['source'].values == 'Val_Tweets'46    return meta, image_embedding, text_embedding, validation_subset_index47 48def embed_images(model, images, processor):49    inputs = processor(images=images)50    pixel_values = torch.tensor(np.array(inputs["pixel_values"]))51 52    with torch.no_grad():53        embeddings = model.get_image_features(pixel_values=pixel_values)54    return embeddings55 56def embed_texts(model, texts, processor):57    inputs = processor(text=texts, padding="longest")58    input_ids = torch.tensor(inputs["input_ids"])59    attention_mask = torch.tensor(inputs["attention_mask"])60 61    with torch.no_grad():62        embeddings = model.get_text_features(63            input_ids=input_ids, attention_mask=attention_mask64        )65    return embeddings66def app():67    st.title('Image to Image Retrieval')68    st.markdown('#### A pathology image search engine that correlate images with images.')69    st.markdown("Image-to-image retrieval can be used to retrieve pathology images that have contents similar to the target image input, with the ability to comprehend the key components from the input image.")70    71    st.markdown('#### Demo')72 73    meta, image_embedding, text_embedding, validation_subset_index = init()74    model, processor = load_path_clip()75 76 77    col1, col2 = st.columns(2)78    with col1:79        data_options = ["All twitter data (03/21/2006 — 01/15/2023)",80                        "Twitter validation data (11/16/2022 — 01/15/2023)"]81        st.radio(82            "Choose dataset for image retrieval 👉",83            key="datapool",84            options=data_options,85        )86    with col2:87        retrieval_options = ["Image only",88                            "Text and image (beta)",89                             ]90        st.radio(91            "Similarity calcuation 👉",92            key="calculation_option",93            options=retrieval_options,94        )95 96 97    st.markdown('Try out following examples:')98    example_path = 'data/example_images'99    list_of_examples = [os.path.join(example_path, v) for v in os.listdir(example_path)]100    example_imgs = []101    for file in list_of_examples:102        with open(file, "rb") as image:103            encoded = base64.b64encode(image.read()).decode()104            example_imgs.append(f"data:image/jpeg;base64,{encoded}")105    clicked = clickable_images(106        example_imgs,107        titles=[f"Image #{str(i)}" for i in range(len(example_imgs))],108        div_style={"display": "flex", "justify-content": "center", "flex-wrap": "wrap"},109        img_style={"margin": "5px", "height": "70px"},110    )111    isExampleClicked = False112    if clicked > -1:113        image = Image.open(list_of_examples[clicked])114        isExampleClicked = True115        116 117 118 119 120 121    col1, col2, _ = st.columns(3)122    with col1:123        query = st.file_uploader("Choose a file to upload")124 125 126    proceed = False127    if query:128        image = Image.open(query)129        proceed = True130    elif isExampleClicked:131        proceed = True132 133    if proceed:134        with col2:135            st.image(image, caption='Your upload')136 137        input_image = embed_images(model, [image], processor)[0].detach().cpu().numpy()138 139        input_image = input_image/np.linalg.norm(input_image)140        141        # Sort IDs by cosine-similarity from high to low142 143        if st.session_state.calculation_option == retrieval_options[0]: # Image only144            similarity_scores = input_image.dot(image_embedding.T)145        else: # Text and Image146            similarity_scores_i = input_image.dot(image_embedding.T)147            similarity_scores_t = input_image.dot(text_embedding.T)148            similarity_scores_i = similarity_scores_i/np.max(similarity_scores_i)149            similarity_scores_t = similarity_scores_t/np.max(similarity_scores_t)150            similarity_scores = (similarity_scores_i + similarity_scores_t)/2151 152 153        ############################################################154        # Get top results155        ############################################################156        topn = 5157        df = pd.DataFrame(np.c_[np.arange(len(meta)), similarity_scores, meta['weblink'].values], columns = ['idx', 'score', 'twitterlink'])158        if st.session_state.datapool == data_options[1]: #Use val twitter data159            df = df.loc[validation_subset_index,:]160        df = df.sort_values('score', ascending=False)161        df = df.drop_duplicates(subset=['twitterlink'])162        best_id_topk = df['idx'].values[:topn]163        target_scores = df['score'].values[:topn]164        target_weblinks = df['twitterlink'].values[:topn]165 166 167 168        ############################################################169        # Display results170        ############################################################171        172        st.markdown('#### Top 5 results:')173        topk_options = ['1st', '2nd', '3rd', '4th', '5th']174        tab = {}175        tab[0], tab[1], tab[2] = st.columns(3)176        for i in [0,1,2]:177            with tab[i]:178                topn_value = i179                topn_txt = topk_options[i]180                st.caption(f'The {topn_txt} relevant image (similarity = {target_scores[topn_value]:.4f})')181                components.html('''182                    <blockquote class="twitter-tweet">183                        <a href="%s"></a>184                    </blockquote>185                    <script async src="https://platform.twitter.com/widgets.js" charset="utf-8">186                    </script>187                    ''' % target_weblinks[topn_value],188                height=600)189 190        tab[3], tab[4], tab[5] = st.columns(3)191        for i in [3,4]:192            with tab[i]:193                topn_value = i194                topn_txt = topk_options[i]195                st.caption(f'The {topn_txt} relevant image (similarity = {target_scores[topn_value]:.4f})')196                components.html('''197                    <blockquote class="twitter-tweet">198                        <a href="%s"></a>199                    </blockquote>200                    <script async src="https://platform.twitter.com/widgets.js" charset="utf-8">201                    </script>202                    ''' % target_weblinks[topn_value],203                height=800)204 205 206 207 208 209 210 211 212 213 214    st.markdown("""---""")215    st.markdown('Disclaimer')216    st.caption('Please be advised that this function has been developed in compliance with the Twitter policy of data usage and sharing. It is important to note that the results obtained from this function are not intended to constitute medical advice or replace consultation with a qualified medical professional. The use of this function is solely at your own risk and should be consistent with applicable laws, regulations, and ethical considerations. We do not warrant or guarantee the accuracy, completeness, suitability, or usefulness of this function for any particular purpose, and we hereby disclaim any liability arising from any reliance placed on this function or any results obtained from its use. If you wish to review the original Twitter post, you should access the source page directly on Twitter.')217 218    st.markdown('Privacy statement')219    st.caption('In accordance with the privacy and control policy of Twitter, we hereby declared that the data redistributed by us shall only comprise of Tweet IDs. The Tweet IDs will be employed to establish a linkage with the original Twitter post, as long as the original post is still accessible. The hyperlink will cease to function if the user deletes the original post. It is important to note that all tweets displayed on our service have already been classified as non-sensitive by Twitter. It is strictly prohibited to redistribute any content apart from the Tweet IDs. Any distribution carried out must adhere to the laws and regulations applicable in your jurisdiction, including export control laws and embargoes.')220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240