vinid/webplip

1import streamlit as st2import pandas as pd3import numpy as np4from PIL import Image5import pickle6import tokenizers7import torch8 9from transformers import (10 11    CLIPModel,12    AutoProcessor13)14import streamlit.components.v1 as components15import base6416 17def render_svg(svg_filename):18    with open(svg_filename,"r") as f:19        lines = f.readlines()20        svg=''.join(lines)21    """Renders the given svg string."""22    b64 = base64.b64encode(svg.encode('utf-8')).decode("utf-8")23    html = r'<img src="data:image/svg+xml;base64,%s"/>' % b6424    st.write(html, unsafe_allow_html=True)25 26@st.cache(27    hash_funcs={28        torch.nn.parameter.Parameter: lambda _: None,29        tokenizers.Tokenizer: lambda _: None,30        tokenizers.AddedToken: lambda _: None31    }32)33def load_path_clip():34    model = CLIPModel.from_pretrained("vinid/plip")35    processor = AutoProcessor.from_pretrained("vinid/plip")36    return model, processor37 38@st.cache39def init():40    with open('data/twitter.asset', 'rb') as f:41        data = pickle.load(f)42    meta = data['meta'].reset_index(drop=True)43    image_embedding = data['image_embedding']44    text_embedding = data['text_embedding']45    print(meta.shape, image_embedding.shape)46    validation_subset_index = meta['source'].values == 'Val_Tweets'47    return meta, image_embedding, text_embedding, validation_subset_index48 49def embed_images(model, images, processor):50    inputs = processor(images=images)51    pixel_values = torch.tensor(np.array(inputs["pixel_values"]))52 53    with torch.no_grad():54        embeddings = model.get_image_features(pixel_values=pixel_values)55    return embeddings56 57def embed_texts(model, texts, processor):58    inputs = processor(text=texts, padding="longest")59    input_ids = torch.tensor(inputs["input_ids"])60    attention_mask = torch.tensor(inputs["attention_mask"])61 62    with torch.no_grad():63        embeddings = model.get_text_features(64            input_ids=input_ids, attention_mask=attention_mask65        )66    return embeddings67 68 69def app():70 71    st.title('Text to Image Retrieval')72    st.markdown('#### A pathology image search engine that geos from texts to images.')73    74    col1, col2 = st.columns([1,1])75    with col1:76        st.markdown("The text-to-image retrieval system can serve as an image search engine, enabling users to match images from multiple queries and retrieve the most relevant image based on a sentence description. This generic system can comprehend semantic and interrelated knowledge, such as “Breast tumor surrounded by fat”.")77        st.markdown("Unlike searching keywords and sentences from Google and indirectly matching the images from the target text, our proposed pathology image retrieval allows direct comparison between input sentences and images.")78    with col2:79        render_svg("resources/SVG/Asset 54.svg")80 81    meta, image_embedding, text_embedding, validation_subset_index = init()82    model, processor = load_path_clip()83 84    st.markdown('### Search')85    st.markdown('How to use this: first of all, select a dataset on which to do retrieval.\n'86                'Then, either select a predefined search query or input one yourself.')87 88 89    col1, col2 = st.columns(2)90    with col1:91        data_options = ["All Twitter Data (03/21/2006 — 01/15/2023)",92                        "Validation Twitter data (11/16/2022 — 01/15/2023)"]93        st.selectbox(94            "Dataset",95            key="datapool",96            options=data_options,97        )98 99    with col2:100        retrieval_options = ["Image only",101                             "Text and image (beta)",102                             ]103        st.radio(104            "Similarity calcuation 👉",105            key="calculation_option",106            options=retrieval_options,107        )108 109    col1, col2 = st.columns(2)110 111    with col1:112        # Create selectbox113        examples = ['Breast tumor surrounded by fat',114                    'HER2+ breast tumor',115                    'Colorectal cancer tumor on epithelium',116                    'An image of endometrium epithelium',117                    'Breast cancer DCIS',118                    'Papillary carcinoma in breast tissue',119                    ]120        query_1 = st.selectbox("Select an example", options=examples)121 122        col1_submit = True123 124    with col2:125        form = st.form(key='my_form')126        query_2 = form.text_input(label='Or input your custom query:')127        submit_button = form.form_submit_button(label='Submit')128 129    if submit_button:130        col1_submit = False131 132    if col1_submit:133        query = query_1134    else:135        query = query_2136 137    input_text = embed_texts(model, [query], processor)[0].detach().cpu().numpy()138    input_text = input_text/np.linalg.norm(input_text)139 140    # Sort IDs by cosine-similarity from high to low141 142    if st.session_state.calculation_option == retrieval_options[0]:  # Image only143        similarity_scores = input_text.dot(image_embedding.T)144    else:  # Text and Image145        similarity_scores_i = input_text.dot(image_embedding.T)146        similarity_scores_t = input_text.dot(text_embedding.T)147        similarity_scores_i = similarity_scores_i / np.max(similarity_scores_i)148        similarity_scores_t = similarity_scores_t / np.max(similarity_scores_t)149        similarity_scores = (similarity_scores_i + similarity_scores_t) / 2150 151    ############################################################152    # Get top results153    ############################################################154    topn = 5155    df = pd.DataFrame(np.c_[np.arange(len(meta)), similarity_scores, meta['weblink'].values], columns = ['idx', 'score', 'twitterlink'])156    if st.session_state.datapool == data_options[1]: #Use val twitter data157        df = df.loc[validation_subset_index,:]158    df = df.sort_values('score', ascending=False)159    df = df.drop_duplicates(subset=['twitterlink'])160    best_id_topk = df['idx'].values[:topn]161    target_scores = df['score'].values[:topn]162    target_weblinks = df['twitterlink'].values[:topn]163 164 165    ############################################################166    # Display results167    ############################################################168 169    text = '<font size="4">Your input query: <span style="background-color: rgb(230,230,230);"><b>%s</b></span>' % query + \170        ' (Try search it directly on [Twitter](https://twitter.com/search?q=%s&src=typed_query) or [Google](https://www.google.com/search?q=%s))</font>' % (query.replace(' ', '%20'), query.replace(' ', '+'))171    st.markdown(text, unsafe_allow_html=True)172 173    st.markdown('#### Top 5 results:')174    topk_options = ['1st', '2nd', '3rd', '4th', '5th']175    tab = {}176    tab[0], tab[1], tab[2] = st.columns(3)177    for i in [0,1,2]:178        with tab[i]:179            topn_value = i180            topn_txt = topk_options[i]181            st.caption(f'The {topn_txt} relevant image (similarity = {target_scores[topn_value]:.4f})')182            components.html('''183                <blockquote class="twitter-tweet">184                    <a href="%s"></a>185                </blockquote>186                <script async src="https://platform.twitter.com/widgets.js" charset="utf-8">187                </script>188                ''' % target_weblinks[topn_value],189            height=600)190 191    tab[3], tab[4], tab[5] = st.columns(3)192    for i in [3,4]:193        with tab[i]:194            topn_value = i195            topn_txt = topk_options[i]196            st.caption(f'The {topn_txt} relevant image (similarity = {target_scores[topn_value]:.4f})')197            components.html('''198                <blockquote class="twitter-tweet">199                    <a href="%s"></a>200                </blockquote>201                <script async src="https://platform.twitter.com/widgets.js" charset="utf-8">202                </script>203                ''' % target_weblinks[topn_value],204            height=800)205 206 207 208    st.markdown("""---""")209    st.markdown('Disclaimer')210    st.caption('Please be advised that this function has been developed in compliance with the Twitter policy of data usage and sharing. It is important to note that the results obtained from this function are not intended to constitute medical advice or replace consultation with a qualified medical professional. The use of this function is solely at your own risk and should be consistent with applicable laws, regulations, and ethical considerations. We do not warrant or guarantee the accuracy, completeness, suitability, or usefulness of this function for any particular purpose, and we hereby disclaim any liability arising from any reliance placed on this function or any results obtained from its use. If you wish to review the original Twitter post, you should access the source page directly on Twitter.')211 212    st.markdown('Privacy statement')213    st.caption('In accordance with the privacy and control policy of Twitter, we hereby declared that the data redistributed by us shall only comprise of Tweet IDs. The Tweet IDs will be employed to establish a linkage with the original Twitter post, as long as the original post is still accessible. The hyperlink will cease to function if the user deletes the original post. It is important to note that all tweets displayed on our service have already been classified as non-sensitive by Twitter. It is strictly prohibited to redistribute any content apart from the Tweet IDs. Any distribution carried out must adhere to the laws and regulations applicable in your jurisdiction, including export control laws and embargoes.')214 215 216 217 218 219 220 221 222 223 224 225 226 227