ml6team/post-processing-summarization

1from typing import AnyStr, Dict2 3import itertools4import streamlit as st5import en_core_web_lg6 7import torch.nn.parameter8from bs4 import BeautifulSoup9import numpy as np10import base6411 12from spacy_streamlit.util import get_svg13from streamlit.proto.SessionState_pb2 import SessionState14 15from custom_renderer import render_sentence_custom16from sentence_transformers import SentenceTransformer17 18from transformers import AutoTokenizer, AutoModelForTokenClassification19from transformers import pipeline20import os21 22device = torch.device("cuda" if torch.cuda.is_available() else "cpu")23HTML_WRAPPER = """<div style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; 24margin-bottom: 2.5rem">{}</div> """25 26 27@st.experimental_singleton28def get_sentence_embedding_model():29    return SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')30 31 32@st.experimental_singleton33def get_spacy():34    nlp = en_core_web_lg.load()35    return nlp36 37 38@st.experimental_singleton39def get_transformer_pipeline():40    tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-large-finetuned-conll03-english")41    model = AutoModelForTokenClassification.from_pretrained("xlm-roberta-large-finetuned-conll03-english")42    return pipeline("ner", model=model, tokenizer=tokenizer, grouped_entities=True)43 44 45@st.experimental_singleton46def get_summarizer_model():47    model_name = 'google/pegasus-cnn_dailymail'48    summarizer_model = pipeline("summarization", model=model_name, tokenizer=model_name,49                                device=0 if torch.cuda.is_available() else -1)50 51    return summarizer_model52 53 54# Page setup55st.set_page_config(56    page_title="📜 Hallucination detection in summaries 📜",57    page_icon="",58    layout="centered",59    initial_sidebar_state="auto",60    menu_items={61        'Get help': None,62        'Report a bug': None,63        'About': None,64    }65)66 67 68def list_all_article_names() -> list:69    filenames = []70    for file in sorted(os.listdir('./sample-articles/')):71        if file.endswith('.txt'):72            filenames.append(file.replace('.txt', ''))73    # Append free use possibility:74    filenames.append("Provide your own input")75    return filenames76 77 78def fetch_article_contents(filename: str) -> AnyStr:79    if filename == "Provide your own input":80        return " "81    with open(f'./sample-articles/{filename}.txt', 'r') as f:82        data = f.read()83    return data84 85 86def fetch_summary_contents(filename: str) -> AnyStr:87    with open(f'./sample-summaries/{filename}.txt', 'r') as f:88        data = f.read()89    return data90 91 92def fetch_entity_specific_contents(filename: str) -> AnyStr:93    with open(f'./entity-specific-text/{filename}.txt', 'r') as f:94        data = f.read()95    return data96 97 98def fetch_dependency_specific_contents(filename: str) -> AnyStr:99    with open(f'./dependency-specific-text/{filename}.txt', 'r') as f:100        data = f.read()101    return data102 103 104def fetch_ranked_summaries(filename: str, ranknumber: int) -> AnyStr:105    with open(f'./ranked-summaries/{filename}/Rank{ranknumber}.txt', 'r') as f:106        data = f.read()107    return data108 109 110def fetch_dependency_svg(filename: str) -> AnyStr:111    with open(f'./dependency-images/{filename}.txt', 'r') as f:112        lines = [line.rstrip() for line in f]113    return lines114 115 116def display_summary(summary_content: str):117    st.session_state.summary_output = summary_content118    soup = BeautifulSoup(summary_content, features="html.parser")119    return HTML_WRAPPER.format(soup)120 121 122def get_all_entities_per_sentence(text):123    doc = nlp(text)124 125    sentences = list(doc.sents)126 127    entities_all_sentences = []128    for sentence in sentences:129        entities_this_sentence = []130 131        # SPACY ENTITIES132        for entity in sentence.ents:133            entities_this_sentence.append(str(entity))134 135        # FLAIR ENTITIES (CURRENTLY NOT USED)136        # sentence_entities = Sentence(str(sentence))137        # tagger.predict(sentence_entities)138        # for entity in sentence_entities.get_spans('ner'):139        #     entities_this_sentence.append(entity.text)140 141        # XLM ENTITIES142        entities_xlm = [entity["word"] for entity in ner_model(str(sentence))]143        for entity in entities_xlm:144            entities_this_sentence.append(str(entity))145 146        entities_all_sentences.append(entities_this_sentence)147 148    return entities_all_sentences149 150 151def get_all_entities(text):152    all_entities_per_sentence = get_all_entities_per_sentence(text)153    return list(itertools.chain.from_iterable(all_entities_per_sentence))154 155 156def get_and_compare_entities(first_time: bool):157    if first_time:158        article_content = st.session_state.article_text159        all_entities_per_sentence = get_all_entities_per_sentence(article_content)160        entities_article = list(itertools.chain.from_iterable(all_entities_per_sentence))161        st.session_state.entities_article = entities_article162    else:163        entities_article = st.session_state.entities_article164 165    summary_content = st.session_state.summary_output166    all_entities_per_sentence = get_all_entities_per_sentence(summary_content)167    entities_summary = list(itertools.chain.from_iterable(all_entities_per_sentence))168 169    matched_entities = []170    unmatched_entities = []171    for entity in entities_summary:172        if any(entity.lower() in substring_entity.lower() for substring_entity in entities_article):173            matched_entities.append(entity)174        elif any(175                np.inner(sentence_embedding_model.encode(entity, show_progress_bar=False),176                         sentence_embedding_model.encode(art_entity, show_progress_bar=False)) > 0.9 for177                art_entity in entities_article):178            matched_entities.append(entity)179        else:180            unmatched_entities.append(entity)181 182    matched_entities = list(dict.fromkeys(matched_entities))183    unmatched_entities = list(dict.fromkeys(unmatched_entities))184 185    matched_entities_to_remove = []186    unmatched_entities_to_remove = []187 188    for entity in matched_entities:189        for substring_entity in matched_entities:190            if entity != substring_entity and entity.lower() in substring_entity.lower():191                matched_entities_to_remove.append(entity)192 193    for entity in unmatched_entities:194        for substring_entity in unmatched_entities:195            if entity != substring_entity and entity.lower() in substring_entity.lower():196                unmatched_entities_to_remove.append(entity)197 198    matched_entities_to_remove = list(dict.fromkeys(matched_entities_to_remove))199    unmatched_entities_to_remove = list(dict.fromkeys(unmatched_entities_to_remove))200 201    for entity in matched_entities_to_remove:202        matched_entities.remove(entity)203    for entity in unmatched_entities_to_remove:204        unmatched_entities.remove(entity)205 206    return matched_entities, unmatched_entities207 208 209def highlight_entities():210    summary_content = st.session_state.summary_output211    markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"212    markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"213    markdown_end = "</mark>"214 215    matched_entities, unmatched_entities = get_and_compare_entities(True)216 217    for entity in matched_entities:218        summary_content = summary_content.replace(entity, markdown_start_green + entity + markdown_end)219 220    for entity in unmatched_entities:221        summary_content = summary_content.replace(entity, markdown_start_red + entity + markdown_end)222    soup = BeautifulSoup(summary_content, features="html.parser")223    return HTML_WRAPPER.format(soup)224 225 226def highlight_entities_new(summary_str: str):227    st.session_state.summary_output = summary_str228    summary_content = st.session_state.summary_output229    markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"230    markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"231    markdown_end = "</mark>"232 233    matched_entities, unmatched_entities = get_and_compare_entities(False)234 235    for entity in matched_entities:236        summary_content = summary_content.replace(entity, markdown_start_green + entity + markdown_end)237 238    for entity in unmatched_entities:239        summary_content = summary_content.replace(entity, markdown_start_red + entity + markdown_end)240    soup = BeautifulSoup(summary_content, features="html.parser")241    return HTML_WRAPPER.format(soup)242 243 244def render_dependency_parsing(text: Dict):245    html = render_sentence_custom(text, nlp)246    html = html.replace("\n\n", "\n")247    st.write(get_svg(html), unsafe_allow_html=True)248 249 250def check_dependency(article: bool):251    if article:252        text = st.session_state.article_text253        all_entities = get_all_entities_per_sentence(text)254    else:255        text = st.session_state.summary_output256        all_entities = get_all_entities_per_sentence(text)257    doc = nlp(text)258    tok_l = doc.to_json()['tokens']259    test_list_dict_output = []260 261    sentences = list(doc.sents)262    for i, sentence in enumerate(sentences):263        start_id = sentence.start264        end_id = sentence.end265        for t in tok_l:266            if t["id"] < start_id or t["id"] > end_id:267                continue268            head = tok_l[t['head']]269            if t['dep'] == 'amod' or t['dep'] == "pobj":270                object_here = text[t['start']:t['end']]271                object_target = text[head['start']:head['end']]272                if t['dep'] == "pobj" and str.lower(object_target) != "in":273                    continue274                # ONE NEEDS TO BE ENTITY275                if object_here in all_entities[i]:276                    identifier = object_here + t['dep'] + object_target277                    test_list_dict_output.append({"dep": t['dep'], "cur_word_index": (t['id'] - sentence.start),278                                                  "target_word_index": (t['head'] - sentence.start),279                                                  "identifier": identifier, "sentence": str(sentence)})280                elif object_target in all_entities[i]:281                    identifier = object_here + t['dep'] + object_target282                    test_list_dict_output.append({"dep": t['dep'], "cur_word_index": (t['id'] - sentence.start),283                                                  "target_word_index": (t['head'] - sentence.start),284                                                  "identifier": identifier, "sentence": str(sentence)})285                else:286                    continue287    return test_list_dict_output288 289 290def render_svg(svg_file):291    with open(svg_file, "r") as f:292        lines = f.readlines()293        svg = "".join(lines)294 295        # """Renders the given svg string."""296        b64 = base64.b64encode(svg.encode("utf-8")).decode("utf-8")297        html = r'<img src="data:image/svg+xml;base64,%s"/>' % b64298        return html299 300 301def generate_abstractive_summary(text, type, min_len=120, max_len=512, **kwargs):302    text = text.strip().replace("\n", " ")303    if type == "top_p":304        text = summarization_model(text, min_length=min_len,305                                   max_length=max_len,306                                   top_k=50, top_p=0.95, clean_up_tokenization_spaces=True, truncation=True, **kwargs)307    elif type == "greedy":308        text = summarization_model(text, min_length=min_len,309                                   max_length=max_len, clean_up_tokenization_spaces=True, truncation=True, **kwargs)310    elif type == "top_k":311        text = summarization_model(text, min_length=min_len, max_length=max_len, top_k=50,312                                   clean_up_tokenization_spaces=True, truncation=True, **kwargs)313    elif type == "beam":314        text = summarization_model(text, min_length=min_len,315                                   max_length=max_len,316                                   clean_up_tokenization_spaces=True, truncation=True, **kwargs)317    summary = text[0]['summary_text'].replace("<n>", " ")318    return summary319 320 321# Load all different models (cached) at start time of the hugginface space322sentence_embedding_model = get_sentence_embedding_model()323ner_model = get_transformer_pipeline()324nlp = get_spacy()325summarization_model = get_summarizer_model()326 327# Page328st.title('📜 Hallucination detection 📜')329st.subheader("🔎 Detecting errors in generated abstractive summaries")330#st.title('📜 Error detection in summaries 📜')331 332# INTRODUCTION333st.header("🧑‍🏫 Introduction")334 335#introduction_checkbox = st.checkbox("Show introduction text", value=True)336#if introduction_checkbox:337st.markdown("""338Recent work using 🤖 **transformers** 🤖 on large text corpora has shown great success when fine-tuned on 339several different downstream NLP tasks. One such task is that of text summarization. The goal of text summarization 340is to generate concise and accurate summaries from input document(s). There are 2 types of summarization:341 342 - **Extractive summarization** merely copies informative fragments from the input. 343 - **Abstractive summarization** 344 may generate novel words. A good abstractive summary should cover principal information in the input and has to be 345 linguistically fluent. This interactive blogpost will focus on this more difficult task of abstractive summary 346 generation. Furthermore we will focus mainly on hallucination errors, and less on sentence fluency.""")347 348st.markdown("###")349st.markdown("🤔 **Why is this important?** 🤔 Let's say we want to summarize news articles for a popular "350            "newspaper. If an article tells the story of Elon Musk buying **Twitter**, we don't want our summarization "351            "model to say that he bought **Facebook** instead. Summarization could also be done for financial reports "352            "for example. In such environments, these errors can be very critical, so we want to find a way to "353            "detect them.")354st.markdown("###")355st.markdown("""To generate summaries we will use the 🐎 [PEGASUS](https://huggingface.co/google/pegasus-cnn_dailymail) 🐎356model, producing abstractive summaries from large articles. These summaries often contain sentences with different 357kinds of errors. Rather than improving the core model, we will look into possible post-processing steps to detect errors 358from the generated summaries. Throughout this blog, we will also explain the results for some methods on specific 359examples. These text blocks will be indicated and they change according to the currently selected article.""")360 361# GENERATING SUMMARIES PART362st.header("🪶 Generating summaries")363st.markdown("Let’s start by selecting an article text for which we want to generate a summary, or you can provide "364            "text yourself. Note that it’s suggested to provide a sufficiently large article, as otherwise the "365            "summary generated from it might not be optimal, leading to suboptimal performance of the post-processing "366            "steps. However, too long articles will be truncated and might miss information in the summary.")367 368st.markdown("####")369selected_article = st.selectbox('Select an article or provide your own:',370                                list_all_article_names(), index=2)371st.session_state.article_text = fetch_article_contents(selected_article)372article_text = st.text_area(373    label='Full article text',374    value=st.session_state.article_text,375    height=250376)377 378summarize_button = st.button(label='🤯 Process article content',379                             help="Start interactive blogpost")380 381if summarize_button:382    st.session_state.article_text = article_text383    st.markdown("####")384    st.markdown(385        "*Below you can find the generated summary for the article. We will discuss two approaches that we found are "386        "able to detect some common errors. Based on these errors, one could then score different summaries, indicating how "387        "factual a summary is for a given article. The idea is that in production, you could generate a set of "388        "summaries for the same article, with different parameters (or even different models). By using "389        "post-processing error detection, we can then select the best possible summary.*")390    st.markdown("####")391    if st.session_state.article_text:392        with st.spinner('Generating summary, this might take a while...'):393            if selected_article != "Provide your own input" and article_text == fetch_article_contents(394                    selected_article):395                st.session_state.unchanged_text = True396                summary_content = fetch_summary_contents(selected_article)397            else:398                summary_content = generate_abstractive_summary(article_text, type="beam", do_sample=True, num_beams=15,399                                                               no_repeat_ngram_size=4)400                st.session_state.unchanged_text = False401            summary_displayed = display_summary(summary_content)402            st.write("✍ **Generated summary:** ✍", summary_displayed, unsafe_allow_html=True)403    else:404        st.error('**Error**: No comment to classify. Please provide a comment.')405 406    # ENTITY MATCHING PART407    st.header("1️⃣ Entity matching")408    st.markdown("The first method we will discuss is called **Named Entity Recognition** (NER). NER is the task of "409                "identifying and categorising key information (entities) in text. An entity can be a singular word or a "410                "series of words that consistently refers to the same thing. Common entity classes are person names, "411                "organisations, locations and so on. By applying NER to both the article and its summary, we can spot "412                "possible **hallucinations**. ")413 414    st.markdown("Hallucinations are words generated by the model that are not supported by "415                "the source input. Deep learning based generation is [prone to hallucinate]("416                "https://arxiv.org/pdf/2202.03629.pdf) unintended text. These hallucinations degrade "417                "system performance and fail to meet user expectations in many real-world scenarios. By applying entity matching, we can improve this problem"418                " for the downstream task of summary generation.")419 420    st.markdown(" In theory all entities in the summary (such as dates, locations and so on), "421                "should also be present in the article. Thus we can extract all entities from the summary and compare "422                "them to the entities of the original article, spotting potential hallucinations. The more unmatched "423                "entities we find, the lower the factualness score of the summary. ")424    with st.spinner("Calculating and matching entities, this takes about 10-20 seconds..."):425        entity_match_html = highlight_entities()426        st.markdown("####")427        st.write(entity_match_html, unsafe_allow_html=True)428        red_text = """<font color="black"><span style="background-color: rgb(238, 135, 135); opacity: 429        1;">red</span></font> """430        green_text = """<font color="black">431            <span style="background-color: rgb(121, 236, 121); opacity: 1;">green</span>432        </font>"""433 434        markdown_start_red = "<mark class=\"entity\" style=\"background: rgb(238, 135, 135);\">"435        markdown_start_green = "<mark class=\"entity\" style=\"background: rgb(121, 236, 121);\">"436        st.markdown(437            "We call this technique **entity matching** and here you can see what this looks like when we apply this "438            "method on the summary. Entities in the summary are marked  " + green_text + " when the entity also "439                                                                                         "exists in the article, "440                                                                                         "while unmatched entities "441                                                                                         "are marked " + red_text +442            ". Several of the example articles and their summaries indicate different errors we find by using this "443            "technique. Based on the current article, we provide a short explanation of the results below **(only for "444            "example articles)**. ", unsafe_allow_html=True)445        if st.session_state.unchanged_text:446            entity_specific_text = fetch_entity_specific_contents(selected_article)447            soup = BeautifulSoup(entity_specific_text, features="html.parser")448            st.markdown("####")449            st.write("💡👇 **Specific example explanation** 👇💡", HTML_WRAPPER.format(soup), unsafe_allow_html=True)450 451    # DEPENDENCY PARSING PART452    st.header("2️⃣ Dependency comparison")453    st.markdown(454        "The second method we use for post-processing is called **Dependency Parsing**: the process in which the "455        "grammatical structure in a sentence is analysed, to find out related words as well as the type of the "456        "relationship between them. For the sentence “Jan’s wife is called Sarah” you would get the following "457        "dependency graph:")458 459    # TODO: I wonder why the first doesn't work but the second does (it doesn't show deps otherwise)460    # st.image("ExampleParsing.svg")461    st.write(render_svg('ExampleParsing.svg'), unsafe_allow_html=True)462    st.markdown(463        "Here, *“Jan”* is the *“poss”* (possession modifier) of *“wife”*. If suddenly the summary would read *“Jan’s"464        " husband…”*, there would be a dependency in the summary that is non-existent in the article itself (namely "465        "*“Jan”* is the “poss” of *“husband”*)."466        "However, often new dependencies are introduced in the summary that "467        "are still correct, as can be seen in the example below. ")468    st.write(render_svg('SecondExampleParsing.svg'), unsafe_allow_html=True)469 470    st.markdown("*“The borders of Ukraine”* have a different dependency between *“borders”* and "471                "*“Ukraine”* "472                "than *“Ukraine’s borders”*, while both descriptions have the same meaning. So just matching all "473                "dependencies between article and summary (as we did with entity matching) would not be a robust method."474                " More on the different sorts of dependencies and their description can be found [here](https://universaldependencies.org/docs/en/dep/).")475    st.markdown("However, we have found that **there are specific dependencies that are often an "476                "indication of a wrongly constructed sentence** when there is no article match. We (currently) use 2 "477                "common dependencies which - when present in the summary but not in the article - are highly "478                "indicative of factualness errors. "479                "Furthermore, we only check dependencies between an existing **entity** and its direct connections. "480                "Below we highlight all unmatched dependencies that satisfy the discussed constraints. We also "481                "discuss the specific results for the currently selected example article.")482    with st.spinner("Doing dependency parsing..."):483        if st.session_state.unchanged_text:484            for cur_svg_image in fetch_dependency_svg(selected_article):485                st.write(cur_svg_image, unsafe_allow_html=True)486            dep_specific_text = fetch_dependency_specific_contents(selected_article)487            soup = BeautifulSoup(dep_specific_text, features="html.parser")488            st.write("💡👇 **Specific example explanation** 👇💡", HTML_WRAPPER.format(soup), unsafe_allow_html=True)489        else:490            summary_deps = check_dependency(False)491            article_deps = check_dependency(True)492            total_unmatched_deps = []493            for summ_dep in summary_deps:494                if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):495                    total_unmatched_deps.append(summ_dep)496            if total_unmatched_deps:497                for current_drawing_list in total_unmatched_deps:498                    render_dependency_parsing(current_drawing_list)499 500    # CURRENTLY DISABLED501    # OUTRO/CONCLUSION502    st.header("🤝 Bringing it together")503    st.markdown("We have presented 2 methods that try to detect errors in summaries via post-processing steps. Entity "504                "matching can be used to solve hallucinations, while dependency comparison can be used to filter out "505                "some bad sentences (and thus worse summaries). These methods highlight the possibilities of "506                "post-processing AI-made summaries, but are only a first introduction. As the methods were "507                "empirically tested they are definitely not sufficiently robust for general use-cases.")508    st.markdown("####")509    st.markdown(510        "*Below we generate 3 different kind of summaries, and based on the two discussed methods, their errors are "511        "detected to estimate a summary score. Based on this basic approach, "512        "the best summary (read: the one that a human would prefer or indicate as the best one) "513        "will hopefully be at the top. We currently "514        "only do this for the example articles (for which the different summmaries are already generated). The reason "515        "for this is that HuggingFace spaces are limited in their CPU memory. We also highlight the entities as done "516        "before, but note that the rankings are done on a combination of unmatched entities and "517        "dependencies (with the latter not shown here).*")518    st.markdown("####")519 520    if selected_article != "Provide your own input" and article_text == fetch_article_contents(selected_article):521        with st.spinner("Fetching summaries, ranking them and highlighting entities, this might take a minute or two..."):522            summaries_list = []523            deduction_points = []524 525            # FOR NEW GENERATED SUMMARY526            for i in range(1 , 4):527                st.session_state.summary_output = fetch_ranked_summaries(selected_article, i)528                _, amount_unmatched = get_and_compare_entities(False)529 530                summary_deps = check_dependency(False)531                article_deps = check_dependency(True)532                total_unmatched_deps = []533                for summ_dep in summary_deps:534                    if not any(summ_dep['identifier'] in art_dep['identifier'] for art_dep in article_deps):535                        total_unmatched_deps.append(summ_dep)536 537                summaries_list.append(st.session_state.summary_output)538                deduction_points.append(len(amount_unmatched) + len(total_unmatched_deps))539 540 541            # RANKING AND SHOWING THE SUMMARIES542            deduction_points, summaries_list = (list(t) for t in zip(*sorted(zip(deduction_points, summaries_list))))543 544            cur_rank = 1545            rank_downgrade = 0546            for i in range(len(deduction_points)):547                #st.write(f'🏆 Rank {cur_rank} summary: 🏆', display_summary(summaries_list[i]), unsafe_allow_html=True)548                st.write(f'🏆 Rank {cur_rank} summary: 🏆', highlight_entities_new(summaries_list[i]), unsafe_allow_html=True)549                if i < len(deduction_points) - 1:550                    rank_downgrade += 1551                    if not deduction_points[i + 1] == deduction_points[i]:552                        cur_rank += rank_downgrade553                        rank_downgrade = 0554