Saturdays/chatbot_refugiados

1import gradio as gr2from haystack.nodes import FARMReader, PreProcessor, PDFToTextConverter, TfidfRetriever3from haystack.document_stores import InMemoryDocumentStore4from haystack.pipelines import ExtractiveQAPipeline5 6document_store = InMemoryDocumentStore()7model = "Saturdays/mdeberta-v3-base-squad2_refugees_dataset_finetuned"8reader = FARMReader(model_name_or_path=model)9preprocessor = PreProcessor(10    clean_empty_lines=True,11    clean_whitespace=True,12    clean_header_footer=True,13    split_by="word",14    split_length=100,15    split_respect_sentence_boundary=True,16    split_overlap=317)18 19 20def print_answers(results):21    fields = ["answer", "score"]  # "context",22    answers = results["answers"]23    filtered_answers = []24 25    for ans in answers:26        filtered_ans = {27            field: getattr(ans, field)28            for field in fields29            if getattr(ans, field) is not None30        }31        filtered_answers.append(filtered_ans)32 33    return filtered_answers34 35 36def pdf_to_document_store(pdf_file):37    document_store.delete_documents()38    converter = PDFToTextConverter(39        remove_numeric_tables=True, valid_languages=["es"])40    documents = [converter.convert(file_path=pdf_file, meta=None)[0]]41    preprocessed_docs = preprocessor.process(documents)42    document_store.write_documents(preprocessed_docs)43    return None44 45 46def predict(question):47    pdf_to_document_store("data.pdf")48    retriever = TfidfRetriever(document_store=document_store)49    pipe = ExtractiveQAPipeline(reader, retriever)50    result = pipe.run(query=question, params={"Retriever": {51                      "top_k": 5}, "Reader": {"top_k": 3}})52    answers = print_answers(result)53    return answers54 55def respond(message, chat_history):56    if len(message)==0:57            message="¿Dónde puedo solicitar asilo?"58    bot_message = predict(message)[0]['answer']59    chat_history.append((message, bot_message))60    return "", chat_history61 62description= "Our chatbot helps refugees arriving in Spain by providing information on key topics. \n This project is based on the article titled [Desarrollando un chatbot para refugiados: nuestra experiencia en Saturdays.AI](https://medium.com/saturdays-ai/desarrollando-un-chatbot-para-refugiados-nuestra-experiencia-en-saturdays-ai-9bf2551432c9), which outlines the process of building a chatbot for refugees. \n You can find the training script in this [github repo](https://github.com/jsr90/chatbot_refugiados_train)."63 64with gr.Blocks(theme="huggingface") as demo:65    gr.HTML("<h1 style='text-align: center; font-size: xx-large'>Chatbot Refugiados (spanish)</h1>")66    gr.HTML("<h2 style='text-align: center; font-size: large'>The demo you're about to see is from a project currently in development.</h2>")67    68    with gr.Row():69        with gr.Column(scale=2):70            chatbot = gr.Chatbot()71        with gr.Column(scale=1):72            with gr.Row():73                msg = gr.Textbox(label="Write your question:", value="¿Dónde puedo solicitar asilo?")74            with gr.Row():75                submit = gr.Button("Submit")76                clear = gr.Button("Clear")77            gr.Image("OIG.jpeg")78 79    msg.submit(respond, [msg, chatbot], [msg, chatbot])80    submit.click(respond, [msg, chatbot], [msg, chatbot])81    clear.click(lambda: None, None, chatbot, queue=False)82 83    gr.Markdown(description)84 85if __name__ == "__main__":86    demo.launch()