pgurazada1/RAG-evaluation

1import os2import gradio as gr3 4from openai import AzureOpenAI5 6client = AzureOpenAI(7    api_key=os.environ["AZURE_OPENAI_KEY"],8    azure_endpoint=os.environ["AZURE_OPENAI_ENDPOINT"],9    api_version="2024-02-01"10)11 12model_name = "gpt-4o-mini"13 14groundedness_rater_system_message = """15You are tasked with rating AI generated answers to questions posed by users.16You will be presented a question, context used by the AI system to generate the answer and an AI generated answer to the question.17In the input, the question will begin with ###Question, the context will begin with ###Context while the AI generated answer will begin with ###Answer.18 19Evaluation criteria:20The task is to judge the extent to which the metric is followed by the answer.211 - The metric is not followed at all222 - The metric is followed only to a limited extent233 - The metric is followed to a good extent244 - The metric is followed mostly255 - The metric is followed completely26 27Metric:28The answer should be derived only from the information presented in the context29 30Instructions:311. First write down the steps that are needed to evaluate the answer as per the metric.322. Give a step-by-step explanation if the answer adheres to the metric considering the question and context as the input.333. Next, evaluate the extent to which the metric is followed.344. Use the previous information to rate the answer using the evaluaton criteria and assign a score.35"""36 37relevance_rater_system_message = """38You are tasked with rating AI generated answers to questions posed by users.39You will be presented a question, context used by the AI system to generate the answer and an AI generated answer to the question.40In the input, the question will begin with ###Question, the context will begin with ###Context while the AI generated answer will begin with ###Answer.41 42Evaluation criteria:43The task is to judge the extent to which the metric is followed by the answer.441 - The metric is not followed at all452 - The metric is followed only to a limited extent463 - The metric is followed to a good extent474 - The metric is followed mostly485 - The metric is followed completely49 50Metric:51Relevance measures how well the answer addresses the main aspects of the question, based on the context.52Consider whether all and only the important aspects are contained in the answer when evaluating relevance.53 54Instructions:551. First write down the steps that are needed to evaluate the context as per the metric.562. Give a step-by-step explanation if the context adheres to the metric considering the question as the input.573. Next, evaluate the extent to which the metric is followed.584. Use the previous information to rate the context using the evaluaton criteria and assign a score.59"""60 61user_message_template = """62###Question63{question}64 65###Context66{context}67 68###Answer69{answer}70"""71 72def predict(rag_question, rag_context, rag_answer):73 74    groundedness_prompt = [75        {'role':'system', 'content': groundedness_rater_system_message},76        {'role': 'user', 'content': user_message_template.format(77            question=rag_question,78            context=rag_context,79            answer=rag_answer80            )81        }82    ]83 84    relevance_prompt = [85        {'role':'system', 'content': relevance_rater_system_message},86        {'role': 'user', 'content': user_message_template.format(87            question=rag_question,88            context=rag_context,89            answer=rag_answer90         )91        }92    ]93 94    try:95        groundedness_response = client.chat.completions.create(96            model=model_name,97            messages=groundedness_prompt,98            temperature=099        )100 101        groundedness_prediction = groundedness_response.choices[0].message.content102 103        relevance_response = client.chat.completions.create(104            model=model_name,105            messages=relevance_prompt,106            temperature=0107        )108 109        relevance_prediction = relevance_response.choices[0].message.content110 111    except Exception as e:112        prediction = e113    114    return groundedness_prediction + '\n' + '---' + '\n' + relevance_prediction115 116rag_question = gr.Textbox(placeholder="Enter your query here", lines=6)117rag_context = gr.Textbox(placeholder="Enter the retrieved context here", lines=6)118rag_answer = gr.Textbox(placeholder="Enter the LLM response here", lines=6)119 120demo = gr.Interface(121    inputs=[rag_question, rag_context, rag_answer], fn=predict, outputs="text",122    title="Evaluate RAG output for groundedness and relevance",123    description="This web API presents an interface to evaluate RAG output for groundedness and relevance",124    examples=[[["What was the increase in annual revenue in 2022 compared to 2021?"], 125               ["Here are some documents that are relevant to the question mentioned below. In 2022, we recognized total revenues of $81.46 billion, respectively, representing an increase of $27.64 billion, compared to the prior year. We continue to ramp production, build new manufacturing capacity and expand our operations to enable increased deliveries and deployments of our products and further revenue growth."], 126               ["$27.64 billion."]]],127    cache_examples=False,128    concurrency_limit=16129)130 131demo.queue()132demo.launch(auth=("demouser", os.getenv('PASSWD')))