serhany/question_generation

1import cohere2import numpy as np3from sentence_transformers import SentenceTransformer4from transformers import pipeline5from typing import List, Tuple6import os7import logging8import json9import gradio as gr10import pandas as pd11from datasets import load_dataset12import random13 14# Set up logging15logging.basicConfig(level=logging.INFO)16logger = logging.getLogger(__name__)17 18# Initialize Cohere client, SentenceTransformer model, and QA pipeline19co = cohere.Client(api_key=os.environ.get("COHERE_API_KEY"))20sentence_model = SentenceTransformer('all-MiniLM-L6-v2')21qa_pipeline = pipeline("question-answering", model="distilbert-base-cased-distilled-squad")22 23# Load the dataset24dataset = load_dataset("serhany/scaling-qa")25 26# Define sample inputs27samples = [28    {29        "context": "Albert Einstein is an Austrian scientist, who has completed his higher education in ETH Zurich in Zurich, Switzerland. He was later a faculty at Princeton University.",30        "answer": "Switzerland"31    },32    {33        "context": "The Eiffel Tower, located in Paris, France, is one of the most famous landmarks in the world. It was constructed in 1889 as the entrance arch to the 1889 World's Fair. The tower is 324 meters (1,063 ft) tall and is the tallest structure in Paris.",34        "answer": "Paris"35    },36    {37        "context": "The Great Wall of China is a series of fortifications and walls built across the historical northern borders of ancient Chinese states and Imperial China to protect against nomadic invasions. It is the largest man-made structure in the world, with a total length of more than 13,000 miles (21,000 kilometers).",38        "answer": "China"39    }40]41 42def generate_questions(context: str, answer: str) -> List[str]:43    try:44        response = co.chat(45            model="command-r",46            message=f"Based on this context: '{context}' and answer: '{answer}', generate 5 diverse questions which when asked to the context returns the answer.",47            response_format={48                "type": "json_object",49                "schema": {50                    "type": "object",51                    "required": ["question1", "question2", "question3", "question4", "question5"],52                    "properties": {53                        "question1": {"type": "string"},54                        "question2": {"type": "string"},55                        "question3": {"type": "string"},56                        "question4": {"type": "string"},57                        "question5": {"type": "string"}58                    }59                }60            }61        )62        63        json_response = response.text64        logger.info(f"Raw JSON response: {json_response}")65        66        parsed_response = json.loads(json_response)67        questions = [parsed_response[f"question{i}"] for i in range(1, 6)]68        return questions69    except Exception as e:70        logger.error(f"Error in generate_questions: {e}")71        return [f"Failed to generate question {i}" for i in range(1, 6)]72 73def calculate_structural_diversity(questions: List[str]) -> List[float]:74    def get_question_type(q):75        q = q.lower()76        if q.startswith('what'): return 177        elif q.startswith('why'): return 278        elif q.startswith('how'): return 379        elif q.startswith('when'): return 480        elif q.startswith('where'): return 581        else: return 082 83    lengths = [len(q.split()) for q in questions]84    types = [get_question_type(q) for q in questions]85    86    length_scores = [1 - (abs(l - np.mean(lengths)) / np.max(lengths)) for l in lengths]87    type_scores = [len(set(types)) / len(types) for _ in types]88    89    return [(l + t) / 2 for l, t in zip(length_scores, type_scores)]90 91def calculate_semantic_relevance(context: str, answer: str, questions: List[str]) -> List[float]:92    context_embedding = sentence_model.encode(context + " " + answer)93    question_embeddings = sentence_model.encode(questions)94    95    similarities = [np.dot(context_embedding, q_emb) / (np.linalg.norm(context_embedding) * np.linalg.norm(q_emb)) 96                    for q_emb in question_embeddings]97    98    return [(sim + 1) / 2 for sim in similarities]  # Normalize to 0-1 range99 100def check_answer_precision(context: str, questions: List[str], original_answer: str) -> Tuple[List[float], List[str]]:101    precision_scores = []102    generated_answers = []103    for question in questions:104        result = qa_pipeline(question=question, context=context)105        generated_answer = result['answer']106        generated_answers.append(generated_answer)107        answer_embedding = sentence_model.encode(original_answer)108        generated_embedding = sentence_model.encode(generated_answer)109        similarity = np.dot(answer_embedding, generated_embedding) / (np.linalg.norm(answer_embedding) * np.linalg.norm(generated_embedding))110        precision_scores.append((similarity + 1) / 2)  # Normalize to 0-1 range111    return precision_scores, generated_answers112 113def calculate_composite_scores(sd_scores: List[float], sr_scores: List[float], ap_scores: List[float]) -> List[float]:114    # Normalize other scores based on answer precision115    max_other_score = max(max(sd_scores), max(sr_scores))116    normalized_sd_scores = [sd * (ap / max_other_score) for sd, ap in zip(sd_scores, ap_scores)]117    normalized_sr_scores = [sr * (ap / max_other_score) for sr, ap in zip(sr_scores, ap_scores)]118    119    # Calculate composite scores with higher weight for answer precision120    return [0.6 * ap + 0.2 * sd + 0.2 * sr for ap, sd, sr in zip(ap_scores, normalized_sd_scores, normalized_sr_scores)]121 122def rank_questions_with_details(context: str, answer: str) -> Tuple[pd.DataFrame, List[pd.DataFrame], str]:123    questions = generate_questions(context, answer)124    125    sd_scores = calculate_structural_diversity(questions)126    sr_scores = calculate_semantic_relevance(context, answer, questions)127    ap_scores, generated_answers = check_answer_precision(context, questions, answer)128    129    composite_scores = calculate_composite_scores(sd_scores, sr_scores, ap_scores)130    131    # Create detailed scores dataframe132    detailed_scores = pd.DataFrame({133        'Question': questions,134        'Answer Precision': ap_scores,135        'Composite Score': composite_scores,136        'Structural Diversity': sd_scores,137        'Semantic Relevance': sr_scores,138        'Generated Answer': generated_answers139    })140    detailed_scores = detailed_scores.sort_values('Answer Precision', ascending=False).reset_index(drop=True)141    142    # Create separate ranking dataframes for each metric143    metrics = ['Answer Precision', 'Composite Score', 'Structural Diversity', 'Semantic Relevance']144    rankings = []145    146    for metric in metrics:147        df = pd.DataFrame({148            'Rank': range(1, 6),149            'Question': [questions[i] for i in np.argsort(detailed_scores[metric])[::-1]],150            f'{metric}': sorted(detailed_scores[metric], reverse=True)151        })152        if metric == 'Answer Precision':153            df['Generated Answer'] = [generated_answers[i] for i in np.argsort(detailed_scores[metric])[::-1]]154        rankings.append(df)155    156    best_question = detailed_scores.iloc[0]['Question']157    158    return detailed_scores, rankings, best_question159 160def gradio_interface(context: str, answer: str) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame, str]:161    detailed_scores, rankings, best_question = rank_questions_with_details(context, answer)162    return (163        detailed_scores,164        rankings[0],  # Answer Precision Ranking165        rankings[1],  # Composite Score Ranking166        rankings[2],  # Structural Diversity Ranking167        rankings[3],  # Semantic Relevance Ranking168        f"Best Question: {best_question}"169    )170 171def use_sample(sample_index: int) -> Tuple[str, str]:172    return samples[sample_index]["context"], samples[sample_index]["answer"]173 174def get_random_entry():175    # Get a random entry from the dataset176    random_index = random.randint(0, len(dataset['train']) - 1)177    entry = dataset['train'][random_index]178    return entry['context'], entry['answer']179 180# Create Gradio interface with improved layout and sample buttons181with gr.Blocks(theme=gr.themes.Default()) as iface:182    gr.Markdown("# Question Generator and Ranker")183    gr.Markdown("Enter a context and an answer to generate and rank questions, use one of the sample inputs, or get a random entry from the dataset.")184    185    with gr.Row():186        with gr.Column(scale=1):187            context_input = gr.Textbox(lines=5, label="Context")188            answer_input = gr.Textbox(lines=2, label="Answer")189            submit_button = gr.Button("Generate Questions")190            191            with gr.Row():192                sample_buttons = [gr.Button(f"Sample {i+1}") for i in range(3)]193                random_button = gr.Button("Random Dataset Entry")194        195        with gr.Column(scale=2):196            best_question_output = gr.Textbox(label="Best Question")197            detailed_scores_output = gr.DataFrame(label="Detailed Scores")198    199    with gr.Row():200        with gr.Column():201            answer_precision_ranking_output = gr.DataFrame(label="Answer Precision Ranking")202        with gr.Column():203            composite_ranking_output = gr.DataFrame(label="Composite Score Ranking")204    205    with gr.Row():206        with gr.Column():207            structural_diversity_ranking_output = gr.DataFrame(label="Structural Diversity Ranking")208        with gr.Column():209            semantic_relevance_ranking_output = gr.DataFrame(label="Semantic Relevance Ranking")210 211    submit_button.click(212        fn=gradio_interface,213        inputs=[context_input, answer_input],214        outputs=[215            detailed_scores_output,216            answer_precision_ranking_output,217            composite_ranking_output,218            structural_diversity_ranking_output,219            semantic_relevance_ranking_output,220            best_question_output221        ]222    )223 224    # Set up sample button functionality225    for i, button in enumerate(sample_buttons):226        button.click(227            fn=lambda i=i: use_sample(i),228            outputs=[context_input, answer_input]229        )230 231    # Set up random button functionality232    random_button.click(233        fn=get_random_entry,234        outputs=[context_input, answer_input]235    )236 237# Launch the app238iface.launch()