CoderLakshman/Final_Assignment_Template

1from settings import Settings2from typing import List3from models import Question, QuestionAnswerPair, Results4import requests5import random6import json7import logging8logger = logging.getLogger(__name__)9 10 11class Evaluator():12    def __init__(self, settings: Settings):13        self.settings = settings14 15    def get_questions(self) -> list[Question]:16        """17        Get the questions from the HuggingFace endpoint.18 19        Returns:20            list[Question]: A list of Question objects21        """22        url = str(self.settings.scoring_api_base_url) + "questions"23        try:24            response = requests.get(url, timeout=10)25            response.raise_for_status()26            questions = [Question(**question) for question in response.json()]27            with open("questions.json", "w") as f:28                json.dump([question.model_dump()29                           for question in questions], f, indent=4)30        except:31            # Read local file instead, dealing with rate limits, etc.32            with open("questions.json", "r") as f:33                questions = [Question(**question) for question in json.load(f)]34        return questions35 36    def get_one_question(self, task_id=None) -> Question:37        """38        Get a random, or requested question from the HuggingFace endpoint.39 40        Returns:41            Question: A Question object42        """43        if task_id:44            questions = self.get_questions()45            if task_id:46                for question in questions:47                    if question.task_id == task_id:48                        return question49        try:50            url = str(self.settings.scoring_api_base_url) + "random-question"51            response = requests.get(url, timeout=10)52            response.raise_for_status()53            question = Question(**response.json())54            return question55        except:56            # Read local file instead, dealing with rate limits, etc.57            questions = self.get_questions()58            return questions[random.randint(0, len(questions)-1)]59 60    def _read_answer_file(self, username) -> List[str]:61        """Read the question answer pairs from a user-specific answer file."""62        file_name = f"answers_{username}.json"63        with open(file_name, "r") as f:64            pairs = [QuestionAnswerPair(**pair) for pair in json.load(f)]65            formatted_data = [pair.get_answer() for pair in pairs]66        return formatted_data67 68    def submit_answers(self, username: str) -> str:69        """Submits saved answers to the scoring endpoint and returns the result."""70        try:71            answers_payload = self._read_answer_file(username)72        except FileNotFoundError:73            return "Click 'Get One Answer' or 'Get All Answers' to run agent before trying to submit."74        75        agent_code = f"https://huggingface.co/spaces/{self.settings.space_id}/tree/main"76        submission_data = {77            "username": self.settings.username,78            "agent_code": agent_code,79            "answers": answers_payload}80        submit_url = str(self.settings.scoring_api_base_url) + "submit"81        logger.info(f"Submitting {len(answers_payload)} answers to: {submit_url}")82        try:83            response = requests.post(84                submit_url, json=submission_data, timeout=60)85            response.raise_for_status()86            results = Results.model_validate(response.json())87            logger.info(88                f"Submission successful.\n"89                f"User: {results.username}.\n"90                f"Overall Score: {results.score}%.\n"91                f"Correct Count: {results.correct_count}.\n"92                f"Total Attempted: {results.total_attempted}.\n"93                f"Message: {results.message}.\n"94                f"Timestamp: {results.timestamp}.\n"95            )96            status_message = (97                f"Submission Successful!\n"98                f"User: {results.username}\n"99                f"Overall Score: {results.score}% "100                f"({results.correct_count}/{results.total_attempted} correct)\n"101                f"Message: {results.message}"102            )103            return status_message104        except requests.exceptions.HTTPError as e:105            error_detail = f"Server responded with status {e.response.status_code}."106            try:107                error_json = e.response.json()108                error_detail += f" Detail: {error_json.get('detail', e.response.text)}"109            except requests.exceptions.JSONDecodeError:110                error_detail += f" Response: {e.response.text[:500]}"111            status_message = f"Submission Failed: {error_detail}"112            logger.info(status_message)113            return status_message114        except requests.exceptions.Timeout:115            status_message = "Submission Failed: The request timed out."116            logger.info(status_message)117            return status_message118        except requests.exceptions.RequestException as e:119            status_message = f"Submission Failed: Network error - {e}"120            logger.info(status_message)121            return status_message122        except Exception as e:123            status_message = f"An unexpected error occurred during submission: {e}"124            logger.info(status_message)125            return status_message126