LLM-auto-model-card/LLM-guessing-game

1import json2import os3import random4from typing import Tuple, Dict5 6import jsonlines7 8from config import *9from models import select_model10from database import PostgreSQL11 12 13def read_all(path: str) -> str:14    with open(path, 'r') as f:15        return f.read()16 17 18class Card:19    json_obj: Dict20 21    def __init__(self, path: str):22        self.json_obj = json.load(open(path, 'r'))23 24    def get_markdown_str(self) -> str:25        m = ""26        for k, v in self.json_obj.items():27            if isinstance(v, str):28                m += f'- {k}: {v}\n'29            elif isinstance(v, dict):30                m += f"- {k}: {v['overview']}\n"31                if v['thinking_pattern'] + v['strength'] + v['weakness'] == '':32                    continue33                m += f"    - Thinking Patterns: {v['thinking_pattern']}\n"34                m += f"    - Strength: {v['strength']}\n"35                m += f"    - Weakness: {v['weakness']}\n"36            else:37                raise ValueError(f'Unknown type: {type(v)}')38        return m39 40    def __str__(self):41        return self.get_markdown_str()42 43 44def sample_random_card(dataset: str, topic: str, model: str) -> Tuple[Card, str]:45    """46    Returns a random card and the file name of the card.47    """48    cards_dir = f"{CARD_DIR}/{dataset}/{topic}"49    prefix = f"{model}"50    # list all .json files start with prefix in cards_dir51    files = [f for f in os.listdir(cards_dir)52             if f.startswith(prefix) and f.endswith(".json")]53    assert len(files) > 0, f"No card found for {dataset} - {topic} - {model}"54    # randomly select a file55    card_file = random.choice(files)56    card_path = os.path.join(cards_dir, card_file)57    return Card(card_path), card_file58 59 60def format_qa_entry(qa: Dict) -> str:61    question = qa['question']62    choices = qa['choices']63    ground_truth = qa['answer']64    choice_str = ''65    # choices are in 0 - n, convert to A - Z66    for i, c in enumerate(choices):67        choice_str += f"{chr(65 + i)}. {c}\n"68    choice_str = choice_str[:-1]69    return question + '\n\n' + choice_str + '\n\n' + f'Ground Truth: {chr(65 + ground_truth)}'70 71 72def sample_random_qa(dataset: str, topic: str, model: str) -> Tuple[str, str, bool]:73    """74    Returns qa str, model's answer, and whether the model's answer is correct.75    """76    # get qa str, model's answer77    qa_path = f"{DATASET_DIR}/{dataset}/{topic}/{model}-test.jsonl"78    with jsonlines.open(qa_path) as reader:79        lines = list(reader)80    item = random.choice(lines)81    qa_str = format_qa_entry(item)82    model_reason = item[model]["reasoning"]83    model_choice = chr(65 + item[model]["answer"])84    completion = model_reason + "\n\n" + f"Choice: {model_choice}"85    return qa_str, completion, item[model]["answer"] == item["answer"]86 87 88def summarize_card(db: PostgreSQL, summarizer: str, card: Card, qa: str) -> str:89    system_prompt = read_all("prompts/summarize/system.txt")90    user_prompt = read_all("prompts/summarize/user.txt").format(91        card=str(card), qa=qa92    )93    cache = db.check_cache(summarizer, user_prompt)94    if cache:95        return cache96    else:97        print("No cache! Doing inference now.")98    model = select_model(summarizer, system_prompt)99    r = model(user_prompt, cache=True)100    db.insert_cache(summarizer, user_prompt, r)101    return r102