radna/eval_llm
0
1 2def main(args):3 MODEL_NAME = args.model4 5 EVAL_FILE = args.file6 print(f"Using evaluation file: {EVAL_FILE}")7 8 # copy file from ../data/aime/{EVAL_FILE}.csv to reference.csv9 import shutil10 import os11 import time12 13 run_start_time = time.time()14 15 os.makedirs("tmp", exist_ok=True)16 os.makedirs("evals_res", exist_ok=True)17 18 # get base path for eval_file19 EVAL_FILE_BASENAME = os.path.basename(EVAL_FILE)20 MODEL_NAME_STR = "+".join(args.model.split("/"))21 SAVED_EVAL_FILE = f"{str(run_start_time)}_{MODEL_NAME_STR}_{EVAL_FILE_BASENAME}_seq{args.num_seqs}_tok{args.tokens}_q{args.quant_policy}_tpp{args.top_p}_mnp{args.min_p}_tpk{args.top_k}"22 23 import os24 25 os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3"26 os.environ["TOKENIZERS_PARALLELISM"] = "false"27 os.environ["TRITON_PTXAS_PATH"] = "/usr/local/cuda/bin/ptxas"28 import re29 import random30 import warnings31 from collections import Counter32 import numpy as np, pandas as pd, polars as pl33 34 import torch35 import lmdeploy36 from lmdeploy import pipeline, TurbomindEngineConfig, GenerationConfig37 from transformers import AutoTokenizer38 39 warnings.simplefilter("ignore")40 print("PyTorch version:", torch.__version__)41 print("LMDeploy:", lmdeploy.__version__)42 43 def seed_everything(seed):44 os.environ["PYTHONHASHSEED"] = str(seed)45 random.seed(seed)46 np.random.seed(seed)47 torch.manual_seed(seed)48 torch.cuda.manual_seed(seed)49 torch.backends.cudnn.benchmark = True50 torch.backends.cudnn.deterministic = True51 52 seed_everything(seed=0)53 54 # cutoff_time = start_time + (1 * 60 + 50) * 6055 # cutoff_times = [56 # int(x) for x in np.linspace(cutoff_time, start_time + 10 * 60, 50 + 1)57 # ]58 59 llm_model_pth = MODEL_NAME60 61 MAX_NUM_SEQS = args.num_seqs62 MAX_MODEL_LEN = 1024 * 1263 EVAL = True64 EVAL_SELECTED_QUESTIONS_ONLY = False65 66 engine_config = TurbomindEngineConfig(67 # tp=1,68 quant_policy=args.quant_policy,69 cache_max_entry_count=0.95,70 session_len=MAX_MODEL_LEN,71 enable_prefix_caching=True,72 max_batch_size=MAX_NUM_SEQS,73 )74 75 pipe = pipeline(llm_model_pth, backend_config=engine_config)76 77 tokenizer = AutoTokenizer.from_pretrained(llm_model_pth, trust_remote_code=False)78 79 import re80 81 def extract_boxed_text(text):82 pattern = r"oxed{(.*?)}"83 matches = re.findall(pattern, text)84 if not matches:85 return ""86 for match in matches[::-1]:87 if match != "":88 return match89 return ""90 91 def batch_message_filter(list_of_messages) -> tuple[list[list[dict]], list[str]]:92 extracted_answers = []93 list_of_messages_to_keep = []94 for messages in list_of_messages:95 answer = extract_boxed_text(messages[-1]["content"])96 if answer:97 extracted_answers.append(answer)98 else:99 list_of_messages_to_keep.append(messages)100 return list_of_messages_to_keep, extracted_answers101 102 def select_answer(answers):103 counter = Counter()104 for answer in answers:105 try:106 if int(answer) == float(answer):107 counter[int(answer)] += 1 + random.random() / 1_000108 except:109 pass110 if not counter:111 return 210112 _, answer = sorted([(v, k) for k, v in counter.items()], reverse=True)[0]113 return answer % 1000114 115 def batch_message_generate(list_of_messages) -> list[list[dict]]:116 max_tokens = args.tokens117 # if time.time() > cutoff_times[-1]:118 # print("Speedrun")119 # max_tokens = 1024 * 8120 121 list_of_texts = [122 tokenizer.apply_chat_template(123 conversation=messages, tokenize=False, add_generation_prompt=True124 )125 for messages in list_of_messages126 ]127 128 gen_configs = [129 GenerationConfig(130 do_sample=True,131 temperature=1.0, # Randomness of the sampling132 top_k=args.top_k,133 top_p=args.top_p, # Cumulative probability of the top tokens to consider134 min_p=args.min_p, # Minimum probability for a token to be considered135 skip_special_tokens=True, # Whether to skip special tokens in the output136 max_new_tokens=max_tokens, # Maximum number of tokens to generate137 stop_words=["</think>"], # List of strings that stop the generation138 )139 for prompt in list_of_texts140 ]141 142 request_output = pipe(143 list_of_texts,144 gen_config=gen_configs,145 )146 print(147 [148 single_request_output.generate_token_len149 for single_request_output in request_output150 ]151 )152 153 sort_keys_and_list_of_messages = []154 for messages, single_request_output in zip(list_of_messages, request_output):155 # print()156 # print(single_request_output.outputs[0].text)157 # print()158 messages.append(159 {"role": "assistant", "content": single_request_output.text}160 )161 162 sort_keys_and_list_of_messages.append(163 (single_request_output.generate_token_len, messages)164 )165 print([sort_key for sort_key, _ in sort_keys_and_list_of_messages])166 sort_keys_and_list_of_messages.sort(167 key=lambda sort_key_and_messages: sort_key_and_messages[0]168 )169 print([sort_key for sort_key, _ in sort_keys_and_list_of_messages])170 171 list_of_messages = [messages for _, messages in sort_keys_and_list_of_messages]172 return list_of_messages173 174 def create_starter_messages(question: str, index: int) -> str:175 options = []176 for _ in range(1):177 options.append(178 [179 {180 "role": "system",181 "content": "You are a helpful and harmless assistant. You are Qwen developed by Alibaba. You should think step-by-step. Return final answer within \\boxed{}, after taking modulo 1000.",182 },183 {"role": "user", "content": question},184 ]185 )186 187 return options[index % len(options)]188 189 def predict_for_question(question: str, question_id=time.time()) -> int:190 import os191 import time192 193 start_time = time.time()194 195 if EVAL_SELECTED_QUESTIONS_ONLY and not os.getenv(196 "KAGGLE_IS_COMPETITION_RERUN"197 ):198 # if "Triangle" not in question:199 # return 210200 if (201 "Triangle" not in question202 and "delightful" not in question203 and "George" not in question204 ):205 return 210206 207 """ if time.time() > cutoff_time:208 return 210 """209 210 print(question)211 212 num_seqs = MAX_NUM_SEQS213 214 list_of_messages = [215 create_starter_messages(question, index) for index in range(num_seqs)216 ]217 218 all_extracted_answers = []219 for _ in range(1):220 list_of_messages = batch_message_generate(list_of_messages)221 222 if not os.getenv("KAGGLE_IS_COMPETITION_RERUN"):223 df = pd.DataFrame(224 {225 "question": [question] * len(list_of_messages),226 "message": [227 messages[-1]["content"] for messages in list_of_messages228 ],229 }230 )231 df.to_csv(f"tmp/{str(question_id)}_{SAVED_EVAL_FILE}.csv", index=False)232 233 list_of_messages, extracted_answers = batch_message_filter(list_of_messages)234 all_extracted_answers.extend(extracted_answers)235 236 print(all_extracted_answers)237 answer = select_answer(all_extracted_answers)238 print(answer)239 240 print("\n\n")241 # cutoff_times.pop()242 print(f"Time taken: {time.time() - start_time}")243 return answer244 245 # Replace this function with your inference code.246 # The function should return a single integer between 0 and 999, inclusive.247 # Each prediction (except the very first) must be returned within 30 minutes of the question being provided.248 249 # Path to the temporary CSV file250 import uuid251 252 TEMP_CSV = f"tmp/evals_{SAVED_EVAL_FILE}.csv"253 254 def predict(255 id_: pl.DataFrame, question: pl.DataFrame256 ) -> pl.DataFrame | pd.DataFrame:257 id_ = id_["id"][0]258 print("------")259 print(id_)260 261 question = question["problem"][0]262 answer = predict_for_question(question, question_id=id_)263 print("------\n\n\n")264 265 if EVAL and not os.getenv("KAGGLE_IS_COMPETITION_RERUN"):266 # Prepare a row to log (you can add more columns if needed)267 row = {"id": id_, "question": question, "answer": answer}268 269 # Create a temporary DataFrame for this single prediction270 temp_df = pd.DataFrame([row])271 272 # If the CSV file doesn't exist, write with headers;273 # otherwise, append without writing the header.274 if not os.path.exists(TEMP_CSV):275 temp_df.to_csv(TEMP_CSV, index=False)276 else:277 temp_df.to_csv(TEMP_CSV, mode="a", header=False, index=False)278 279 return pl.DataFrame({"id": id_, "answer": answer})280 281 """ predict_for_question(282 "Fred and George take part in a tennis tournament with $4046$ other players. In each round, the players are paired into $2024$ matches. How many ways are there to arrange the first round such that Fred and George do not have to play each other? (Two arrangements for the first round are \\textit{different} if there is a player with a different opponent in the two arrangements.)"283 )284 predict_for_question(285 "Triangle $ABC$ has side length $AB = 120$ and circumradius $R = 100$. Let $D$ be the foot of the perpendicular from $C$ to the line $AB$. What is the greatest possible length of segment $CD$?"286 )287 288 return """289 290 def sample_and_predict(csv_file: str) -> None:291 """292 Reads all rows from the given CSV file, and for each row,293 calls the predict() function to process the problem.294 """295 # Attempt to read the CSV file.296 df = pd.read_csv(csv_file)297 298 # randomly shuffle the rows299 df = df.sample(frac=1, random_state=2024).reset_index(drop=True)300 301 # Loop through every row in the DataFrame.302 for index, row in df.iterrows():303 id_value = row["id"]304 problem_value = row["problem"]305 306 print(f"Processing row {index}: id = {id_value}, problem = {problem_value}")307 308 # Convert the values to single-row polars DataFrames.309 id_df = pl.DataFrame({"id": [id_value]})310 problem_df = pl.DataFrame({"problem": [problem_value]})311 312 # Call the predict function.313 result = predict(id_df, problem_df)314 print("Prediction result:")315 print(result)316 print("\n")317 # Optionally add a small delay if needed.318 # time.sleep(1)319 320 sample_and_predict(EVAL_FILE)321 322 # if EVAL and not EVAL_SELECTED_QUESTIONS_ONLY and not os.getenv('KAGGLE_IS_COMPETITION_RERUN'):323 if (324 EVAL325 and not EVAL_SELECTED_QUESTIONS_ONLY326 and not os.getenv("KAGGLE_IS_COMPETITION_RERUN")327 ):328 import pandas as pd329 330 # File paths (adjust if needed)331 reference_input_path = EVAL_FILE332 predictions_path = TEMP_CSV333 334 # Load the CSV files335 reference_df = pd.read_csv(reference_input_path)336 predictions_df = pd.read_csv(predictions_path)337 338 # Ensure the 'id' columns are strings and strip any extra whitespace339 reference_df["id"] = reference_df["id"].astype(str).str.strip()340 predictions_df["id"] = predictions_df["id"].astype(str).str.strip()341 342 # Optionally, normalize the answer columns (e.g., lowercasing and stripping whitespace)343 reference_df["answer"] = (344 reference_df["answer"].astype(str).str.strip().str.lower()345 )346 predictions_df["answer"] = (347 predictions_df["answer"].astype(str).str.strip().str.lower()348 )349 350 # Merge the predictions with the reference data on the common 'id' column.351 merged_df = pd.merge(352 reference_df,353 predictions_df,354 on="id",355 how="inner",356 suffixes=("_ref", "_pred"),357 )358 359 # Compare the answers. (Adjust this comparison if your answers require special handling.)360 merged_df["is_correct"] = merged_df["answer_ref"] == merged_df["answer_pred"]361 362 # Calculate metrics363 total = len(merged_df)364 correct = merged_df["is_correct"].sum()365 accuracy = correct / total366 367 std_outputs = ""368 std_outputs = std_outputs + f"Total predictions compared: {total}" + "\n"369 std_outputs = std_outputs + f"Number of correct predictions: {correct}" + "\n"370 std_outputs = std_outputs + f"Accuracy: {accuracy:.2%}" + "\n"371 372 # Optionally, list the rows where the prediction did not match the reference.373 incorrect_df = merged_df[~merged_df["is_correct"]]374 if not incorrect_df.empty:375 std_outputs = std_outputs + "\nIncorrect predictions:" + "\n"376 # Adjust the columns below if your CSVs have different column names.377 std_outputs = (378 std_outputs379 + str(incorrect_df[["id", "problem", "answer_ref", "answer_pred"]])380 + "\n"381 )382 else:383 std_outputs = std_outputs + "\nAll predictions match the reference!" + "\n"384 385 time_taken = time.time() - run_start_time386 std_outputs = std_outputs + f"Time taken: {time_taken:.2f} seconds" + "\n"387 print(std_outputs)388 389 # write stdoutputs to evals_res/outputs_{SAVED_EVAL_FILE}.log390 391 # write stdoutputs to evals_res/outputs_{SAVED_EVAL_FILE}.log392 with open(f"evals_res/outputs_{SAVED_EVAL_FILE}.log", "w") as f:393 f.write(std_outputs)394 395 # save the merged DataFrame to a new CSV file396 # randomize with uuid397 merged_df.to_csv(f"evals_res/evals_{SAVED_EVAL_FILE}.csv", index=False)398 399 400if __name__ == "__main__":401 import argparse402 import time403 404 start = time.time()405 406 parser = argparse.ArgumentParser()407 parser.add_argument(408 "--model",409 type=str,410 default="casperhansen/deepseek-r1-distill-qwen-7b-awq",411 help="Model to use",412 )413 parser.add_argument(414 "--file",415 type=str,416 default="hard_batch_1",417 help="Eval File to use",418 )419 parser.add_argument(420 "--num_seqs",421 type=int,422 default=48,423 help="Number of sequences to generate per prompt",424 )425 parser.add_argument(426 "--tokens",427 type=int,428 default=1024 * 12,429 help="Number of sequences to generate per prompt",430 )431 432 parser.add_argument(433 "--quant_policy",434 type=int,435 default=8,436 choices=[8, 4, 0],437 help="Number of sequences to generate per prompt",438 )439 440 parser.add_argument(441 "--top_k",442 type=int,443 default=50,444 help="Number of sequences to generate per prompt",445 )446 447 parser.add_argument(448 "--top_p",449 type=float,450 default=0.90,451 help="Number of sequences to generate per prompt",452 )453 454 parser.add_argument(455 "--min_p",456 type=float,457 default=0.05,458 help="Number of sequences to generate per prompt",459 )460 461 args = parser.parse_args()462 main(args)463 464 print(f"Time Taken: {time.time() - start}")465 466