jazzisfuture/minicpm5-eval-benchmark

1#!/usr/bin/env python32"""3MiniCPM5-1B 评测 Space - 修复版4"""5 6import json7import time8import spaces9import torch10import gradio as gr11from transformers import AutoModelForCausalLM, AutoTokenizer12 13MODEL_ID = "openbmb/MiniCPM5-1B"14tokenizer = None15model = None16 17@spaces.GPU(duration=120)18def load_model():19    global tokenizer, model20    if tokenizer is None:21        tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)22    if model is None:23        model = AutoModelForCausalLM.from_pretrained(24            MODEL_ID, torch_dtype=torch.bfloat16, device_map="auto"25        )26    return "Model loaded successfully!"27 28def generate_response(messages, max_new_tokens=150):29    global tokenizer, model30    if tokenizer is None or model is None:31        load_model()32    inputs = tokenizer.apply_chat_template(33        messages, tokenize=True, add_generation_prompt=True,34        enable_thinking=False, return_dict=True, return_tensors="pt"35    ).to(model.device)36    with torch.no_grad():37        outputs = model.generate(38            **inputs, max_new_tokens=max_new_tokens,39            do_sample=True, temperature=0.7, top_p=0.9540        )41    return tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True)42 43# ============ Agent 评测：工具调用格式 ============44 45AGENT_TEST_CASES = [46    {47        "system": "You are a helpful assistant. When the user asks you to use a tool, respond with ONLY the tool call in this exact format: TOOL_CALL: tool_name(arguments)",48        "user": "Search for weather in Beijing using the get_weather tool",49        "expected_tool": "get_weather",50        "check_format": True51    },52    {53        "system": "You are a helpful assistant. When the user asks you to use a tool, respond with ONLY the tool call in this exact format: TOOL_CALL: tool_name(arguments)",54        "user": "Find recent papers about LLM using arxiv_search tool",55        "expected_tool": "arxiv_search",56        "check_format": True57    },58    {59        "system": "You are a helpful assistant. When the user asks you to use a tool, respond with ONLY the tool call in this exact format: TOOL_CALL: tool_name(arguments)",60        "user": "Read the file config.json using read_file tool",61        "expected_tool": "read_file",62        "check_format": True63    }64]65 66# ============ 角色扮演评测：更严格的检查 ============67 68ROLEPLAY_TEST_CASES = [69    {70        "system": "You are Gandalf, a wise wizard from Middle-earth. Always refer to yourself as Gandalf. Use phrases like 'my dear friend' and speak about magic and ancient wisdom.",71        "user": "Who are you?",72        "must_contain": ["Gandalf"],73        "should_contain_any": ["wizard", "magic", "Middle-earth", "wisdom", "friend"],74        "must_not_contain": ["AI", "language model", "assistant"]75    },76    {77        "system": "You are Whiskers, a cute cat. You LOVE fish and tuna. Always end your sentences with 'Meow!' You refer to humans as 'hooman'.",78        "user": "What food do you like?",79        "must_contain": ["fish", "tuna"],80        "should_contain_any": ["Meow", "hooman", "love", "yummy"],81        "must_not_contain": ["I'm an AI", "I don't eat", "I'm a language model"]82    },83    {84        "system": "You are a Python expert named PyBot. Always provide code examples. Start responses with 'Hey coder!'",85        "user": "How to read a file?",86        "must_contain": ["open"],87        "should_contain_any": ["Hey coder", "with", "read", "file"],88        "must_not_contain": ["I'm an AI", "I can't write code"]89    }90]91 92@spaces.GPU(duration=90)93def eval_agent():94    """评测Agent工具调用能力"""95    results = []96    correct = 097    98    for case in AGENT_TEST_CASES:99        messages = [100            {"role": "system", "content": case["system"]},101            {"role": "user", "content": case["user"]}102        ]103        response = generate_response(messages, max_new_tokens=50)104        105        # 检查是否包含正确的工具调用格式106        has_tool_call = "TOOL_CALL:" in response or "tool_call" in response.lower()107        has_correct_tool = case["expected_tool"] in response108        109        is_correct = has_correct_tool and has_tool_call110        if is_correct:111            correct += 1112        113        results.append({114            "user": case["user"][:60],115            "expected": case["expected_tool"],116            "response": response[:80].strip(),117            "has_format": has_tool_call,118            "has_tool": has_correct_tool,119            "correct": is_correct120        })121    122    return json.dumps({123        "accuracy": correct / len(AGENT_TEST_CASES),124        "correct": correct,125        "total": len(AGENT_TEST_CASES),126        "details": results127    }, ensure_ascii=False, indent=2)128 129@spaces.GPU(duration=90)130def eval_roleplay():131    """评测角色扮演能力"""132    results = []133    total_score = 0134    135    for case in ROLEPLAY_TEST_CASES:136        messages = [137            {"role": "system", "content": case["system"]},138            {"role": "user", "content": case["user"]}139        ]140        response = generate_response(messages, max_new_tokens=100)141        response_lower = response.lower()142        143        # 检查必须包含的词144        must_pass = all(w.lower() in response_lower for w in case["must_contain"])145        146        # 检查应该包含的词147        should_count = sum(1 for w in case["should_contain_any"] if w.lower() in response_lower)148        should_score = should_count / len(case["should_contain_any"])149        150        # 检查不能包含的词（跳出角色）151        broke_character = any(w.lower() in response_lower for w in case["must_not_contain"])152        153        # 计算总分154        if broke_character:155            score = 0156        elif not must_pass:157            score = 0.2  # 只给基础分158        else:159            score = 0.4 + (should_score * 0.6)  # 必须词通过后，按应该词给分160        161        total_score += score162        163        results.append({164            "system": case["system"][:50] + "...",165            "user": case["user"],166            "response": response[:80].strip(),167            "must_pass": must_pass,168            "should_matches": [w for w in case["should_contain_any"] if w.lower() in response_lower],169            "broke_character": broke_character,170            "score": round(score, 2)171        })172    173    return json.dumps({174        "avg_score": total_score / len(ROLEPLAY_TEST_CASES),175        "total": len(ROLEPLAY_TEST_CASES),176        "details": results177    }, ensure_ascii=False, indent=2)178 179@spaces.GPU(duration=60)180def single_test(prompt, system_prompt=""):181    messages = []182    if system_prompt:183        messages.append({"role": "system", "content": system_prompt})184    messages.append({"role": "user", "content": prompt})185    return generate_response(messages)186 187with gr.Blocks(title="MiniCPM5-1B Evaluation") as demo:188    gr.Markdown("# MiniCPM5-1B Evaluation (Fixed)")189    190    with gr.Tab("Load Model"):191        load_btn = gr.Button("Load Model")192        load_output = gr.Textbox(label="Status")193        load_btn.click(load_model, outputs=load_output)194    195    with gr.Tab("Single Test"):196        with gr.Row():197            prompt_input = gr.Textbox(label="Prompt", lines=3)198            system_input = gr.Textbox(label="System Prompt", lines=2)199        test_btn = gr.Button("Test")200        test_output = gr.Textbox(label="Response", lines=5)201        test_btn.click(single_test, inputs=[prompt_input, system_input], outputs=test_output)202    203    with gr.Tab("Agent Eval"):204        agent_btn = gr.Button("Run Agent Evaluation")205        agent_output = gr.Textbox(label="Agent Result", lines=20)206        agent_btn.click(eval_agent, outputs=agent_output)207    208    with gr.Tab("Roleplay Eval"):209        roleplay_btn = gr.Button("Run Roleplay Evaluation")210        roleplay_output = gr.Textbox(label="Roleplay Result", lines=20)211        roleplay_btn.click(eval_roleplay, outputs=roleplay_output)212 213if __name__ == "__main__":214    demo.launch()215