AMD21/codefixerenv

1"""2inference.py — CodeFixerEnv baseline agent.3 4Environment variables injected by the hackathon validator:5  API_BASE_URL  — LiteLLM proxy URL (used for ALL LLM calls)6  API_KEY       — LiteLLM proxy key7  MODEL_NAME    — model identifier (has default)8  ENV_BASE_URL  — CodeFixerEnv server URL (has default)9"""10 11import os12import sys13import json14import requests15from openai import OpenAI16 17# ── Env vars ──────────────────────────────────────────────────────────────────18API_BASE_URL = os.environ["API_BASE_URL"]19API_KEY      = os.environ["API_KEY"]20MODEL_NAME   = os.getenv("MODEL_NAME", "gpt-4o-mini")21ENV_BASE_URL = os.getenv("ENV_BASE_URL", "http://localhost:7860")22 23DIFFICULTIES = ["easy", "medium", "hard"]24MAX_STEPS    = 525 26# ── OpenAI client — MUST use injected API_BASE_URL + API_KEY ─────────────────27client = OpenAI(28    base_url=API_BASE_URL,29    api_key=API_KEY,30)31 32SYSTEM_PROMPT = """You are an expert Python debugger.33You will receive a buggy Python function and a description of what it should do.34Fix the code and return ONLY a valid JSON object with no markdown, no explanation:35{36  "action_type": "fix",37  "action_content": "<full corrected Python function>"38}39action_type must be one of: fix, explain, give_up40action_content must be the complete corrected function, not a diff.41"""42 43 44def log(msg: str):45    print(msg, flush=True)46    sys.stdout.flush()47 48 49def strict(value: float) -> float:50    """Strictly (0.0, 1.0) — 0.0 → 0.01, 1.0 → 0.99."""51    return round(max(0.01, min(0.99, float(value))), 4)52 53 54def call_env(endpoint: str, method: str = "GET", payload: dict = None) -> dict:55    url = f"{ENV_BASE_URL.rstrip('/')}/{endpoint}"56    if method == "POST":57        resp = requests.post(url, json=payload, timeout=30)58    else:59        resp = requests.get(url, timeout=30)60    resp.raise_for_status()61    return resp.json()62 63 64def call_model(obs: dict) -> tuple[str, str]:65    history_str = ""66    if obs.get("history"):67        history_str = "\n\nPrevious attempts:\n" + "\n".join(68            f"  Step {h['step']}: {h['feedback']}" for h in obs["history"]69        )70 71    user_prompt = (72        f"Task: {obs.get('context', '')}\n\n"73        f"Buggy code:\n{obs.get('input', '')}"74        f"{history_str}\n\n"75        f"Step {obs.get('step_number', 0) + 1} of {obs.get('max_steps', 5)}. Fix the code."76    )77 78    try:79        completion = client.chat.completions.create(80            model=MODEL_NAME,81            messages=[82                {"role": "system", "content": SYSTEM_PROMPT},83                {"role": "user",   "content": user_prompt},84            ],85            temperature=0.2,86            max_tokens=800,87        )88        raw = completion.choices[0].message.content or ""89        raw = raw.replace("```json", "").replace("```", "").strip()90        parsed = json.loads(raw)91        return parsed.get("action_type", "give_up"), parsed.get("action_content", "")92    except Exception as exc:93        log(f"[MODEL ERROR] {exc}")94        return "give_up", ""95 96 97def run_episode(difficulty: str) -> dict:98    # Start at 0.01 — never 0.099    final_score = 0.01100    step_num    = 1101 102    log(f"[START] task={difficulty}")103 104    try:105        obs = call_env("reset", "POST", {"difficulty": difficulty})106    except Exception as e:107        log(f"[ERROR] reset failed: {e}")108        log(f"[STEP] step=1 reward=0.0100")109        log(f"[END] task={difficulty} score=0.0100 steps=1")110        return {"difficulty": difficulty, "final_grader_score": 0.01}111 112    for step_num in range(1, MAX_STEPS + 1):113        action_type, action_content = call_model(obs)114 115        try:116            result     = call_env("step", "POST", {117                "action_type":    action_type,118                "action_content": action_content,119            })120            reward_val = result["reward"]["value"]121            done       = result["done"]122            info       = result["info"]123            obs        = result["observation"]124 125            # Always clamp grader_score to strict (0, 1)126            if "grader_score" in info and info["grader_score"] is not None:127                final_score = strict(info["grader_score"])128 129            step_reward = strict(reward_val)130 131        except Exception as e:132            log(f"[ERROR] step failed: {e}")133            step_reward = 0.01134            done = True135 136        # [STEP] reward must also be strictly in (0, 1)137        log(f"[STEP] step={step_num} reward={step_reward:.4f}")138 139        if done:140            break141 142    # [END] score strictly in (0, 1) — guaranteed by strict()143    log(f"[END] task={difficulty} score={final_score:.4f} steps={step_num}")144 145    return {146        "difficulty":         difficulty,147        "final_grader_score": final_score,148    }149 150 151def main():152    log(f"[INFO] API_BASE_URL={API_BASE_URL}")153    log(f"[INFO] MODEL_NAME={MODEL_NAME}")154    log(f"[INFO] ENV_BASE_URL={ENV_BASE_URL}")155 156    try:157        h = call_env("health")158        log(f"[INFO] Env server={h.get('status', 'ok')}")159    except Exception as e:160        log(f"[WARN] Env server unreachable: {e}")161 162    results = []163    for difficulty in DIFFICULTIES:164        result = run_episode(difficulty)165        results.append(result)166 167    avg = strict(sum(r["final_grader_score"] for r in results) / len(results))168    log(f"[SUMMARY] average_score={avg:.4f}")169 170    with open("inference_results.json", "w") as f:171        json.dump({"results": results, "average_score": avg}, f, indent=2)172    log("[INFO] Saved inference_results.json")173 174 175if __name__ == "__main__":176    main()