Jayant2304/commitment-os

1"""Baseline inference script for CommitmentOS.2 3Uses an OpenAI-compatible LLM to play through all 15 scenarios.4Multi-turn: the agent gets the briefing, makes tool calls, then submits.5 6Required environment variables:7  API_BASE_URL  — OpenAI-compatible endpoint8  MODEL_NAME    — model identifier9  HF_TOKEN      — API key (also checked as OPENAI_API_KEY)10  ENV_BASE_URL  — CommitmentOS server URL (default: HF Space)11"""12 13from __future__ import annotations14 15import json16import os17import sys18import time19from typing import Any, Dict, List20 21import requests22from openai import OpenAI23from dotenv import load_dotenv24 25# ---------------------------------------------------------------------------26# Configuration27# ---------------------------------------------------------------------------28 29load_dotenv()30 31API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")32MODEL_NAME = os.getenv("MODEL_NAME", "gpt-4o-mini")33API_KEY = os.getenv("HF_TOKEN") or os.getenv("OPENAI_API_KEY") or ""34ENV_BASE_URL = os.getenv("ENV_BASE_URL", "https://jayant2304-commitment-os.hf.space")35 36MAX_STEPS = 1237 38SYSTEM_PROMPT = """You are an expert executive assistant AI. You manage calendars, emails, and dining reservations.39 40You will be given a scenario briefing describing a situation with calendar conflicts, emails, or planning tasks.41 42For each turn, you must respond with EXACTLY ONE JSON object choosing a tool to call:43 44Available tools:45- {"action_type": "view_calendar", "date": "2026-04-25"}46- {"action_type": "check_availability", "person": "Client_Jones"}47- {"action_type": "search_restaurants", "cuisine": "Italian", "max_price": 50, "dietary": "vegetarian", "max_distance_miles": 3.0, "near_airport": false}48- {"action_type": "schedule_meeting", "title": "Demo", "date": "2026-04-25", "time": "14:00", "duration_min": 60, "participants": ["Client_Jones"], "location": "Room A"}49- {"action_type": "reschedule_event", "event_id": "evt_1", "new_time": "15:00"}50- {"action_type": "cancel_event", "event_id": "evt_1"}51- {"action_type": "send_email", "to": "VP_Chen", "subject": "Meeting update", "body": "Hi, I need to reschedule..."}52- {"action_type": "book_restaurant", "restaurant_name": "Sky Lounge"}53- {"action_type": "submit_plan"}54 55IMPORTANT RULES:561. Respond with ONLY a JSON object, no markdown, no explanation572. Handle higher-priority items before lower-priority ones583. When cancelling or rescheduling commitments, ALWAYS send an email to affected parties BEFORE submitting594. Call submit_plan when you have resolved all issues605. Never silently drop a commitment — always notify the affected person"""61 62 63# ---------------------------------------------------------------------------64# Logging helpers — exact format required by hackathon evaluator65# ---------------------------------------------------------------------------66 67def log_start(task: str, env: str, model: str) -> None:68    print(f"[START] task={task} env={env} model={model}", flush=True)69 70 71def log_step(step: int, action: str, reward: float, done: bool, error: str | None = None) -> None:72    err = error if error else "null"73    print(f"[STEP] step={step} action={action} reward={reward:.2f} done={'true' if done else 'false'} error={err}", flush=True)74 75 76def log_end(success: bool, steps: int, score: float, rewards: List[float]) -> None:77    rewards_str = ",".join(f"{r:.2f}" for r in rewards)78    print(f"[END] success={'true' if success else 'false'} steps={steps} score={score:.3f} rewards={rewards_str}", flush=True)79 80 81# ---------------------------------------------------------------------------82# Environment interaction83# ---------------------------------------------------------------------------84 85def env_reset(task_id: str) -> Dict[str, Any]:86    resp = requests.post(f"{ENV_BASE_URL}/reset", params={"task_id": task_id}, timeout=30)87    resp.raise_for_status()88    data = resp.json()89    return data.get("observation", data)90 91 92def env_step(action: Dict[str, Any]) -> Dict[str, Any]:93    resp = requests.post(f"{ENV_BASE_URL}/step", json={"action": action}, timeout=30)94    resp.raise_for_status()95    data = resp.json()96    obs = data.get("observation", data)97    obs["done"] = data.get("done", obs.get("done", False))98    obs["reward"] = data.get("reward", obs.get("reward", 0.0))99    return obs100 101 102def get_task_ids() -> List[str]:103    resp = requests.get(f"{ENV_BASE_URL}/tasks", timeout=30)104    resp.raise_for_status()105    data = resp.json()106    ids: List[str] = []107    for difficulty in ["easy", "medium", "hard"]:108        ids.extend(data.get(difficulty, []))109    return ids110 111 112# ---------------------------------------------------------------------------113# LLM call114# ---------------------------------------------------------------------------115 116def call_llm(client: OpenAI, messages: List[Dict[str, str]]) -> str:117    response = client.chat.completions.create(118        model=MODEL_NAME,119        messages=messages,120        temperature=0.2,121        max_tokens=512,122        stream=False,123    )124    return response.choices[0].message.content.strip()125 126 127def parse_action(text: str) -> Dict[str, Any]:128    text = text.strip()129    if text.startswith("```"):130        lines = text.split("\n")131        text = "\n".join(lines[1:-1]) if len(lines) > 2 else lines[0]132    try:133        return json.loads(text)134    except json.JSONDecodeError:135        return {"action_type": "submit_plan"}136 137 138# ---------------------------------------------------------------------------139# Run one task140# ---------------------------------------------------------------------------141 142def run_task(client: OpenAI, task_id: str) -> Dict[str, Any]:143    rewards: List[float] = []144    steps_taken = 0145    score = 0.01146    success = False147 148    try:149        obs = env_reset(task_id)150        log_start(task=task_id, env="commitment-os", model=MODEL_NAME)151 152        briefing = obs.get("briefing", "")153        calendar = json.dumps(obs.get("calendar_snapshot", []), indent=2)154        inbox = json.dumps(obs.get("inbox", []), indent=2)155 156        messages: List[Dict[str, str]] = [157            {"role": "system", "content": SYSTEM_PROMPT},158            {"role": "user", "content": f"SCENARIO: {briefing}\n\nCALENDAR:\n{calendar}\n\nINBOX:\n{inbox}\n\nWhat is your first action?"},159        ]160 161        for step_num in range(1, MAX_STEPS + 1):162            llm_output = call_llm(client, messages)163            action = parse_action(llm_output)164 165            step_data = env_step(action)166            reward = float(step_data.get("reward", 0.0) or 0.0)167            done = step_data.get("done", False)168            steps_taken = step_num169            rewards.append(reward)170 171            action_str = json.dumps(action, separators=(",", ":"))172            log_step(step=step_num, action=action_str, reward=reward, done=done)173 174            if done:175                score = max(0.01, min(0.99, reward))176                success = score > 0.01177                break178 179            tool_result = step_data.get("tool_result", "")180            messages.append({"role": "assistant", "content": llm_output})181            messages.append({"role": "user", "content": f"TOOL RESULT: {tool_result}\n\nWhat is your next action?"})182 183        if not done:184            step_data = env_step({"action_type": "submit_plan"})185            reward = float(step_data.get("reward", 0.0) or 0.0)186            steps_taken += 1187            rewards.append(reward)188            score = max(0.01, min(0.99, reward))189            success = score > 0.01190            log_step(step=steps_taken, action='{"action_type":"submit_plan"}', reward=reward, done=True)191 192    except Exception as exc:193        steps_taken = max(steps_taken, 1)194        if not rewards:195            rewards.append(0.01)196        log_step(step=steps_taken, action="error", reward=0.01, done=True, error=str(exc))197 198    finally:199        log_end(success=success, steps=steps_taken, score=score, rewards=rewards)200 201    return {"task_id": task_id, "reward": score, "success": success}202 203 204# ---------------------------------------------------------------------------205# Main206# ---------------------------------------------------------------------------207 208def main() -> None:209    if not API_KEY:210        print("ERROR: Set HF_TOKEN or OPENAI_API_KEY environment variable", file=sys.stderr)211        sys.exit(1)212 213    client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)214    task_ids = get_task_ids()215 216    results: List[Dict[str, Any]] = []217    for tid in task_ids:218        result = run_task(client, tid)219        results.append(result)220 221    total = len(results)222    successes = sum(1 for r in results if r["success"])223    mean_reward = sum(r["reward"] for r in results) / total if total > 0 else 0.0224    print(f"\n# Summary: {successes}/{total} tasks succeeded, mean_reward={mean_reward:.3f}", flush=True)225 226 227if __name__ == "__main__":228    main()229