Jayant2304/commitment-os

1"""Comprehensive test suite for CommitmentOS.2 3Tests cover:4  - Grader (perfect/partial/zero for each component)5  - Environment lifecycle (reset/step/state/multi-turn)6  - Commitment ledger (creation, violation, renegotiation)7  - Task dataset integrity8  - API endpoints9  - Difficulty verification10"""11 12from __future__ import annotations13 14import sys15from pathlib import Path16 17sys.path.insert(0, str(Path(__file__).resolve().parent.parent))18 19import json20from typing import Any, Dict21 22import pytest23 24from models import CommitmentAction, CommitmentObservation, CommitmentState25from server.domain import CalendarEvent, ConstraintDef, ScenarioDef26from server.environment import CommitmentEnvironment27from server.graders import (28    _calendar_has_no_overlaps,29    _keyword_score,30    _score_commitment_coherence,31    _score_conflict_resolution,32    _score_step_efficiency,33    grade_scenario,34)35from server.tasks import get_all_scenarios, get_scenario, get_scenarios_by_difficulty36from server.world import WorldState, _time_to_min37 38 39# ===================================================================40# Fixtures41# ===================================================================42 43@pytest.fixture44def env() -> CommitmentEnvironment:45    return CommitmentEnvironment()46 47 48@pytest.fixture49def easy_env(env: CommitmentEnvironment) -> CommitmentEnvironment:50    env.reset(task_id="easy_001")51    return env52 53 54# ===================================================================55# 1. Task dataset integrity56# ===================================================================57 58class TestTaskDataset:59    def test_15_scenarios_loaded(self) -> None:60        scenarios = get_all_scenarios()61        assert len(scenarios) == 1562 63    def test_5_easy_5_medium_5_hard(self) -> None:64        for difficulty, count in [("easy", 5), ("medium", 5), ("hard", 5)]:65            tasks = get_scenarios_by_difficulty(difficulty)66            assert len(tasks) == count, f"Expected {count} {difficulty} tasks, got {len(tasks)}"67 68    def test_each_scenario_has_required_fields(self) -> None:69        for sid, scenario in get_all_scenarios().items():70            assert scenario.scenario_id == sid71            assert scenario.difficulty in ("easy", "medium", "hard")72            assert len(scenario.briefing) > 20, f"{sid}: briefing too short"73            assert scenario.optimal_steps >= 2, f"{sid}: optimal_steps too low"74            assert scenario.max_steps >= scenario.optimal_steps75            assert len(scenario.constraints) >= 1, f"{sid}: no constraints defined"76 77    def test_scenario_ids_unique(self) -> None:78        ids = list(get_all_scenarios().keys())79        assert len(ids) == len(set(ids))80 81    def test_get_scenario_returns_none_for_missing(self) -> None:82        assert get_scenario("nonexistent_999") is None83 84    def test_get_scenario_returns_correct(self) -> None:85        s = get_scenario("easy_001")86        assert s is not None87        assert s.difficulty == "easy"88 89 90# ===================================================================91# 2. Grader unit tests92# ===================================================================93 94class TestKeywordScore:95    def test_full_match(self) -> None:96        score, matched = _keyword_score("I need to reschedule the standup meeting", ["reschedule", "standup"], min_matches=2)97        assert score == 1.098        assert len(matched) == 299 100    def test_partial_match(self) -> None:101        score, matched = _keyword_score("I need to reschedule", ["reschedule", "standup"], min_matches=2)102        assert score == 0.5103        assert len(matched) == 1104 105    def test_no_match(self) -> None:106        score, matched = _keyword_score("Hello world", ["reschedule", "standup"], min_matches=2)107        assert score == 0.0108        assert len(matched) == 0109 110    def test_case_insensitive(self) -> None:111        score, _ = _keyword_score("RESCHEDULE THE STANDUP", ["reschedule", "standup"], min_matches=2)112        assert score == 1.0113 114 115class TestCalendarConflicts:116    def test_no_conflicts(self) -> None:117        scenario = get_scenario("easy_002")118        assert scenario is not None119        world = WorldState(scenario)120        assert _calendar_has_no_overlaps(world) is True121 122    def test_conflict_detected(self) -> None:123        scenario = get_scenario("easy_001")124        assert scenario is not None125        world = WorldState(scenario)126        assert _calendar_has_no_overlaps(world) is False127 128 129class TestCommitmentCoherence:130    def test_no_commitments_full_score(self) -> None:131        scenario = get_scenario("easy_005")132        assert scenario is not None133        world = WorldState(scenario)134        score, _ = _score_commitment_coherence(world)135        assert score == 1.0136 137    def test_honored_commitment(self, env: CommitmentEnvironment) -> None:138        env.reset(task_id="easy_001")139        env.step(CommitmentAction(action_type="reschedule_event", event_id="evt_2", new_time="15:00"))140        assert env._world is not None141        score, feedback = _score_commitment_coherence(env._world)142        assert score == 1.0143 144    def test_silent_violation_detected(self, env: CommitmentEnvironment) -> None:145        env.reset(task_id="easy_001")146        env.step(CommitmentAction(action_type="schedule_meeting", title="New Meeting", date="2026-04-25", time="16:00", participants=["Alice"]))147        assert env._world is not None148        env._world.calendar.pop("evt_100", None)149        for c in env._world.commitment_ledger:150            if c.commitment_type == "meeting_scheduled" and "16:00" in c.constraint:151                event_key = c.constraint152                for eid, ev in list(env._world.calendar.items()):153                    if ev.time == "16:00" and ev.date == "2026-04-25" and ev.title == "New Meeting":154                        del env._world.calendar[eid]155                        break156        violations = env._world.get_silent_violations()157        assert len(violations) >= 1158 159 160class TestStepEfficiency:161    def test_optimal_steps(self) -> None:162        scenario = get_scenario("easy_001")163        assert scenario is not None164        world = WorldState(scenario)165        world.step_count = 3166        score, _ = _score_step_efficiency(scenario, world)167        assert score == 1.0168 169    def test_over_optimal(self) -> None:170        scenario = get_scenario("easy_001")171        assert scenario is not None172        world = WorldState(scenario)173        world.step_count = 8174        score, _ = _score_step_efficiency(scenario, world)175        assert score == 0.5176 177 178# ===================================================================179# 3. Environment lifecycle180# ===================================================================181 182class TestEnvironmentLifecycle:183    def test_reset_returns_observation(self, env: CommitmentEnvironment) -> None:184        obs = env.reset(task_id="easy_001")185        assert isinstance(obs, CommitmentObservation)186        assert obs.scenario_id == "easy_001"187        assert obs.done is False188        assert obs.reward == 0.0189        assert len(obs.briefing) > 0190 191    def test_step_before_reset_raises(self, env: CommitmentEnvironment) -> None:192        with pytest.raises(ValueError, match="No active episode"):193            env.step(CommitmentAction(action_type="view_calendar", date="2026-04-25"))194 195    def test_step_after_done_raises(self, env: CommitmentEnvironment) -> None:196        env.reset(task_id="easy_001")197        env.step(CommitmentAction(action_type="submit_plan"))198        with pytest.raises(ValueError, match="already completed"):199            env.step(CommitmentAction(action_type="view_calendar", date="2026-04-25"))200 201    def test_state_property(self, env: CommitmentEnvironment) -> None:202        env.reset(task_id="easy_001")203        state = env.state204        assert isinstance(state, CommitmentState)205        assert state.scenario_id == "easy_001"206        assert state.completed is False207        assert len(state.available_tasks) == 15208 209    def test_multi_turn_episode(self, env: CommitmentEnvironment) -> None:210        env.reset(task_id="easy_001")211        obs = env.step(CommitmentAction(action_type="view_calendar", date="2026-04-25"))212        assert obs.done is False213        assert obs.step_number == 1214 215        obs = env.step(CommitmentAction(action_type="reschedule_event", event_id="evt_2", new_time="15:00"))216        assert obs.done is False217        assert obs.step_number == 2218 219        obs = env.step(CommitmentAction(action_type="submit_plan"))220        assert obs.done is True221        assert obs.reward > 0222 223    def test_max_steps_auto_submits(self, env: CommitmentEnvironment) -> None:224        env.reset(task_id="easy_002")225        for _ in range(20):226            obs = env.step(CommitmentAction(action_type="view_calendar", date="2026-04-25"))227            if obs.done:228                break229        assert obs.done is True230 231    def test_reset_clears_state(self, env: CommitmentEnvironment) -> None:232        env.reset(task_id="easy_001")233        env.step(CommitmentAction(action_type="view_calendar", date="2026-04-25"))234        env.reset(task_id="easy_002")235        assert env.state.scenario_id == "easy_002"236        assert env.state.step_count == 0237 238    def test_unknown_action_type(self, env: CommitmentEnvironment) -> None:239        env.reset(task_id="easy_001")240        obs = env.step(CommitmentAction(action_type="fly_to_moon"))241        assert "Unknown action_type" in obs.tool_result242 243    def test_random_reset(self, env: CommitmentEnvironment) -> None:244        obs = env.reset(seed=42)245        assert obs.scenario_id in get_all_scenarios()246 247    def test_difficulty_filter_reset(self, env: CommitmentEnvironment) -> None:248        obs = env.reset(difficulty="hard", seed=1)249        assert obs.difficulty == "hard"250 251 252# ===================================================================253# 4. World simulation (tool functions)254# ===================================================================255 256class TestWorldTools:257    def test_view_calendar(self) -> None:258        scenario = get_scenario("easy_001")259        assert scenario is not None260        world = WorldState(scenario)261        result = world.view_calendar("2026-04-25")262        assert "evt_1" in result263        assert "14:00" in result264 265    def test_view_calendar_empty(self) -> None:266        scenario = get_scenario("easy_001")267        assert scenario is not None268        world = WorldState(scenario)269        result = world.view_calendar("2099-01-01")270        assert "No events" in result271 272    def test_check_availability(self) -> None:273        scenario = get_scenario("easy_003")274        assert scenario is not None275        world = WorldState(scenario)276        result = world.check_availability("Client_Jones")277        assert "09:00" in result278 279    def test_check_availability_unknown(self) -> None:280        scenario = get_scenario("easy_001")281        assert scenario is not None282        world = WorldState(scenario)283        result = world.check_availability("NonExistentPerson")284        assert "not found" in result285 286    def test_search_restaurants_filters(self) -> None:287        scenario = get_scenario("med_007")288        assert scenario is not None289        world = WorldState(scenario)290        result = world.search_restaurants(dietary="vegan", max_price=45, max_distance_miles=3.0)291        assert "Green Garden" in result292        assert "Steak House Prime" not in result293 294    def test_schedule_meeting_creates_commitment(self) -> None:295        scenario = get_scenario("easy_002")296        assert scenario is not None297        world = WorldState(scenario)298        result = world.schedule_meeting("Test Meeting", "2026-04-25", "14:00", turn=1)299        assert "scheduled" in result.lower()300        assert len(world.commitment_ledger) == 1301        assert world.commitment_ledger[0].commitment_type == "meeting_scheduled"302 303    def test_schedule_meeting_conflict(self) -> None:304        scenario = get_scenario("easy_001")305        assert scenario is not None306        world = WorldState(scenario)307        result = world.schedule_meeting("Conflicting", "2026-04-25", "14:00", turn=1)308        assert "CONFLICT" in result309 310    def test_reschedule_event(self) -> None:311        scenario = get_scenario("easy_001")312        assert scenario is not None313        world = WorldState(scenario)314        result = world.reschedule_event("evt_2", "15:00", turn=1)315        assert "Rescheduled" in result316        assert world.calendar["evt_2"].time == "15:00"317 318    def test_cancel_event(self) -> None:319        scenario = get_scenario("easy_001")320        assert scenario is not None321        world = WorldState(scenario)322        result = world.cancel_event("evt_2", turn=1)323        assert "Cancelled" in result324        assert "evt_2" not in world.calendar325 326    def test_send_email(self) -> None:327        scenario = get_scenario("easy_001")328        assert scenario is not None329        world = WorldState(scenario)330        result = world.send_email("Team", "Hello", "Testing email body", turn=1)331        assert "sent" in result.lower()332        assert len(world.emails_sent) == 1333 334    def test_book_restaurant(self) -> None:335        scenario = get_scenario("easy_002")336        assert scenario is not None337        world = WorldState(scenario)338        result = world.book_restaurant("Bella Italia", turn=1)339        assert "confirmed" in result.lower()340        assert world.booked_restaurant == "Bella Italia"341 342 343# ===================================================================344# 5. Commitment ledger behaviour345# ===================================================================346 347class TestCommitmentLedger:348    def test_schedule_creates_commitment(self) -> None:349        scenario = get_scenario("easy_002")350        assert scenario is not None351        world = WorldState(scenario)352        world.schedule_meeting("Test", "2026-04-25", "10:00", turn=1)353        assert len(world.commitment_ledger) == 1354        c = world.commitment_ledger[0]355        assert c.turn_created == 1356        assert c.active is True357        assert c.renegotiated_at is None358 359    def test_reschedule_marks_old_renegotiated(self) -> None:360        scenario = get_scenario("easy_001")361        assert scenario is not None362        world = WorldState(scenario)363        world.reschedule_event("evt_2", "15:00", turn=1)364        renegotiated = [c for c in world.commitment_ledger if c.renegotiated_at is not None]365        assert len(renegotiated) == 0  # initial events don't create ledger entries366        new_commits = [c for c in world.commitment_ledger if c.active]367        assert len(new_commits) >= 1368 369    def test_email_renegotiation_detection(self) -> None:370        scenario = get_scenario("easy_001")371        assert scenario is not None372        world = WorldState(scenario)373        world.schedule_meeting("Important", "2026-04-25", "16:00", participants=["Alice"], turn=1)374        world.send_email("Alice", "Change of plans", "I need to reschedule our meeting", turn=2)375        renegotiated = [c for c in world.commitment_ledger if c.renegotiated_at is not None]376        assert len(renegotiated) >= 1377 378    def test_cancel_personal_marks_renegotiated(self) -> None:379        scenario = get_scenario("easy_001")380        assert scenario is not None381        world = WorldState(scenario)382        # evt_3 is Lunch (personal)383        world.cancel_event("evt_3", turn=1)384        # Personal cancellations are auto-OK385 386 387# ===================================================================388# 6. Full scenario scoring389# ===================================================================390 391class TestFullScoring:392    def test_perfect_easy_001(self, env: CommitmentEnvironment) -> None:393        env.reset(task_id="easy_001")394        env.step(CommitmentAction(action_type="reschedule_event", event_id="evt_2", new_time="15:00"))395        env.step(CommitmentAction(action_type="send_email", to="Team", subject="Standup moved", body="Hi team, I've rescheduled the standup to 3:00 PM. Sorry for the move."))396        obs = env.step(CommitmentAction(action_type="submit_plan"))397        assert obs.done is True398        assert obs.reward >= 0.85399 400    def test_zero_effort_gets_low_score(self, env: CommitmentEnvironment) -> None:401        env.reset(task_id="easy_001")402        obs = env.step(CommitmentAction(action_type="submit_plan"))403        assert obs.done is True404        assert obs.reward <= 0.50405 406    def test_hard_011_perfect_run(self, env: CommitmentEnvironment) -> None:407        env.reset(task_id="hard_011")408        env.step(CommitmentAction(action_type="view_calendar", date="2026-04-25"))409        env.step(CommitmentAction(action_type="cancel_event", event_id="evt_90"))410        env.step(CommitmentAction(action_type="search_restaurants", dietary="vegetarian", near_airport=True, max_price=60))411        env.step(CommitmentAction(action_type="book_restaurant", restaurant_name="Sky Lounge"))412        env.step(CommitmentAction(action_type="send_email", to="Team", subject="Happy Hour Rescheduled", body="Sorry team, I need to reschedule the happy hour to Thursday. An investor dinner came up tonight. Apologies!"))413        env.step(CommitmentAction(action_type="send_email", to="VP_Chen", subject="Investor dinner plan", body="I've booked Sky Lounge for dinner tonight with Investor_Park. Vegetarian options available, near the airport."))414        obs = env.step(CommitmentAction(action_type="submit_plan"))415        assert obs.done is True416        assert obs.reward >= 0.85417 418    def test_hard_015_sre_crisis(self, env: CommitmentEnvironment) -> None:419        env.reset(task_id="hard_015")420        env.step(CommitmentAction(action_type="view_calendar", date="2026-04-25"))421        env.step(CommitmentAction(action_type="cancel_event", event_id="evt_130"))422        env.step(CommitmentAction(action_type="send_email", to="Team", subject="Lunch cancelled - incident", body="Team, I'm cancelling our lunch due to a production incident. Payment service returning 503s. Will handle this first."))423        env.step(CommitmentAction(action_type="send_email", to="Client_Jones", subject="Demo reschedule needed", body="Hi Client_Jones, I sincerely apologize but I need to reschedule our demo. We have a production incident with the payment system. Can we find another time this week?"))424        env.step(CommitmentAction(action_type="send_email", to="VP_Chen", subject="Incident + 1-on-1", body="VP_Chen, we have a production incident — payment service is returning 503s. I'm on-call and handling it. May need to reschedule our 1-on-1 depending on resolution time."))425        obs = env.step(CommitmentAction(action_type="submit_plan"))426        assert obs.done is True427        assert obs.reward >= 0.60428 429 430# ===================================================================431# 7. Reward clamping432# ===================================================================433 434class TestRewardClamping:435    def test_reward_never_zero(self, env: CommitmentEnvironment) -> None:436        env.reset(task_id="easy_001")437        obs = env.step(CommitmentAction(action_type="submit_plan"))438        assert obs.reward >= 0.01439 440    def test_reward_never_one(self, env: CommitmentEnvironment) -> None:441        env.reset(task_id="easy_001")442        env.step(CommitmentAction(action_type="reschedule_event", event_id="evt_2", new_time="15:00"))443        env.step(CommitmentAction(action_type="send_email", to="Team", subject="Standup moved", body="Hi team, the standup is rescheduled to 3pm. Sorry for the move."))444        obs = env.step(CommitmentAction(action_type="submit_plan"))445        assert obs.reward <= 0.99446        assert obs.reward > 0.01447 448 449# ===================================================================450# 8. Time utility451# ===================================================================452 453class TestTimeUtil:454    def test_time_to_min(self) -> None:455        assert _time_to_min("00:00") == 0456        assert _time_to_min("09:30") == 570457        assert _time_to_min("14:00") == 840458        assert _time_to_min("23:59") == 1439459 460 461# ===================================================================462# 9. API endpoint tests (via TestClient)463# ===================================================================464 465class TestAPI:466    @pytest.fixture467    def client(self):468        from fastapi.testclient import TestClient469        from server.app import app470        return TestClient(app)471 472    def test_health(self, client) -> None:473        resp = client.get("/health")474        assert resp.status_code == 200475 476    def test_tasks(self, client) -> None:477        resp = client.get("/tasks")478        assert resp.status_code == 200479        data = resp.json()480        assert len(data["easy"]) == 5481        assert len(data["medium"]) == 5482        assert len(data["hard"]) == 5483 484    def test_reset_step_state(self, client) -> None:485        resp = client.post("/reset", params={"task_id": "easy_001"})486        assert resp.status_code == 200487 488        resp = client.post("/step", json={"action": {"action_type": "view_calendar", "date": "2026-04-25"}})489        assert resp.status_code == 200490        data = resp.json()491        assert data.get("done") is False492 493        resp = client.get("/state")494        assert resp.status_code == 200495        state = resp.json()496        assert "step_count" in state497 498    def test_mcp_initialize(self, client) -> None:499        resp = client.post("/mcp", json={500            "jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {},501        })502        assert resp.status_code == 200503        data = resp.json()504        assert data["result"]["serverInfo"]["name"] == "commitment-os"505 506    def test_mcp_tools_list(self, client) -> None:507        resp = client.post("/mcp", json={508            "jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {},509        })510        assert resp.status_code == 200511        tools = resp.json()["result"]["tools"]512        assert len(tools) == 3513        names = {t["name"] for t in tools}514        assert names == {515            "cos_episode_reset",516            "cos_environment_step",517            "cos_session_snapshot",518        }519 520 521# ===================================================================522# 10. Metadata523# ===================================================================524 525class TestMetadata:526    def test_get_metadata(self, env: CommitmentEnvironment) -> None:527        meta = env.get_metadata()528        assert meta.name == "commitment-os"529        assert "Jayant" in meta.author530