Hemil087/content_moderation_env
0
1"""2Baseline inference script for the Content Moderation Environment.3 4Runs an LLM agent against all 3 tasks (easy, medium, hard) and produces5reproducible baseline scores.6 7Usage:8 python inference.py9 10Required environment variables:11 API_BASE_URL - The API endpoint for the LLM12 MODEL_NAME - The model identifier to use for inference13 HF_TOKEN - Hugging Face API key (primary)14 OPENAI_API_KEY - OpenAI API key (fallback)15"""16 17import json18import os19import sys20 21from openai import OpenAI22 23# ---------------------------------------------------------------------------24# Configuration25# ---------------------------------------------------------------------------26 27API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")28MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/Qwen2.5-72B-Instruct")29API_KEY = os.getenv("HF_TOKEN") or os.getenv("OPENAI_API_KEY", "")30ENV_URL = os.getenv(31 "ENV_URL",32 "https://hemil087-content-moderation-env.hf.space",33)34 35TASKS = ["easy", "medium", "hard"]36MAX_STEPS = 837ENV_NAME = "content_moderation_env"38 39# ---------------------------------------------------------------------------40# OpenAI client41# ---------------------------------------------------------------------------42 43client = OpenAI(44 base_url=API_BASE_URL,45 api_key=API_KEY,46)47 48# ---------------------------------------------------------------------------49# Structured logging helpers — exact format required by judges50# ---------------------------------------------------------------------------51 52def log_start(task: str, env: str, model: str) -> None:53 print(f"[START] task={task} env={env} model={model}", flush=True)54 55 56def log_step(step: int, action: str, reward: float, done: bool, error) -> None:57 error_val = error if error else "null"58 done_val = str(done).lower()59 print(60 f"[STEP] step={step} action={action} reward={reward:.2f} done={done_val} error={error_val}",61 flush=True,62 )63 64 65def log_end(success: bool, steps: int, score: float, rewards: list) -> None:66 rewards_str = ",".join(f"{r:.2f}" for r in rewards)67 print(68 f"[END] success={str(success).lower()} steps={steps} score={score:.2f} rewards={rewards_str}",69 flush=True,70 )71 72 73# ---------------------------------------------------------------------------74# System prompt for the LLM agent75# ---------------------------------------------------------------------------76 77SYSTEM_PROMPT = """78You are an expert content moderator for a social media platform.79 80Your job is to review posts and make fair, consistent moderation decisions81based on the platform's community policy.82 83## AVAILABLE ACTIONS84 85You must respond with a JSON object choosing ONE of these actions:86 871. retrieve_precedents — Search for similar past cases to guide your decision88 {"action_type": "retrieve_precedents", "query": "your search query here"}89 902. remove_content — Remove the post for policy violation91 {"action_type": "remove_content", "reason": "explanation here"}92 933. allow_content — Approve the post as policy-compliant94 {"action_type": "allow_content", "reason": "explanation here"}95 964. add_warning_label — Keep post but add a content warning97 {"action_type": "add_warning_label", "reason": "explanation here"}98 995. escalate — Send to senior human moderator100 {"action_type": "escalate", "reason": "explanation here"}101 1026. overturn_removal — For appeals: reverse the prior removal decision103 {"action_type": "overturn_removal", "reason": "explanation here"}104 1057. uphold_removal — For appeals: confirm the prior removal was correct106 {"action_type": "uphold_removal", "reason": "explanation here"}107 108## MANDATORY STRATEGY109 110You MUST follow this exact two-step process:111 112STEP 1 (ALWAYS): Call retrieve_precedents first with a query describing the key issue.113 - For hate speech cases, query: "dehumanizing [target group]"114 - For political content, query: "political criticism opinion"115 - For satire/comedy, query: "satire comedy offensive"116 - For graphic content, query: "graphic disturbing news"117 - For appeals, query: "appeal [original removal reason]"118 119STEP 2: After reviewing precedents, make your final decision based on:120 - The platform policy121 - The precedent cases you retrieved122 - The specific content and context123 124## DECISION GUIDELINES125 126- Content that dehumanizes protected groups (race, religion, gender, sexuality, disability) → remove_content127- Political criticism targeting policies or public figures' actions → allow_content128- Factual news reporting, even on sensitive topics → allow_content129- Historical quotes with educational context → allow_content130- Graphic/disturbing content with news value → add_warning_label131- Edgy comedy or satire that is ambiguous → add_warning_label132- User threatens legal action or to leave platform → escalate133- Appeals where original removal was wrong (e.g., news/satire removed) → overturn_removal134- Appeals where original removal was justified → uphold_removal135 136## CRITICAL RULES137- ALWAYS retrieve precedents first. Never skip this step.138- ONLY escalate when the user explicitly threatens legal action or to leave the platform.139- When unsure between remove and warning label, prefer add_warning_label for comedy/satire.140- Never call the same action twice — it will be penalized.141- ALWAYS include a "reason" field citing the specific policy clause that applies.142 143## RESPONSE FORMAT144 145Always respond with ONLY a valid JSON object. No explanation outside the JSON.146Example: {"action_type": "retrieve_precedents", "query": "hate speech dehumanizing ethnicity"}147""".strip()148 149 150# ---------------------------------------------------------------------------151# Agent loop152# ---------------------------------------------------------------------------153 154def build_user_message(observation: dict) -> str:155 """Build the user message from the current observation."""156 parts = []157 158 parts.append(f"POST TO REVIEW:\n{observation.get('post_content', '')}")159 160 metadata = observation.get("post_metadata", {})161 if metadata:162 parts.append(f"POST METADATA:\n{json.dumps(metadata, indent=2)}")163 164 policy = observation.get("policy_summary", "")165 if policy:166 parts.append(f"PLATFORM POLICY:\n{policy}")167 168 precedents = observation.get("precedents", [])169 if precedents:170 prec_text = "\n".join([171 f"[{p['case_id']}] Decision: {p['decision']} | Reason: {p['reason']}\nContent: {p['content']}"172 for p in precedents173 ])174 parts.append(f"RETRIEVED PRECEDENTS:\n{prec_text}")175 176 actions_taken = observation.get("actions_taken", [])177 if actions_taken:178 parts.append(f"ACTIONS TAKEN SO FAR: {', '.join(actions_taken)}")179 180 message = observation.get("message", "")181 if message:182 parts.append(f"ENVIRONMENT FEEDBACK: {message}")183 184 return "\n\n".join(parts)185 186 187def get_llm_action(conversation_history: list) -> tuple:188 """Call LLM and parse action from response."""189 response = client.chat.completions.create(190 model=MODEL_NAME,191 messages=conversation_history,192 temperature=0.0,193 max_tokens=256,194 )195 196 content = response.choices[0].message.content.strip()197 198 # Strip markdown fences if present199 if content.startswith("```"):200 content = content.split("```")[1]201 if content.startswith("json"):202 content = content[4:]203 content = content.strip()204 205 try:206 action = json.loads(content)207 except json.JSONDecodeError:208 action = {"action_type": "escalate", "reason": "Unable to parse LLM response"}209 210 return action, content211 212 213def run_episode(env_client, task_id: str) -> float:214 """215 Run one full episode for the given task_id.216 Returns the final score (0.0-1.0).217 Emits structured stdout logs in [START]/[STEP]/[END] format.218 """219 rewards = []220 steps_taken = 0221 score = 0.0222 223 log_start(task_id, ENV_NAME, MODEL_NAME)224 225 try:226 # Reset environment227 result = env_client.reset(options={"task_id": task_id})228 obs = result.observation229 230 conversation_history = [{"role": "system", "content": SYSTEM_PROMPT}]231 232 for step in range(MAX_STEPS):233 if obs.done:234 break235 236 # Build user message from current observation237 user_message = build_user_message(obs.__dict__)238 conversation_history.append({"role": "user", "content": user_message})239 240 # Get LLM action241 action_dict, raw_response = get_llm_action(conversation_history)242 conversation_history.append({"role": "assistant", "content": raw_response})243 244 # Build action object245 from content_moderation_env.models import ContentModerationAction246 action = ContentModerationAction(247 action_type=action_dict.get("action_type", "escalate"),248 reason=action_dict.get("reason"),249 query=action_dict.get("query"),250 )251 252 # Step environment253 result = env_client.step(action)254 obs = result.observation255 reward = result.reward or 0.0256 done = result.done257 258 steps_taken = step + 1259 rewards.append(reward)260 261 log_step(262 step=steps_taken,263 action=action_dict.get("action_type", "unknown"),264 reward=reward,265 done=done,266 error=None,267 )268 269 if done:270 score = obs.reward # final graded score271 break272 273 except Exception as e:274 print(f"[DEBUG] Episode error: {e}", file=sys.stderr)275 276 finally:277 success = score > 0.0278 log_end(success, steps_taken, score, rewards)279 280 return score281 282 283# ---------------------------------------------------------------------------284# Main285# ---------------------------------------------------------------------------286 287def main():288 from content_moderation_env.client import ContentModerationEnv289 290 with ContentModerationEnv(base_url=ENV_URL).sync() as env:291 for task_id in TASKS:292 run_episode(env, task_id)293 294 295if __name__ == "__main__":296 main()