Dhrona1421/multimodal-content-moderation
0
1"""2inference.py — Advanced inference with chain-of-thought LLM agent and3 deterministic rule-based baseline.4 5LLM agent features:6 • Multi-step chain-of-thought reasoning (image → user → text → decision)7 • 3-shot examples covering easy/medium/hard cases8 • Structured JSON output: action + confidence + reasoning chain9 • Retry with exponential backoff on API errors10 • Conversation-style context window (last 2 steps retained)11 12Rule-based agent features:13 • Weighted keyword scoring with per-group thresholds14 • Cross-modal conflict detection (safe text + harmful image)15 • Repeat-offender escalation via session history16 • Confidence calibrated to signal strength17 18Both agents support the (action, confidence, reasoning) -> Tuple return signature19required by the grader for ECE scoring.20"""21 22from __future__ import annotations23 24import json25import os26import re27import sys28import time29from typing import Any, Dict, List, Optional, Tuple30 31from features import (32 FEATURE_DIM, ACTIONS, IMAGE_TAGS, USER_TYPES,33 KEYWORD_GROUPS, BIGRAM_SIGNALS,34)35from grader import ModerationGrader36 37for stream_name in ("stdout", "stderr"):38 stream = getattr(sys, stream_name, None)39 if hasattr(stream, "reconfigure"):40 stream.reconfigure(encoding="utf-8")41 42# ── Configuration ─────────────────────────────────────────────────────────────43 44MODEL_NAME = os.environ.get("MODEL_NAME", "gpt-4o-mini")45API_BASE_URL = os.environ.get("API_BASE_URL", "https://api.openai.com/v1")46HF_TOKEN = os.environ.get("HF_TOKEN", os.environ.get("OPENAI_API_KEY", ""))47 48CONFIDENCE_THRESHOLD = 0.45 # below → escalate49 50# ── LLM client ───────────────────────────────────────────────────────────────51 52try:53 from openai import OpenAI # type: ignore54 _client = OpenAI(api_key=HF_TOKEN, base_url=API_BASE_URL) if HF_TOKEN else None55 LLM_AVAILABLE = _client is not None56except ImportError:57 _client = None58 LLM_AVAILABLE = False59 60# ─────────────────────────────────────────────────────────────────────────────61# Chain-of-thought system prompt62# ─────────────────────────────────────────────────────────────────────────────63 64_FEW_SHOT = """65--- EXAMPLE 1 (EASY — remove) ---66Post: "DM me for Oxy, Xanax, no script needed 🔥 Discreet shipping worldwide"67Image: drugs | User: suspicious | History: 068Reasoning:69 1. IMAGE: drugs tag confirms pharmaceutical content70 2. USER: suspicious account with no session context71 3. TEXT: names controlled substances, explicit "no script" = illegal sale72 4. CROSS: all signals align — no conflict73Decision: remove (confidence 0.97)74 75--- EXAMPLE 2 (MEDIUM — flag) ---76Post: "Just saying, crime rates went way up after THOSE people moved in. Funny how that works 👀"77Image: safe | User: new78Reasoning:79 1. IMAGE: safe — no direct visual harm80 2. USER: new account, no history81 3. TEXT: "those people" + crime correlation = coded racist dog-whistle, no explicit slur82 4. CROSS: text harm, image safe — moderate severity83Decision: flag (confidence 0.78)84 85--- EXAMPLE 3 (HARD — allow) ---86Post: "As an ICU nurse, knowing acetaminophen toxicity thresholds can save lives. Always check labels 🏥"87Image: drugs | User: trusted88Reasoning:89 1. IMAGE: drugs tag — pills in photo90 2. USER: trusted, no session flags91 3. TEXT: professional medical harm-reduction context, no instructional misuse92 4. CROSS: drugs image + trusted + educational context → net safe93Decision: allow (confidence 0.82)94""".strip()95 96SYSTEM_PROMPT = f"""You are a senior content moderator at a major social media platform.97 98You will receive a social media post with:99 text — caption or body text100 image_tag — safe | nudity | violence | drugs | misleading101 user_type — new | trusted | suspicious102 user_history — weighted recency-decayed session flag count (0–5)103 step — current position in the moderation queue104 105YOUR TASK: decide one action and rate your confidence.106 107ACTIONS:108 allow — safe content, publish immediately109 flag — borderline, route to human review queue110 remove — clear violation, immediate takedown111 112CHAIN-OF-THOUGHT PROTOCOL — you must evaluate ALL four signals:113 1. IMAGE: Does the image tag alone warrant action?114 2. USER: Does user trust level shift the decision?115 3. TEXT: What does the text signal (keywords, tone, intent)?116 4. CROSS: Do signals conflict? (safe text + harmful image → image wins)117 118CONFIDENCE CALIBRATION:119 ≥ 0.85 = very certain (clear violation or clear safe)120 0.60–0.84 = confident with minor ambiguity121 0.45–0.59 = uncertain, leaning one way122 < 0.45 = genuinely unsure → triggers human escalation (partial credit)123 124MODERATION RULES:125 • Harmful image overrides safe-looking text126 • Trusted users get moderate benefit of the doubt — NOT for serious violations127 • Suspicious user with history ≥ 2 warrants extra scrutiny128 • Medical/educational context with drugs/violence can be allow129 • Satire/sports framing does NOT automatically justify graphic content130 • When torn between flag and remove → choose flag131 132EXAMPLES:133{_FEW_SHOT}134 135Respond ONLY with valid JSON (no markdown fences):136{{137 "reasoning": {{138 "image_signal": "one sentence",139 "user_signal": "one sentence",140 "text_signal": "one sentence",141 "cross_modal": "one sentence",142 "final_logic": "one sentence"143 }},144 "action": "allow|flag|remove",145 "confidence": 0.0-1.0146}}"""147 148 149# ─────────────────────────────────────────────────────────────────────────────150# Observation formatter151# ─────────────────────────────────────────────────────────────────────────────152 153def format_observation(obs: Dict[str, Any]) -> str:154 return (155 f"[Post #{obs.get('post_id','?')} | "156 f"Step {obs.get('step','?')}/{obs.get('max_steps','?')} | "157 f"Difficulty: {obs.get('difficulty','?').upper()}]\n\n"158 f"text : {obs.get('text','')}\n"159 f"image_tag : {obs.get('image_tag','safe')}\n"160 f"user_type : {obs.get('user_type','new')}\n"161 f"user_history : {obs.get('user_history', 0):.2f} "162 f"(weighted recency-decayed session flags)\n"163 f"step : {obs.get('step',1)}/{obs.get('max_steps',12)}"164 )165 166 167# ─────────────────────────────────────────────────────────────────────────────168# LLM agent169# ─────────────────────────────────────────────────────────────────────────────170 171def _clean_json(text: str) -> str:172 """Extract and clean a JSON block from potentially messy LLM output."""173 # 1. Strip markdown code fences (```json or ```)174 text = re.sub(r"```(?:json)?", "", text)175 text = re.sub(r"```", "", text)176 # 2. Extract anything between the first { and the last }177 match = re.search(r"\{.*\}", text, re.DOTALL)178 if match:179 return match.group(0).strip()180 return text.strip()181 182 183def llm_agent(184 obs: Dict[str, Any],185 max_retries: int = 3,186) -> Tuple[str, float, Dict[str, str]]:187 """188 Chain-of-thought LLM agent. Falls back to rule_based_agent on error.189 """190 if not LLM_AVAILABLE:191 return rule_based_agent(obs)192 193 for attempt in range(max_retries):194 try:195 resp = _client.chat.completions.create( # type: ignore[union-attr]196 model=MODEL_NAME,197 messages=[198 {"role": "system", "content": SYSTEM_PROMPT},199 {"role": "user", "content": format_observation(obs)},200 ],201 temperature=0.0,202 max_tokens=450,203 )204 raw = resp.choices[0].message.content or ""205 clean = _clean_json(raw)206 parsed = json.loads(clean)207 208 action = str(parsed.get("action", "flag")).lower().strip()209 confidence = float(parsed.get("confidence", 0.75))210 reasoning = parsed.get("reasoning", {})211 212 if action not in ACTIONS:213 action = "flag"214 confidence = float(max(0.0, min(1.0, confidence)))215 return action, confidence, reasoning216 217 except Exception as exc:218 wait = 1 + (attempt * 2)219 print(f" [LLM] Attempt {attempt+1} failed: {exc}. Retrying in {wait}s...",220 file=sys.stderr)221 time.sleep(wait)222 223 a, c, r = rule_based_agent(obs)224 return a, c, r225 226 227# ─────────────────────────────────────────────────────────────────────────────228# Rule-based agent (deterministic, no API)229# ─────────────────────────────────────────────────────────────────────────────230 231def _keyword_score(text: str, group: str) -> float:232 """Return fraction of group keywords present in text."""233 kws = KEYWORD_GROUPS.get(group, [])234 hits = sum(1 for kw in kws if kw in text)235 return min(hits / max(len(kws) * 0.25, 1.0), 1.0)236 237 238def _phrase_hit(text: str, group: str) -> bool:239 return any(p in text for p in BIGRAM_SIGNALS.get(group, []))240 241 242def rule_based_agent(obs: Dict[str, Any]) -> Tuple[str, float, Dict[str, str]]:243 """244 Deterministic rule-based agent using weighted keyword scoring245 and cross-modal conflict resolution.246 247 Returns (action, confidence, reasoning_dict).248 """249 image = obs.get("image_tag", "safe")250 user = obs.get("user_type", "new")251 text = obs.get("text", "").lower()252 history = float(obs.get("user_history", 0))253 254 # ── Keyword scores ────────────────────────────────────────────────────────255 spam_s = _keyword_score(text, "spam_scam")256 hate_s = _keyword_score(text, "hate_speech")257 viol_s = _keyword_score(text, "violence")258 drug_s = _keyword_score(text, "drugs")259 misinfo_s = _keyword_score(text, "misleading")260 harm_s = _keyword_score(text, "self_harm")261 fraud_s = _keyword_score(text, "financial_fraud")262 safe_s = _keyword_score(text, "safe_positive")263 edu_s = _keyword_score(text, "professional_educational")264 265 # ── Phrase signals ────────────────────────────────────────────────────────266 credible_threat = _phrase_hit(text, "credible_threat")267 coded_sales = _phrase_hit(text, "coded_sales")268 authoritative = _phrase_hit(text, "authoritative_deny")269 community_pos = _phrase_hit(text, "community_positive")270 271 # ── Trust modifier ────────────────────────────────────────────────────────272 trust_factor = {"trusted": 0.7, "new": 1.0, "suspicious": 1.3}.get(user, 1.0)273 # history amplifier: each unit of decayed history adds 10% weight274 hist_factor = 1.0 + min(history * 0.1, 0.5)275 276 # ─────────────────────────────────────────────────────────────────────────277 # Decision tree (ordered by severity)278 # ─────────────────────────────────────────────────────────────────────────279 280 # 1. Absolute removes — image-driven, regardless of text/user281 if image == "nudity":282 return "remove", 0.97, {"image_signal": "Nudity tag requires immediate takedown."}283 284 if credible_threat:285 return "remove", 0.95, {"text_signal": "Credible threat detected in caption."}286 287 if image == "drugs" and spam_s > 0.4 * trust_factor:288 conf = round(min(0.85 + spam_s * 0.1, 0.97), 3)289 return "remove", conf, {"cross_modal": "Drug image combined with pharmaceutical sales text."}290 291 if spam_s > 0.55 * trust_factor and user != "trusted":292 conf = round(min(0.82 + spam_s * 0.1, 0.97), 3)293 return "remove", conf, {"text_signal": "High spam score detected for non-trusted user."}294 295 if fraud_s > 0.5 and image in ("safe", "misleading"):296 act = "remove" if user == "suspicious" else "flag"297 return act, 0.85, {"text_signal": f"Financial fraud indicators ({fraud_s:.2f}) lead to {act}."}298 299 # 2. Remove on high harm text + suspicious300 if user == "suspicious" and hist_factor > 1.3:301 if hate_s > 0.3 or viol_s > 0.3:302 conf = round(min(0.72 + hist_factor * 0.05, 0.90), 3)303 return "remove", conf, {"user_signal": "Suspicious user with harmful text history."}304 305 # 3. Flag — image-driven306 if image == "misleading":307 if authoritative and edu_s > 0.2:308 return "allow", 0.75, {"final_logic": "Agent actively debunking misinformation."}309 conf = 0.80 if user != "trusted" else 0.72310 return "flag", round(conf * trust_factor, 3), {"image_signal": "Misleading image tag requires review."}311 312 if image == "violence":313 sports = ["boxing", "mma", "ufc", "wrestling", "match", "fight night",314 "professional", "tournament"]315 if any(k in text for k in sports) and edu_s > 0.1:316 return "allow", 0.76, {"final_logic": "Content recognized as professional sports."}317 return "flag", round(0.79 * trust_factor, 3), {"image_signal": "Violent image tag requires review."}318 319 if image == "drugs":320 if edu_s > 0.3 and user == "trusted":321 return "allow", 0.80, {"final_logic": "Medical/educational drug content from trusted user."}322 return "flag", round(0.76 * trust_factor, 3), {"image_signal": "Drug image tag requires review."}323 324 # 4. Flag — text-driven325 weighted_hate = hate_s * trust_factor * hist_factor326 if weighted_hate > 0.25:327 conf = round(min(0.65 + weighted_hate * 0.2, 0.90), 3)328 return "flag", conf, {"text_signal": "Hate speech indicators above threshold."}329 330 if misinfo_s > 0.35:331 return "flag", round(0.74 * trust_factor, 3), {"text_signal": "Misinformation indicators detected."}332 333 if harm_s > 0.3:334 return "flag", 0.78, {"text_signal": "Self-harm indicators detected."}335 336 if drug_s > 0.4:337 return "flag", round(0.72 * trust_factor, 3), {"text_signal": "Drug-related keywords detected."}338 339 if coded_sales and user != "trusted":340 return "flag", round(0.68 * trust_factor * hist_factor, 3), {"text_signal": "Coded trafficking language detected."}341 342 # 5. Allow343 if image == "safe":344 if user == "trusted":345 conf = 0.93346 elif user == "new":347 conf = 0.82348 else:349 if hist_factor > 1.3:350 return "flag", 0.58, {"user_signal": "Suspicious user with high flag history."}351 conf = 0.70352 if safe_s > 0.4 or community_pos:353 conf = min(conf + 0.05, 0.97)354 return "allow", round(conf, 3), {"final_logic": "Standard safe content with no flags."}355 356 return "allow", 0.62, {"final_logic": "Default allow (no clear violations detected)."}357 358 359# ─────────────────────────────────────────────────────────────────────────────360# Full evaluation runner361# ─────────────────────────────────────────────────────────────────────────────362 363def run_inference(364 force_rule_based: bool = False,365 dataset_path: str = "moderation_dataset.json",366 seed: int = 42,367 task_filter: Optional[str] = None,368 verbose: bool = False,369 extra_agents: Optional[Dict[str, Any]] = None,370) -> Dict[str, Any]:371 """372 Run full evaluation and return grading reports for all agents.373 374 Args:375 force_rule_based: Skip LLM, run only rule-based.376 dataset_path: JSON dataset path.377 seed: RNG seed.378 task_filter: Grade only this task ('easy'/'medium'/'hard').379 verbose: Step-by-step breakdown.380 extra_agents: Optional dict {name: agent_fn} for comparison.381 """382 agents: Dict[str, Any] = {}383 384 if not force_rule_based and LLM_AVAILABLE:385 # Wrap llm_agent to only return (action, confidence) for the grader386 agents[f"LLM ({MODEL_NAME})"] = lambda obs: llm_agent(obs)[:2]387 388 # Wrap rule_based_agent to only return (action, confidence) for the grader389 agents["Rule-Based"] = lambda obs: rule_based_agent(obs)[:2]390 391 if extra_agents:392 agents.update(extra_agents)393 394 grader = ModerationGrader(dataset_path=dataset_path, seed=seed)395 reports: Dict[str, Any] = {}396 397 print(f"\n{'═'*78}")398 print(f" Content Moderation Environment — Inference & Evaluation")399 print(f" Agents: {', '.join(agents.keys())} Seed: {seed}")400 print(f"{'═'*78}\n")401 402 for agent_name, agent_fn in agents.items():403 print(f" Running: {agent_name}")404 if task_filter:405 result = grader.grade_single_task(task_filter, agent_fn)406 report = {407 "aggregate_score": result["score"],408 "tasks": {task_filter: result},409 "summary": grader._build_summary({task_filter: result}, result["score"]),410 }411 else:412 report = grader.grade_all_tasks(agent_fn)413 414 reports[agent_name] = report415 grader.print_report(report, verbose=verbose)416 417 # ── Comparison table ──────────────────────────────────────────────────────418 if len(agents) > 1:419 print(f"\n{'═'*78}")420 print(f" AGENT COMPARISON")421 print(f"{'─'*78}")422 tasks_shown = list(reports[list(agents.keys())[0]]["tasks"].keys())423 header = f" {'Agent':<25}" + "".join(f" {t.upper():>10}" for t in tasks_shown) + " AGGREGATE"424 print(header)425 print(f"{'─'*78}")426 for name, rpt in reports.items():427 row = f" {name:<25}"428 for t in tasks_shown:429 row += f" {rpt['tasks'][t]['score']:>10.4f}"430 row += f" {rpt['aggregate_score']:>9.4f}"431 print(row)432 print(f"{'═'*78}\n")433 434 # Save results435 slim: Dict[str, Any] = {436 "seed": seed,437 "agents": {438 name: {439 "aggregate_score": rpt["aggregate_score"],440 "tasks": {441 t: {k: v for k, v in d.items() if k not in ("step_results",)}442 for t, d in rpt["tasks"].items()443 },444 }445 for name, rpt in reports.items()446 },447 }448 with open("results.json", "w", encoding="utf-8") as fh:449 json.dump(slim, fh, indent=2)450 print(" [Results saved → results.json]\n")451 452 return reports453 454 455# ─────────────────────────────────────────────────────────────────────────────456# CLI457# ─────────────────────────────────────────────────────────────────────────────458 459if __name__ == "__main__":460 args = sys.argv[1:]461 seed = int(args[args.index("--seed") + 1]) if "--seed" in args else 42462 reports = run_inference(463 force_rule_based="--rule-based" in args,464 verbose="--verbose" in args,465 seed=seed,466 task_filter=(args[args.index("--task") + 1] if "--task" in args else None),467 )468