spk-22/Context_Aware_Content_Moderation_Environment_using_OpenEnv

1import asyncio2import os3from typing import List, Optional4from openai import OpenAI5 6from content_moderation.client import ContentModerationEnv7from content_moderation.models import ContentModerationAction8 9# ENV CONFIG10API_KEY = os.getenv("HF_TOKEN") or os.getenv("API_KEY")11API_BASE_URL = os.getenv("API_BASE_URL") or "https://router.huggingface.co"12MODEL_NAME = os.getenv("MODEL_NAME") or "Qwen/Qwen2.5-72B-Instruct-VL"13 14# PROMPT15SYSTEM_PROMPT = """16You are a content moderation system.17 18You will receive:19- content20- context21- intent22 23Decide ONE:24allow / remove / review25 26Rules:27- Threats or abuse → remove28- Uncertain / sarcastic / harsh → review29- Safe / positive → allow30 31Reply ONLY with one word:32allow OR remove OR review33"""34 35# LOGGING (STRICT FORMAT)36def log_start(task: str, env: str, model: str):37    print(f"[START] task={task} env={env} model={model}", flush=True)38 39def log_step(step: int, action: str, reward: float, done: bool, error: Optional[str]):40    error_val = error if error else "null"41    done_val = str(done).lower()42    print(43        f"[STEP] step={step} action={action} reward={reward:.2f} done={done_val} error={error_val}",44        flush=True,45    )46 47def log_end(success: bool, steps: int, score: float, rewards: List[float]):48    rewards_str = ",".join(f"{r:.2f}" for r in rewards)49    print(50        f"[END] success={str(success).lower()} steps={steps} score={score:.2f} rewards={rewards_str}",51        flush=True,52    )53 54# LLM DECISION55def get_decision(client: OpenAI, content, context, intent):56    prompt = f"""57Content: {content}58Context: {context}59Intent: {intent}60 61Decision:62"""63 64    try:65        completion = client.chat.completions.create(66            model=MODEL_NAME,67            messages=[68                {"role": "system", "content": SYSTEM_PROMPT},69                {"role": "user", "content": prompt},70            ],71            temperature=0.0,72            max_tokens=5,73        )74 75        text = (completion.choices[0].message.content or "").strip().lower()76 77        # ensure valid output78        if text not in ["allow", "remove", "review"]:79            return "review"80 81        return text82 83    except Exception as e:84        print(f"[DEBUG] LLM error: {e}", flush=True)85        return "review"86 87 88# MAIN89async def main():90    client = OpenAI(base_url=API_BASE_URL, api_key=API_KEY)91    env = ContentModerationEnv(base_url="http://localhost:8000")92 93    rewards: List[float] = []94    steps_taken = 095    score = 0.096    success = False97 98    log_start(task="content_moderation", env="content_moderation", model=MODEL_NAME)99 100    try:101        # RESET ENV102        result = await env.reset()103        obs = result.observation104 105        # GET DECISION106        decision = get_decision(client, obs.content, obs.context, obs.intent)107 108        # STEP ENV (ONLY ONCE)109        result = await env.step(ContentModerationAction(decision=decision))110 111        reward = result.reward or 0.0112        done = result.done113 114        rewards.append(reward)115        steps_taken = 1116 117        log_step(step=1, action=decision, reward=reward, done=done, error=None)118 119        # SCORE120        score = reward121        success = score >= 0.5122 123    finally:124        try:125            await env.close()126        except Exception:127            pass128 129        log_end(success=success, steps=steps_taken, score=score, rewards=rewards)130 131 132if __name__ == "__main__":133    asyncio.run(main())