dispatchAI/mcp-safety-classifier

1import gradio as gr2import json3import re4 5TOXIC_KEYWORDS = ["kill","harm","violent","attack","hate","stupid","idiot","racist","destroy","weapon","bomb","poison","abuse","threat","curse"]6 7INJECTION_PATTERNS = [8    r"ignore (all|previous) instructions", r"disregard (the above|previous|your)",9    r"you are now (a|an) ", r"system prompt", r"\[system\]",10    r"act as (if you are|a different)", r"override (your|the) rules",11    r"forget (everything|all|your instructions)", r"new instructions:",12    r"jailbreak", r"DAN mode", r"developer mode",13    r"reveal (your|the) (system|prompt|instructions)",14]15 16PII_PATTERNS = {17    "email": r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b",18    "phone": r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b",19    "ssn": r"\b\d{3}-\d{2}-\d{4}\b",20    "credit_card": r"\b(?:\d[ -]*?){13,16}\b",21    "ip_address": r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b",22}23 24def classify_safety(text: str) -> str:25    """Classify text for safety: toxicity, prompt injection, and PII detection."""26    lower = text.lower()27    toxic_hits = [kw for kw in TOXIC_KEYWORDS if kw in lower]28    toxicity_score = min(len(toxic_hits) / 3, 1.0)29    is_toxic = toxicity_score >= 0.530    31    injection_hits = [p for p in INJECTION_PATTERNS if re.search(p, text, re.IGNORECASE)]32    is_injection = len(injection_hits) > 033    34    pii_found = {}35    for pii_type, pattern in PII_PATTERNS.items():36        matches = re.findall(pattern, text, re.IGNORECASE)37        if matches:38            pii_found[pii_type] = len(matches)39    has_pii = len(pii_found) > 040    41    is_safe = not is_toxic and not is_injection and not has_pii42    43    return json.dumps({44        "is_safe": is_safe,45        "toxicity": {"detected": is_toxic, "score": round(toxicity_score, 2), "flagged_words": toxic_hits},46        "prompt_injection": {"detected": is_injection, "matched_patterns": len(injection_hits)},47        "pii_detected": {"found": has_pii, "types": pii_found},48        "recommendation": "ALLOW" if is_safe else "BLOCK",49    }, indent=2)50 51def detect_pii(text: str) -> str:52    """Detect personally identifiable information in text."""53    results = {}54    for pii_type, pattern in PII_PATTERNS.items():55        matches = re.findall(pattern, text, re.IGNORECASE)56        if matches:57            results[pii_type] = {"count": len(matches), "examples": matches[:3]}58    return json.dumps({"pii_found": len(results) > 0, "types": results}, indent=2)59 60with gr.Blocks(theme=gr.themes.Soft(primary_hue="blue"), title="dispatchAI Safety Classifier") as demo:61    gr.Markdown("# 🛡️ dispatchAI On-Device Safety Classifier (MCP)")62    with gr.Tab("Safety Check"):63        s_input = gr.Textbox(label="Text to check", lines=5)64        s_btn = gr.Button("Classify Safety", variant="primary")65        s_out = gr.Textbox(label="Assessment (JSON)", lines=15)66        s_btn.click(fn=classify_safety, inputs=s_input, outputs=s_out)67    with gr.Tab("PII Detection"):68        p_input = gr.Textbox(label="Text to scan", lines=5)69        p_btn = gr.Button("Detect PII", variant="primary")70        p_out = gr.Textbox(label="PII Report (JSON)", lines=12)71        p_btn.click(fn=detect_pii, inputs=p_input, outputs=p_out)72    gr.Markdown("---\n🚀 [dispatchAI](https://huggingface.co/dispatchAI) — Small. Mobile. Free. UAE-built.")73 74demo.launch(mcp_server=True)75