Parinith-reddy/hinglish-chatbot

1import re2 3# ── 1. Spelling + Chat + Number Slang — unified map ───────────────────────────4ABBREV_MAP = {5    # English shorthand6    "u": "you", "r": "are", "ur": "your", "y": "why",7    "k": "okay", "ok": "okay", "okk": "okay", "okkk": "okay",8    "m": "main", "b": "be",9    "wanna": "want to", "gonna": "going to", "gotta": "got to",10    "lol": "haha", "lmao": "bahut hasa", "rofl": "bahut hasa",11    "omg": "oh my god", "wtf": "kya yaar",12    "brb": "abhi aata hoon", "bbl": "baad mein aata hoon",13    "idk": "pata nahi", "imo": "meri ray mein", "imho": "meri ray mein",14    "tbh": "sach bataaun", "ngl": "jhooth nahi bolunga",15    "smh": "sharam aati hai", "irl": "real life mein",16    "fyi": "jankari ke liye", "asap": "jaldi se jaldi",17    "gg": "acha khela", "np": "koi baat nahi", "nw": "koi baat nahi",18    "ty": "shukriya", "thnx": "shukriya", "thx": "shukriya",19    "plz": "please", "pls": "please", "plss": "please",20    "sry": "sorry", "srry": "sorry",21    "msg": "message", "msgs": "messages",22    "pic": "photo", "pics": "photos",23    "bday": "birthday", "bf": "boyfriend", "gf": "girlfriend",24    "rn": "abhi", "atm": "is waqt",25    "btw": "by the way", "imo": "meri ray mein",26    "dm": "direct message", "pm": "private message",27    "fr": "for real", "istg": "sach bol raha hoon",28    "imo": "meri ray mein", "afaik": "jitna pata hai mujhe",29 30    # Number slang (point 5 from guide — was missing)31    "gr8": "great", "b4": "before", "2day": "today",32    "4u": "for you", "2moro": "tomorrow", "tmrw": "tomorrow",33    "2nite": "tonight", "l8r": "later", "l8": "late",34    "m8": "mate", "gr8": "great", "h8": "hate",35    "w8": "wait", "sk8": "skate", "1ce": "once",36    "2": "to", "4": "for", "8": "ate",37 38    # Hinglish spelling variations (point 1 from guide)39    "nhi": "nahi", "nai": "nahi", "nahin": "nahi", "nh": "nahi", "nahee": "nahi",40    "h": "hai", "hey": "hai", "he": "hai",41    "kyaaa": "kya", "kyaa": "kya",42    "bhut": "bahut", "bht": "bahut", "bohat": "bahut", "bhot": "bahut",43    "krta": "karta", "krte": "karte", "krna": "karna", "krega": "karega",44    "pta": "pata", "ptta": "pata",45    "kyu": "kyun", "kyunki": "kyunki",46    "bc": "yaar", "bro": "bhai",47    "yr": "yaar", "yar": "yaar", "yaara": "yaar",48    "hn": "haan", "han": "haan", "hna": "haan", "hm": "haan",49    "chl": "chalo", "chlo": "chalo",50    "abb": "ab",51    "sb": "sab", "sbse": "sabse",52    "kch": "kuch",53    "acha": "acha", "achha": "acha", "accha": "acha",54    "thnk": "shukriya",55    "kro": "karo",56    "smjh": "samajh", "smajh": "samajh", "samaj": "samajh",57    "lgta": "lagta", "lgti": "lagti",58    "rhna": "rehna",59    "ghr": "ghar", "ghur": "ghar",60    "wha": "wahan", "yha": "yahan",61    "phle": "pehle", "pehele": "pehle",62    "kr": "kar", "krdiya": "kar diya",63    "kyse": "kaise", "kese": "kaise",64    "kbhi": "kabhi",65    "fir": "phir",66    "idhr": "idhar", "udhr": "udhar",67    "jldi": "jaldi",68    "zyda": "zyada", "zada": "zyada",69    "puri": "poori", "puraa": "poora",70    "or": "aur",71    "to": "toh", "toh": "toh",72    "woh": "wo", "voh": "wo",73    "sath": "saath",74    "pass": "paas",75    "lye": "liye",76    "kam": "kaam",77    "tym": "time",78    "dinn": "din",79    "rat": "raat",80    "suba": "subah",81    "dostt": "dost",82    "ghaar": "ghar",83    "khaana": "khana",84    "mtt": "mat",85}86 87# ── 2. Word Merge Fix — merged words split into correct tokens ─────────────────88# (point 2 from guide — was completely missing before)89MERGE_FIX = {90    "karlia": "kar lia",91    "karli": "kar li",92    "karloo": "kar lo",93    "hojayega": "ho jayega",94    "hojayegi": "ho jayegi",95    "hogaya": "ho gaya",96    "hogayi": "ho gayi",97    "hojaega": "ho jaega",98    "dedo": "de do",99    "lelo": "le lo",100    "dekho": "dekho",101    "aajao": "aa jao",102    "aaja": "aa ja",103    "bata": "bata",104    "batao": "batao",105    "boldo": "bol do",106    "chodo": "chod do",107    "chaddo": "chad do",108    "maardo": "maar do",109    "jaane": "jaane",110    "jaado": "jaa do",111    "rakhlo": "rakh lo",112    "rakhdiya": "rakh diya",113    "pakadlo": "pakad lo",114    "pakadlia": "pakad lia",115    "nikaalo": "nikaal lo",116    "nikaaldiya": "nikaal diya",117    "khatamkaro": "khatam karo",118    "khatamhogaya": "khatam ho gaya",119    "theekhai": "theek hai",120    "theekhe": "theek hai",121    "thakhogaya": "thak ho gaya",122    "pagalhogaya": "pagal ho gaya",123    "achhahai": "acha hai",124    "kyahua": "kya hua",125    "kyakaru": "kya karun",126    "kyakaru": "kya karun",127    "nahijaana": "nahi jaana",128    "nahibolunga": "nahi bolunga",129    "milgaya": "mil gaya",130    "milgayi": "mil gayi",131    "bhuldiya": "bhul diya",132    "bhuljaata": "bhul jaata",133    "chalgaya": "chal gaya",134    "rukjao": "ruk jao",135    "rukjaa": "ruk ja",136    "bhaaglo": "bhaag lo",137    "bhaagja": "bhaag ja",138    "sojao": "so jao",139    "sojaata": "so jaata",140    "khajao": "kha jao",141    "pijaata": "pi jaata",142}143 144# ── 3. Emoji map — replace with Hinglish meaning, not remove ──────────────────145EMOJI_MAP = {146    "😂": "haha", "🤣": "bahut hasa", "😭": "ro raha hoon",147    "😢": "dukhi hoon", "😡": "gussa", "🤬": "bahut gussa",148    "😍": "bahut pasand", "🥰": "pyaar", "❤️": "dil",149    "💔": "dil toot gaya", "😊": "khush", "🙂": "theek hai",150    "😐": "kuch nahi", "🙄": "pagalpan", "😤": "frustrating",151    "👍": "theek hai", "👎": "nahi theek", "🙏": "please",152    "😴": "neend aa rahi hai", "🤔": "soch raha hoon",153    "😅": "thoda awkward", "😬": "ajeeb", "🥺": "please yaar",154    "🔥": "mast", "💯": "bilkul sahi", "✅": "theek hai",155    "😎": "cool", "🤝": "deal", "👀": "dekh raha hoon",156    "💀": "haha zyada", "🫡": "respect", "🫠": "thak gaya",157    "😩": "bahut thak gaya", "😫": "pareshan", "🥹": "emotional",158    "😻": "bahut pasand", "🤯": "hairan", "🤑": "paisa",159    "😱": "shock", "😰": "tension", "😨": "dar",160}161 162def replace_emojis(text):163    # collapse consecutive same emojis first164    import unicodedata165    for emoji, replacement in EMOJI_MAP.items():166        text = text.replace(emoji, f" {replacement} ")167    # Remove remaining unmapped emojis168    text = re.sub(r'[^\x00-\x7F\u0900-\u097F\s]', ' ', text)169    return text170 171def fix_merged_words(text):172    """Split common merged Hinglish words — point 2 from guide."""173    words = text.split()174    fixed = []175    for word in words:176        if word in MERGE_FIX:177            fixed.append(MERGE_FIX[word])178        else:179            fixed.append(word)180    return ' '.join(fixed)181 182def normalize_repetition(text):183    """pleaaaase → please, yaaaar → yaar — keeps max 2 of any char."""184    return re.sub(r'(.)\1{2,}', r'\1\1', text)185 186def normalize_punctuation(text):187    """188    kya???? → kya?   hello!!! → hello!189    Guide point 7 — preserve ONE punctuation mark to keep question/exclaim intent.190    Better than stripping all punctuation.191    """192    text = re.sub(r'\?{2,}', '?', text)193    text = re.sub(r'!{2,}', '!', text)194    text = re.sub(r'\.{2,}', '.', text)195    text = re.sub(r',{2,}', ',', text)196    return text197 198def expand_abbreviations(text):199    words = text.split()200    expanded = []201    for word in words:202        clean_word = word.strip('.,!?')203        if clean_word in ABBREV_MAP:204            expanded.append(ABBREV_MAP[clean_word])205        else:206            expanded.append(word)207    return ' '.join(expanded)208 209def remove_urls(text):210    return re.sub(r'http\S+|www\.\S+', '', text)211 212def remove_extra_chars(text):213    # Keep Hindi unicode, English, digits, and single punctuation214    text = re.sub(r'[^a-zA-Z0-9\u0900-\u097F\s\?\!\.\,]', ' ', text)215    return text216 217def normalize_whitespace(text):218    return re.sub(r'\s+', ' ', text).strip()219 220def normalize(text: str) -> str:221    """222    Full v2 normalization pipeline — 9 steps.223    Run this on EVERY input: training data AND live user input.224    """225    if not isinstance(text, str) or not text.strip():226        return ""227 228    text = replace_emojis(text)          # Step 1: emoji → meaning229    text = remove_urls(text)             # Step 2: strip URLs230    text = text.lower()                  # Step 3: lowercase231    text = fix_merged_words(text)        # Step 4: karlia → kar lia  [NEW]232    text = normalize_repetition(text)    # Step 5: pleaaase → please233    text = normalize_punctuation(text)   # Step 6: ??? → ?           [IMPROVED]234    text = expand_abbreviations(text)    # Step 7: nhi → nahi, u → you235    text = remove_extra_chars(text)      # Step 8: strip remaining noise236    text = normalize_whitespace(text)    # Step 9: clean spaces237 238    return text239 240 241# ── Tests ──────────────────────────────────────────────────────────────────────242if __name__ == "__main__":243    test_cases = [244        # Spelling variations245        ("bhai product bhutttt kharab h nhi kya",       "spelling + repetition"),246        ("kyaaa hojayega mera refund",                  "spelling + merge fix"),247        # Merged words248        ("bhai karlia wo cheez return",                 "merge fix"),249        ("hojayega theek sab",                          "merge fix"),250        # Repetition251        ("pleaaaase refund kro!!!!!!",                  "repetition + punctuation"),252        ("yaaaaar nooooo mat karo ye",                  "repetition"),253        # Punctuation254        ("kya tum aaoge???",                            "punctuation preserved"),255        ("bilkul sahi hai!!!",                          "punctuation preserved"),256        # Emojis257        ("ngl dil toot gaya yaar 💔💔💔",               "emoji"),258        ("bhai ye toh mast hai 🔥🔥",                   "emoji"),259        # Number slang260        ("b4 meeting gr8 idea tha",                     "number slang [NEW]"),261        ("2day ka plan kya hai",                        "number slang [NEW]"),262        # Mixed real-world263        ("plz krna ye kaam asap nhi toh prob hoga",     "real world mix"),264        ("omg bhai hogaya theek finally 😂😂",           "real world mix"),265    ]266 267    print(f"{'Input':<48} {'Type':<25} → Normalized Output")268    print("─" * 110)269    for text, kind in test_cases:270        result = normalize(text)271        print(f"{text:<48} {kind:<25} → {result}")272