ridwanalfarezi/inez-normalizer

1import csv2import os3 4csv_path = os.path.join('app', 'data', 'slang_lookup.csv')5 6# Read existing entries7with open(csv_path, 'r', encoding='utf-8') as f:8    reader = csv.DictReader(f)9    rows = list(reader)10 11# Build existing mapping12existing = {r['slang'].lower().strip(): r for r in rows}13 14# Additional entries to add or fix15additions = [16    # Fix: gk should map to tidak (was gak from first duplicate in original)17    ("gk", "tidak", "manual-fix"),18    # Missing common abbreviations19    ("mkn", "makan", "manual"),20    ("mkan", "makan", "manual"),21    ("mknan", "makanan", "manual"),22    ("prgi", "pergi", "manual"),23    ("kntor", "kantor", "manual"),24    ("ksna", "kesana", "manual"),25    ("lp", "lupa", "manual"),26    ("tau", "tahu", "manual"),27    ("tw", "tahu", "manual"),28    ("ktmu", "ketemu", "manual"),29    ("sna", "sana", "manual"),30    ("di", "di", "manual"),  # identity - skip31    ("aneh", "aneh", "manual"),  # identity - skip32    ("santai", "santai", "manual"),  # identity - skip33    ("tidur", "tidur", "manual"),  # identity - skip34    ("ko", "kok", "manual"),35    ("bku", "buku", "manual"),36    ("lpr", "lapar", "manual"),37    ("msh", "masih", "manual"),38    ("plg", "pulang", "manual"),39    ("coba", "coba", "manual"),  # identity - skip40]41 42added = 043fixed = 044skipped = 045 46for slang, formal, source in additions:47    slang = slang.lower().strip()48    formal = formal.lower().strip()49    50    # Skip identity mappings51    if slang == formal:52        skipped += 153        continue54    55    if slang in existing:56        old_formal = existing[slang]['formal']57        if old_formal != formal:58            # Fix existing entry59            existing[slang]['formal'] = formal60            existing[slang]['source'] = source61            print(f"  FIXED: {slang}: {old_formal} -> {formal}")62            fixed += 163        else:64            skipped += 165    else:66        # Add new entry67        existing[slang] = {'slang': slang, 'formal': formal, 'source': source}68        print(f"  ADDED: {slang} -> {formal}")69        added += 170 71# Write back sorted72all_entries = sorted(existing.values(), key=lambda r: r['slang'])73 74with open(csv_path, 'w', encoding='utf-8', newline='') as f:75    writer = csv.DictWriter(f, fieldnames=['slang', 'formal', 'source'])76    writer.writeheader()77    for row in all_entries:78        writer.writerow(row)79 80print(f"\nDone: {added} added, {fixed} fixed, {skipped} skipped")81print(f"Total entries: {len(all_entries)}")82