Kagamicho/cs_chatbot

1"""Normalize curated-entry `topic` values into the 12 canonical buckets.2 3The distillation LLMs produced 54 free-form topics; this folds the ~42 long-tail4variants into the 12 canonical buckets for cleaner faceting in kb_review.html.5 6NOTE: `topic` is metadata only — it does NOT affect retrieval (BM25 + vector +7priority don't read it). This is a browsing/hygiene change, not an accuracy8change. It edits the curated JSONL in place (backs up first). No re-index needed.9"""10from __future__ import annotations11 12import json13import shutil14from collections import Counter15from pathlib import Path16 17ROOT = Path(__file__).resolve().parent.parent18CURATED = ROOT / "data" / "curated"19FILES = ["arekore.jsonl", "cs_low_voltage.jsonl", "agentforce.jsonl"]20 21CANONICAL = {22    "新規申込", "切替", "解約・キャンセル", "引越し", "料金・請求", "支払方法",23    "燃調・市場連動", "ポータル・アカウント", "契約内容変更", "供給・停電",24    "書類・手続き", "その他",25}26 27# every non-canonical topic -> a canonical bucket28MAPPING = {29    "請求・支払い": "料金・請求", "請求・料金": "料金・請求", "請求書": "料金・請求",30    "料金プラン": "料金・請求", "プラン概要": "料金・請求", "プラン": "料金・請求",31    "プラン・サービス": "料金・請求", "プラン・契約": "料金・請求", "料金のしくみ": "料金・請求",32    "オプションサービス・料金": "料金・請求", "オール電化": "料金・請求",33    "支払い・口座振替": "支払方法", "支払い": "支払方法", "支払い方法": "支払方法",34    "お支払い": "支払方法", "口座振替": "支払方法", "支払い・期日": "支払方法",35    "クレジットカード登録": "支払方法",36    "書類・約款": "書類・手続き",37    "アカウント・ポータル": "ポータル・アカウント", "お客様ポータル": "ポータル・アカウント",38    "アンペア・契約容量": "契約内容変更", "契約手続き": "契約内容変更",39    "契約変更・お手続き": "契約内容変更", "契約・名義変更": "契約内容変更",40    "手続き・変更": "契約内容変更",41    "契約・解約": "解約・キャンセル", "解約・変更": "解約・キャンセル",42    "申し込み・切り替え": "新規申込", "申し込み・切替": "新規申込", "新規申込・引越し": "引越し",43    "切替日": "切替",44    "送電停止・復旧": "供給・停電", "電気工事": "供給・停電",45    "キャンペーン・特典": "その他", "制度・キャンペーン": "その他", "概要": "その他",46    "電気の基礎知識": "その他", "電気の仕組み": "その他", "トラブル・問い合わせ": "その他",47    "お問い合わせ・連絡": "その他", "問い合わせ窓口": "その他",48}49 50 51def canon(topic: str) -> str:52    if topic in CANONICAL:53        return topic54    return MAPPING.get(topic, "その他")55 56 57def main() -> None:58    before: Counter = Counter()59    after: Counter = Counter()60    unmapped: set[str] = set()61    for fname in FILES:62        p = CURATED / fname63        if not p.exists():64            continue65        shutil.copy(p, p.with_suffix(".jsonl.bak"))66        out_lines = []67        for line in p.read_text(encoding="utf-8").splitlines():68            line = line.strip()69            if not line:70                continue71            e = json.loads(line)72            t = e.get("topic", "")73            before[t] += 174            if t not in CANONICAL and t not in MAPPING:75                unmapped.add(t)76            e["topic"] = canon(t)77            after[e["topic"]] += 178            out_lines.append(json.dumps(e, ensure_ascii=False))79        p.write_text("\n".join(out_lines) + "\n", encoding="utf-8")80    print(f"distinct topics: {len(before)} -> {len(after)}")81    print("\ncanonical distribution after:")82    for t, n in after.most_common():83        print(f"  {n:4}  {t}")84    if unmapped:85        print("\nWARNING fell through to その他 (not in CANONICAL or MAPPING):")86        for t in sorted(unmapped):87            print("  ", t)88 89 90if __name__ == "__main__":91    main()92