chefpy/tr-ecommerce-nlpp

1import os, re, json, joblib
2import numpy as np
3from sentence_transformers import SentenceTransformer
4from transformers import AutoConfig, pipeline
5from hdbscan import approximate_predict
6
7ART_DIR = os.environ.get("ART_DIR", "artifacts")
8
9# ---- Config oku ----
10CFG_PATH = os.path.join(ART_DIR, "config.json")
11with open(CFG_PATH, "r", encoding="utf-8") as f:
12    CFG = json.load(f)
13
14EMB_MODEL_NAME  = CFG.get("embedding_model_name", "paraphrase-multilingual-MiniLM-L12-v2")
15SENT_MODEL_NAME = CFG.get("sentiment_model_name", "incidelen/xlm-roberta-base-turkish-sentiment-analysis")
16
17CALIB = CFG.get("calibration", {})
18POS_T_DEFAULT   = float(CALIB.get("positive_threshold", 0.55))
19NEG_T_DEFAULT   = float(CALIB.get("negative_threshold", 0.55))
20NOISE_P_DEFAULT = float(CALIB.get("noise_prob_threshold", 0.15))
21
22# ---- Modelleri yükle ----
23# Clustering (UMAP + HDBSCAN) sürüm uyumsuzluğu yaşanırsa servis ayakta kalsın:
24try:
25    umap_model = joblib.load(os.path.join(ART_DIR, "umap_model.joblib"))
26    clusterer  = joblib.load(os.path.join(ART_DIR, "hdbscan_model.joblib"))
27    CLUSTER_OK = True
28except Exception as e:
29    print("UMAP/HDBSCAN yüklenemedi, yalnızca sentiment çalışacak. Sebep:", e)
30    umap_model, clusterer = None, None
31    CLUSTER_OK = False
32
33# Embedding & Sentiment
34emb_model = SentenceTransformer(EMB_MODEL_NAME)
35cfg = AutoConfig.from_pretrained(SENT_MODEL_NAME)
36id2label = {int(k): v for k, v in getattr(cfg, "id2label", {}).items()} or None
37
38sentiment_pipe = pipeline(
39    "sentiment-analysis",
40    model=SENT_MODEL_NAME,
41    tokenizer=SENT_MODEL_NAME,
42    device=-1  # HF Spaces CPU
43)
44
45# ---- Preprocess ----
46_NUM_RE  = re.compile(r"\d+")
47_URL_RE  = re.compile(r"http\S+")
48_PUNC_RE = re.compile(r"[^\w\sçğıöşü]")
49
50def clean_text(s: str) -> str:
51    s = _URL_RE.sub("", str(s)).lower()
52    s = _NUM_RE.sub(" <NUM> ", s)
53    s = _PUNC_RE.sub(" ", s)
54    return re.sub(r"\s+", " ", s).strip()
55
56def norm_label(raw: str) -> str:
57    u = raw.upper()
58    if u.startswith("LABEL_") and id2label:
59        idx = int(u.split("_")[-1])
60        return id2label.get(idx, raw).lower()
61    return raw.lower()
62
63def calibrate(label: str, score: float, pos_t: float, neg_t: float) -> str:
64    if label == "positive" and score < pos_t: return "neutral"
65    if label == "negative" and score < neg_t: return "neutral"
66    return label
67
68# ---- Public API ----
69def analyze_sentence(sentence: str,
70                     pos_t: float = POS_T_DEFAULT,
71                     neg_t: float = NEG_T_DEFAULT,
72                     noise_prob_t: float = NOISE_P_DEFAULT):
73    cleaned = clean_text(sentence)
74
75    # Küme (varsa)
76    if CLUSTER_OK:
77        emb = emb_model.encode([cleaned], convert_to_numpy=True, normalize_embeddings=True, show_progress_bar=False)
78        umv = umap_model.transform(emb)
79        (c_label,), (c_prob,) = approximate_predict(clusterer, umv)
80        c_label, c_prob = int(c_label), float(c_prob)
81    else:
82        c_label, c_prob = -1, 0.0
83
84    # Sentiment
85    out = sentiment_pipe([cleaned], truncation=True, max_length=256)[0]
86    lab  = norm_label(out["label"])
87    scr  = float(out["score"])
88    lab  = calibrate(lab, scr, pos_t=pos_t, neg_t=neg_t)
89
90    # Basit aksiyon
91    if lab == "negative":
92        action = "Şikayet/iade akışına yönlendir; temsilci önceliklendir."
93    elif lab == "positive":
94        action = "Teşekkür + küçük kupon/puan öner."
95    else:
96        action = "Ek bilgi iste veya öneri kutusu göster."
97
98    fallback = None
99    if (not CLUSTER_OK) or (c_label == -1) or (c_prob < noise_prob_t):
100        fallback = "Küme güveni düşük veya kapalı. 'Genel destek' intent’ine yönlendir."
101
102    return {
103        "cleaned": cleaned,
104        "cluster_label": c_label,
105        "cluster_prob": round(c_prob, 3),
106        "sentiment_label": lab,
107        "sentiment_score": round(scr, 3),
108        "fallback": fallback,
109        "action": action
110    }
111