LiProject/LemmaLi

1import os2import re3import torch4import pandas as pd5import gradio as gr6from transformers import AutoTokenizer, AutoModelForSeq2SeqLM7 8# =========================================================================9# 1. Sabitler ve Model Yükleme10# =========================================================================11 12HF_MODEL_ID = "LiProject/BERT-Turkish-Lemmatization-V3"13DEVICE = "cuda" if torch.cuda.is_available() else "cpu"14CONFIDENCE_THRESHOLD = 0.70 # Güven skoru eşiği (Arka planda çalışır)15 16try:17    tok = AutoTokenizer.from_pretrained(HF_MODEL_ID, use_fast=True)18    mdl = AutoModelForSeq2SeqLM.from_pretrained(HF_MODEL_ID).to(DEVICE).eval()19    print(f"Model yükleme başarılı: {HF_MODEL_ID} ({DEVICE} üzerinde)")20except Exception as e:21    print(f"Model veya Tokenizer yüklenirken kritik hata oluştu: {e}")22    raise SystemExit(1)23 24# =========================================================================25# 2. Arka Plan İşlemleri26# =========================================================================27 28def get_lemma_for_word(word: str) -> str:29    """30    Tek kelimeyi temizler, modelden geçirir. 31    """32    clean_word = word.strip(".,!?();:\"'’")33 34    if not clean_word:35        return word36 37    # Eğer kelime sadece sayı veya sayı+ek ise direkt döndür38    num_match = re.match(r"^(\d+(?:[.,]\d+)?)(?:['’.]?[a-zA-ZğüşıöçĞÜŞİÖÇ]*)$", clean_word)39    if num_match:40        return num_match.group(1)41 42    inputs = tok(clean_word, return_tensors="pt", truncation=True, max_length=128).to(DEVICE)43    44    # Olasılık skorlarını almak için parametreler ekleniyor45    outputs = mdl.generate(46        **inputs, 47        max_length=128,48        return_dict_in_generate=True,49        output_scores=True50    )51    52    sequences = outputs.sequences53    54    # Token seviyesinde log-olasılık skorlarını hesapla55    transition_scores = mdl.compute_transition_scores(56        sequences, outputs.scores, normalize_logits=True57    )58    59    # Tüm dizinin (kelimenin) ortak olasılığını hesapla60    seq_log_prob = transition_scores[0].sum().item()61    confidence_score = torch.exp(torch.tensor(seq_log_prob)).item()62    63    lemma = tok.decode(sequences[0], skip_special_tokens=True).strip()64    65    # Üretilen kök boşsa veya arka plandaki güven skoru %85'in altındaysa orijinali kullan66    if not lemma or confidence_score < CONFIDENCE_THRESHOLD:67        return clean_word68 69    return lemma70 71 72@torch.inference_mode()73def lemmatize_rows(multiline_text: str):74    rows = []75    sentences = [s.strip() for s in multiline_text.splitlines() if s.strip()]76 77    if not sentences:78        return pd.DataFrame(columns=["Full_Sentence", "Word", "Lemma"])79 80    for sent in sentences:81        words = sent.split()82        for w in words:83            l = get_lemma_for_word(w)84            rows.append({85                "Full_Sentence": sent,86                "Word": w,87                "Lemma": l88            })89 90    return pd.DataFrame(rows)91 92 93def add_sentence_separators(df: pd.DataFrame, char: str = "-", repeat: int = 10) -> pd.DataFrame:94    if df.empty:95        return df96 97    rows = []98    prev = None99 100    for _, r in df.iterrows():101        if prev is not None and r["Full_Sentence"] != prev:102            sep = char * repeat103            rows.append({104                "Full_Sentence": sep,105                "Word": sep,106                "Lemma": sep107            })108        rows.append(r.to_dict())109        prev = r["Full_Sentence"]110 111    return pd.DataFrame(rows)112 113 114def run_and_save(text):115    df = lemmatize_rows(text)116    df_view = add_sentence_separators(df, char="-", repeat=10)117 118    out_path = "lemma_output.csv"119    df.to_csv(out_path, index=False, encoding="utf-8-sig")120 121    return df_view, out_path122 123 124examples = [125    "Yolcular, zorlu yollarda yolculuk yaparken yoldan çıkmamaya özen gösterirler.",126    "Öğrenciler 2'şerli gruplar halinde 15.30'da içeri alındılar.",127    "Benimki seninkinden daha güzelmiş, dedi usulca."128]129 130# =========================================================================131# 3. Gradio Arayüzü132# =========================================================================133 134theme = gr.themes.Soft(135    primary_hue="blue",136    secondary_hue="slate",137    neutral_hue="slate"138)139 140custom_css = """141.gradio-container {142    max-width: 1100px !important;143    margin: 0 auto !important;144    padding-top: 20px !important;145}146 147#input_text textarea {148    min-height: 190px !important;149    font-size: 15px !important;150    line-height: 1.5 !important;151}152 153#results_table {154    max-height: 420px !important;155    overflow: auto !important;156}157 158#results_table table {159    table-layout: fixed !important;160    width: 100% !important;161}162 163#results_table th, #results_table td {164    white-space: normal !important;165    word-break: break-word !important;166}167 168.main-title {169    text-align: center;170    margin-bottom: 4px;171}172 173.sub-text {174    text-align: center;175    opacity: 0.9;176    margin-bottom: 18px;177}178 179.info-box {180    border: 1px solid #cbd5e1;181    border-radius: 14px;182    padding: 14px 16px;183    margin-top: 12px;184    margin-bottom: 16px;185    background: rgba(148,163,184,0.08);186}187 188footer {189    visibility: hidden !important;190}191"""192 193with gr.Blocks(title="Türkçe Lemmatizer") as demo:194 195    gr.HTML("""196        <div class="main-title">197            <h1>Türkçe Lemmatization Aracı</h1>198        </div>199        <div class="sub-text">200            Türkçe cümleleri kelime kelime işleyerek köklerini çıkarır ve CSV olarak indirmenizi sağlar.201        </div>202    """)203 204    gr.HTML(f"""205        <div class="info-box">206            <b>Model:</b> {HF_MODEL_ID}<br>207            <b>Çalışma mantığı:</b> Metin satır satır, her satır da kelime kelime işlenir. 208    209        </div>210    """)211 212    with gr.Row():213        with gr.Column(scale=3):214            inp = gr.Textbox(215                label="Metin Girişi",216                placeholder="Buraya bir veya birden fazla Türkçe cümle yazın...",217                lines=8,218                elem_id="input_text"219            )220 221            gr.Examples(222                examples=[[e] for e in examples],223                inputs=inp,224                label="Örnek girdiler"225            )226 227        with gr.Column(scale=1):228            btn = gr.Button("Kökleri Bul", variant="primary")229            clr = gr.Button("Temizle", variant="secondary")230 231    out_tbl = gr.Dataframe(232        headers=["Full_Sentence", "Word", "Lemma"],233        label="Sonuç Önizleme",234        interactive=False,235        wrap=True,236        elem_id="results_table"237    )238 239    out_file = gr.File(label="CSV Çıktısı")240 241    btn.click(242        fn=run_and_save,243        inputs=inp,244        outputs=[out_tbl, out_file]245    )246 247    inp.submit(248        fn=run_and_save,249        inputs=inp,250        outputs=[out_tbl, out_file]251    )252 253    clr.click(254        fn=lambda: ("", None, None),255        inputs=None,256        outputs=[inp, out_tbl, out_file]257    )258 259if __name__ == "__main__":260    demo.launch(theme=theme, css=custom_css)