Nefertury/Tatar_language

1 2import os, torch, gradio as gr3from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig4from peft import PeftModel5 6BASE_MODEL   = os.getenv("BASE_MODEL", "Qwen/Qwen2.5-7B")7ADAPTER_REPO = os.getenv("ADAPTER_REPO", "your-username/tt-qwen25-7b-tt-lora")8LOAD_IN_4BIT = os.getenv("LOAD_IN_4BIT", "true").lower() == "true"9 10def load_model():11    tok = AutoTokenizer.from_pretrained(BASE_MODEL, use_fast=False)12    if tok.pad_token_id is None:13        tok.pad_token = tok.eos_token14 15    base = None16    if LOAD_IN_4BIT:17        try:18            bnb_cfg = BitsAndBytesConfig(19                load_in_4bit=True,20                bnb_4bit_use_double_quant=True,21                bnb_4bit_quant_type="nf4",22                bnb_4bit_compute_dtype=torch.float16,  # float16 для Spaces GPU23            )24            base = AutoModelForCausalLM.from_pretrained(25                BASE_MODEL, quantization_config=bnb_cfg, device_map="auto"26            )27            print("Loaded base in 4-bit NF4")28        except Exception as e:29            print("[warn] 4-bit failed:", e)30 31    if base is None:32        try:33            bnb8 = BitsAndBytesConfig(load_in_8bit=True)34            base = AutoModelForCausalLM.from_pretrained(35                BASE_MODEL, quantization_config=bnb8, device_map="auto"36            )37            print("Loaded base in 8-bit")38        except Exception as e:39            print("[warn] 8-bit failed:", e)40            base = AutoModelForCausalLM.from_pretrained(41                BASE_MODEL, torch_dtype=torch.float16, device_map="auto"42            )43            print("Loaded base in FP16 (may offload to CPU)")44 45    base.config.pad_token_id = tok.pad_token_id46    model = PeftModel.from_pretrained(47        base, ADAPTER_REPO, is_trainable=False, torch_dtype=torch.float1648    )49    model = model.to(dtype=torch.float16)50    model.eval()51    return tok, model52 53tok, model = load_model()54 55def format_prompt(user, system, mode):56    if mode == "Qwen chat":57        msgs = [{"role":"system","content":system},{"role":"user","content":user}]58        input_ids = tok.apply_chat_template(msgs, add_generation_prompt=True, return_tensors="pt")59        attn = torch.ones_like(input_ids)60        return {"input_ids": input_ids.to(model.device), "attention_mask": attn.to(model.device)}61    else:62        prompt = f"<|system|> {system}\n<|user|> {user}\n<|assistant|>"63        enc = tok(prompt, return_tensors="pt")64        return {65            "input_ids": enc["input_ids"].to(model.device),66            "attention_mask": enc["attention_mask"].to(model.device)67        }68 69@torch.inference_mode()70def respond(message, history, system_prompt, mode, temperature, top_p, rep_penalty, max_new_tokens):71    inputs = format_prompt(message, system_prompt, mode)72    with torch.autocast("cuda", dtype=torch.float16):73        out = model.generate(74            **inputs,75            do_sample=True,76            temperature=temperature,77            top_p=top_p,78            repetition_penalty=rep_penalty,79            max_new_tokens=max_new_tokens,80            pad_token_id=tok.pad_token_id,81            eos_token_id=tok.eos_token_id,82            no_repeat_ngram_size=483        )84    gen_only = out[0][inputs["input_ids"].shape[1]:]85    text = tok.decode(gen_only, skip_special_tokens=True)86    return text87 88with gr.Blocks() as demo:89    gr.Markdown("## Татарча чат-демо (Qwen2.5-7B + LoRA)")90    gr.Markdown("Бета-версия. Модель обучена отвечать **по-татарски**. Если переключаться на русский/английский — это ошибка; сообщите нам примеры.")91    with gr.Row():92        system_prompt = gr.Textbox(93            value="Син бары тик татарча гына җавап бир. Җавапларың кыска һәм нейтраль булсын.",94            label="System prompt"95        )96        mode = gr.Radio(choices=["SFT tags", "Qwen chat"], value="SFT tags", label="Формат промпта")97    with gr.Row():98        temperature      = gr.Slider(0.1, 1.2, value=0.7, step=0.05, label="temperature")99        top_p            = gr.Slider(0.1, 1.0, value=0.9, step=0.05, label="top_p")100        rep_penalty      = gr.Slider(1.0, 1.4, value=1.15, step=0.05, label="repetition_penalty")101        max_new_tokens   = gr.Slider(16, 512, value=200, step=8, label="max_new_tokens")102 103    gr.ChatInterface(104        fn=respond,105        additional_inputs=[system_prompt, mode, temperature, top_p, rep_penalty, max_new_tokens],106        title=None, undo_btn=None, retry_btn=None, clear_btn="Clear"107    )108 109demo.queue(concurrency_count=1).launch()110