saykeau/cebuano-verse-mistral

!pip install unsloth

from unsloth import FastLanguageModel

max_seq_length = 2048
dtype = None
load_in_4bit = True

alpaca_prompt = """Sa ubos usa ka panudlo nga naghulagway sa usa ka buluhaton. Pagsulat og tubag nga haom nga mokompleto sa hangyo.

### Instruksyon:
{}

### Input:
{}

### Tubag:
{}"""

model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="saykeau/cebuano-verse-mistral",
    max_seq_length=max_seq_length,
    dtype=dtype,
    load_in_4bit=load_in_4bit,
)

FastLanguageModel.for_inference(model)  # Enable native 2x faster inference

inputs = tokenizer(
    [
        alpaca_prompt.format(
            "Pagsulat ug bersikulo bahin sa <topic> in a <mood> tone.",
            "",
            "",  # output - leave this blank for generation!
        ),
    ],
    return_tensors="pt",
).to("cuda")

from transformers import TextStreamer

text_streamer = TextStreamer(tokenizer)
_ = model.generate(
    **inputs,
    max_new_tokens=128,
    streamer=text_streamer,
    use_cache=True,
    do_sample=True,
    top_k=50,
    temperature=0.9
)