alakxender/gemma-3-270m-dhivehi-content-gen

from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
import torch

# Load model
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
import torch

# Load model
model_path = "alakxender/gemma-3-270m-dhivehi-text-gen"
model = AutoModelForCausalLM.from_pretrained(
    model_path, 
    torch_dtype="auto", 
    device_map="auto", 
    attn_implementation="eager"
)
tokenizer = AutoTokenizer.from_pretrained(model_path)
pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)

# Generate content
title_or_prompt = "ދިވެހިރާއްޖެއަކީ އިންޑިޔާ ކަނޑުގައި އޮންނަ ޖަޒީރާ ޤައުމެކެވެ"

# Create the prompt format used during training
prompt = f"Create a dhivehi article for the following topic: {title_or_prompt}"

# Create chat format message for content generation (matching training format)
messages = [
    {"role": "system", "content": "You are a helpful assistant that can generate dhivehi articles based on a given topic."},
    {"role": "user", "content": prompt}
]

# Apply chat template
formatted_prompt = pipe.tokenizer.apply_chat_template(
    messages, 
    tokenize=False, 
    add_generation_prompt=True
)

# Generation parameters
gen_kwargs = {
    "max_new_tokens": 256,
    "temperature": 0.7,
    "top_p": 0.9,
    "top_k": 50,
    "do_sample": True,
    "disable_compile": True,
    "pad_token_id": tokenizer.eos_token_id
}

# Generate content
outputs = pipe(formatted_prompt, **gen_kwargs)

# Extract generated content (remove the prompt)
generated_content = outputs[0]['generated_text'][len(formatted_prompt):].strip()

print(f"Generated content: {generated_content}")