tiam4tt/TinyLlama-1.1B-chat.v1.0-linux-qna

from unsloth import FastLanguageModel
model_name = "tiam4tt/TinyLlama-1.1B-chat.v1.0-linux-qna"
model, tokenizer = FastLanguageModel.from_pretrained(
            model_name=model_name,
            max_seq_length=256,
            dtype=None, # Use None for automatic dtype detection
            load_in_4bit=True,
        )
        # Enable the model for inference
        FastLanguageModel.for_inference(model)

PROMPT = """Below is a question relating to the Linux operating system, paired with a paragraph describing further context. Write a short, simple, concise, and comprehensive response to the question.
### Question
{}
### Context
{}
### Response
{}"""

question = "How do I check the disk usage of a directory in Linux?"

inputs = tokenizer(PROMPT.format(
    question,
    "",
    "" # Leave blank for generation
    ),
    return_tensors="pt").to(model.device)
output = model.generate(
            **inputs,
            max_new_tokens=256,
            do_sample=True,
            temperature=1.0,
            top_p=0.9
        )
answer = tokenizer.decode(output[0], skip_special_tokens=True)
print(answer)