im-sangwoon/chatprot-qwen2.5-32b-lora

from transformers import AutoTokenizer, AutoModelForCausalLM
from peft import PeftModel
import torch

base_model_id = "Qwen/Qwen2.5-32B-Instruct"
adapter_id = "im-sangwoon/chatprot-qwen2.5-32b-lora"

# Load base model
model = AutoModelForCausalLM.from_pretrained(
    base_model_id,
    torch_dtype=torch.bfloat16,
    device_map="auto",
    trust_remote_code=True,
)

# Load LoRA adapter
model = PeftModel.from_pretrained(model, adapter_id)

# Load tokenizer
tokenizer = AutoTokenizer.from_pretrained(base_model_id, trust_remote_code=True)

# Inference
messages = [
    {"role": "system", "content": "You are a helpful assistant for protein analysis."},
    {"role": "user", "content": "What is the function of Hemoglobin subunit alpha?"}
]

text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = tokenizer([text], return_tensors="pt").to(model.device)

outputs = model.generate(**inputs, max_new_tokens=512, temperature=0.7, do_sample=True)
response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
print(response)

python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen2.5-32B-Instruct \
    --enable-lora \
    --lora-modules chatprot=im-sangwoon/chatprot-qwen2.5-32b-lora \
    --dtype bfloat16 \
    --trust-remote-code \
    --max-model-len 8192 \
    --max-lora-rank 64