tepirale/gemma-4-12B-coder-fable5-composer2.5-v1-assistant-safetensors-yuxinlu1

import torch
from transformers import AutoProcessor, AutoModelForImageTextToText, AutoModelForCausalLM

M = "tepirale/gemma-4-12B-coder-fable5-composer2.5-v1-safetensors-yuxinlu1"   # la misma carpeta, ahora con embedders
MA= "tepirale/gemma-4-12B-coder-fable5-composer2.5-v1-assistant-safetensors-yuxinlu1"

model = AutoModelForImageTextToText.from_pretrained(M, dtype=torch.bfloat16, device_map="auto")
assistant_model = AutoModelForCausalLM.from_pretrained(MA, dtype=torch.bfloat16, device_map="auto")

processor = AutoProcessor.from_pretrained(M)


# Prompt - add image before text
messages = [
    {
        "role": "user", "content": [
            {"type": "image", "url": "https://raw.githubusercontent.com/google-gemma/cookbook/refs/heads/main/apps/sample-data/GoldenGate.png"},
            {"type": "text", "text": "What is shown in this image?"}
        ]
    }
]

# Process input
inputs = processor.apply_chat_template(
    messages,
    tokenize=True,
    return_dict=True,
    return_tensors="pt",
    add_generation_prompt=True,
    enable_thinking=False, # False True

).to(model.device)
input_len = inputs["input_ids"].shape[-1]

# Generate output
outputs = model.generate(**inputs, max_new_tokens=512, assistant_model=assistant_model) 
response = processor.decode(outputs[0][input_len:], skip_special_tokens=False)

# Parse output
processor.parse_response(response)

''' response
{'content': 'The Golden Gate Bridge, painted in its iconic International Orange, spans San Francisco Bay in California. In the foreground, a lone seagull perches atop a rocky outcrop rising from the water. The Alcatraz Island penitentiary is visible below the bridge.',
 'role': 'assistant'}
'''