Yash1608/qwen2vl-2b-chartqa-merged

Input:  [bar chart image] + "What is the value for category B in 2022?"
Output: "47.3"

.
├── multimodal_finetune_chartqa_Collab_v2_Final.ipynb   # Main notebook (training + eval + inference)
├── inference.py                                        # Standalone inference & batch-eval script
├── push_to_hub.py                                      # Helper: push adapters + merged model to HF Hub
├── requirements.txt                                    # Pinned dependencies
└── README.md

pip install transformers==4.49.0 peft==0.14.0 accelerate==1.4.0 \
            qwen-vl-utils pillow torch

from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
from qwen_vl_utils import process_vision_info
from PIL import Image
import torch

# 1. Pull the merged (full) model — no adapter handling needed
model = Qwen2VLForConditionalGeneration.from_pretrained(
    "Yash1608/qwen2vl-2b-chartqa-merged",
    torch_dtype=torch.float16,  # FP16: T4 Turing (SM 7.5) has native FP16, no BF16 tensor cores
    device_map="auto",
    trust_remote_code=True,
)
processor = AutoProcessor.from_pretrained(
    "Yash1608/qwen2vl-2b-chartqa-merged",
    trust_remote_code=True,
)
model.eval()

# 2. Prepare input
image = Image.open("chart.png")         # your chart image
question = "What is the highest value shown?"

messages = [{
    "role": "user",
    "content": [
        {"type": "image",  "image": image},
        {"type": "text",   "text": f"Analyze the chart and answer concisely.\n\nQuestion: {question}"},
    ],
}]

# 3. Run inference
text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
image_inputs, _ = process_vision_info(messages)
inputs = processor(text=[text], images=[image_inputs], return_tensors="pt").to("cuda")

with torch.no_grad():
    gen = model.generate(**inputs, max_new_tokens=32, do_sample=False)

answer = processor.decode(gen[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True)
print("Answer:", answer)

from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
from peft import PeftModel
import torch

# Step 1: load base model
base = Qwen2VLForConditionalGeneration.from_pretrained(
    "Qwen/Qwen2-VL-2B-Instruct",
    torch_dtype=torch.float16,  # FP16 for T4 GPU
    device_map="auto",
    trust_remote_code=True,
)

# Step 2: attach LoRA adapters
model = PeftModel.from_pretrained(base, "Yash1608/qwen2vl-2b-chartqa")

# Step 3: merge adapters into base weights (optional but recommended for deployment)
model = model.merge_and_unload()

# Processor from either the adapter repo or the base model
processor = AutoProcessor.from_pretrained("Yash1608/qwen2vl-2b-chartqa", trust_remote_code=True)

# Push both LoRA adapters AND the merged full model (recommended)
python push_to_hub.py --hf_username Yash1608 --repo_name qwen2vl-2b-chartqa

# Push adapters only (smaller upload, ~100–400 MB)
python push_to_hub.py --hf_username Yash1608 --repo_name qwen2vl-2b-chartqa --skip_merged

# Push merged full model only
python push_to_hub.py --hf_username Yash1608 --repo_name qwen2vl-2b-chartqa --skip_adapters

# Evaluate merged model on the test split (500 samples)
python inference.py --evaluate --eval_split test --eval_samples 500

# Evaluate via adapter load + merge on the val split
python inference.py --evaluate --use_adapters --eval_split val --eval_samples 200

from inference import relaxed_accuracy, load_merged_model

model, processor = load_merged_model("Yash1608/qwen2vl-2b-chartqa-merged")
acc = relaxed_accuracy(model, processor, split="test", n_samples=500)
print(f"Relaxed Accuracy: {acc:.4f}")