shahedm2001/qwen3-vl-2b-cataract-sft

.venv/bin/python -u src/train/train_sft.py --model_id Qwen/Qwen3-VL-2B-Instruct \
  --data_path data/sft_train_dataset_sft.json --eval_path data/sft_val_dataset_sft.json \
  --output_dir output/sft_lora --bits 4 --lora_enable True --vision_lora True --use_dora False \
  --lora_rank 16 --lora_alpha 32 --lora_dropout 0.05 --num_lora_modules -1 \
  --lora_namespan_exclude "['lm_head','embed_tokens','merger','pos_embed']" \
  --freeze_vision_tower True --freeze_llm True --freeze_merger False \
  --bf16 True --fp16 False --tf32 True --disable_flash_attn2 True --use_liger_kernel True \
  --num_train_epochs 2 --per_device_train_batch_size 4 --gradient_accumulation_steps 4 \
  --learning_rate 1e-4 --vision_lr 2e-6 --merger_lr 1e-5 --weight_decay 0.1 --warmup_steps 10 \
  --lr_scheduler_type cosine --video_min_pixels 98304 --video_max_pixels 196608 --nframes 64 \
  --max_seq_length 32768 --gradient_checkpointing True --image_folder dataset_sft

git clone https://github.com/shahedmomenzadeh/qwen3-VL-2B-finetune.git
cd qwen3-VL-2B-finetune
bash setup.sh  # uv + .venv (python 3.12)
source .venv/bin/activate
export HF_HOME=$PWD/hf_cache PYTHONPATH=src

import torch
from transformers import AutoProcessor, AutoModelForImageTextToText
from qwen_vl_utils import process_vision_info

model_id = "shahedm2001/qwen3-vl-2b-cataract-sft"  # <-- update after publish
processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
model = AutoModelForImageTextToText.from_pretrained(
    model_id, dtype=torch.bfloat16, device_map="auto",
    attn_implementation="sdpa", trust_remote_code=True)
model.eval()

video = "path/to/clip.mp4"
question = "Describe what is happening in this cataract surgical video clip."

# --- choose ONE of nframes / fps, plus desired pixel bounds ---
messages = [{"role": "user", "content": [
    {"type": "video", "video": video,
     "nframes": 64,                 # ← max frames; use one of nframes/fps
     # "fps": 1.0,                  # ← alternative: frames per second
     "min_pixels": 98_304,          # 96*32*32 — lower bound
     "max_pixels": 196_608},        # 192*32*32 — upper bound (increase for more detail / more tokens)
    {"type": "text", "text": question}
]}]

text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
image_inputs, video_inputs, video_kwargs = process_vision_info(
    messages, return_video_kwargs=True)
video_kwargs = {k: v for k, v in (video_kwargs or {}).items() if k != "fps"}
inputs = processor(text=[text], images=image_inputs, videos=video_inputs,
                   padding=True, return_tensors="pt", **video_kwargs).to(model.device)

with torch.no_grad():
    out = model.generate(**inputs, max_new_tokens=256, do_sample=False)
pred = processor.tokenizer.decode(out[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True)
print(pred)

@misc{qwen3vl2b-cataract-sft-2026,
  title  = {Qwen3-VL-2B Cataract Surgery SFT},
  author = {Momenzadeh, Shahed},
  howpublished = {Hugging Face},
  year   = {2026},
  note   = {Merged full-precision SFT checkpoint, QLoRA r16 on dataset_sft}
}