LanguageBind/Video-LLaVA

1 2DATA_ROOT="llava_all_image_video"3CUDA_VISIBLE_DEVICES=0,1,2,3 deepspeed llava/train/train_mem.py \4    --deepspeed ./scripts/zero2.json \5    --model_name_or_path lmsys/vicuna-7b-v1.5 \6    --version plain \7    --data_path train_json/valley_llavaimage.json \8    --video_folder ${DATA_ROOT} \9    --image_folder ${DATA_ROOT} \10    --X "Video" "Image" \11    --video_tower LanguageBind/LanguageBind_Video \12    --image_tower LanguageBind/LanguageBind_Image \13    --mm_projector_type mlp2x_gelu \14    --tune_mm_mlp_adapter True \15    --mm_vision_select_layer -2 \16    --mm_use_x_start_end False \17    --mm_use_x_patch_token False \18    --bf16 True \19    --output_dir ./checkpoints/Video-LLaVA-Pretrain-7B \20    --num_train_epochs 1 \21    --per_device_train_batch_size 32 \22    --per_device_eval_batch_size 4 \23    --gradient_accumulation_steps 2 \24    --evaluation_strategy "no" \25    --save_strategy "steps" \26    --save_steps 24000 \27    --save_total_limit 1 \28    --learning_rate 1e-3 \29    --weight_decay 0. \30    --warmup_ratio 0.03 \31    --lr_scheduler_type "cosine" \32    --logging_steps 1 \33    --tf32 True \34    --model_max_length 2048 \35    --gradient_checkpointing True \36    --dataloader_num_workers 8 \37    --lazy_preprocess True \38    --report_to tensorboard \39    --cache_dir "./cache_dir"40