WishArdently/InternVideo2Stage2-VisionEncoder

1{2  "architectures": [3    "InternVideo2Stage2VideoEncoder"4  ],5  "auto_map": {6    "AutoConfig": "config.InternVideo2Config",7    "AutoModel": "model.InternVideo2Stage2VideoEncoder"8  },9  "auto_resume": false,10  "batch_size": 64,11  "batch_size_test": 4,12  "best_key": [13    "msrvtt_1k_test_match",14    "t2v_r1"15  ],16  "compile_model": false,17  "criterion": {18    "clip_loss_ratio": [19      1.0,20      1.021    ],22    "distill_final_features": true,23    "loss_weight": {24      "mlm": 1.0,25      "mvm": 0.0,26      "uta": 0.0,27      "vtc": 1.0,28      "vtm": 1.029    },30    "mlm_masking_prob": 0.5,31    "vtm_hard_neg": true32  },33  "debug": false,34  "deep_fusion": false,35  "deepspeed": {36    "enable": true,37    "stage": 138  },39  "delete_ds_optim_states": true,40  "device": "cuda",41  "dist_url": "env://",42  "evaluate": false,43  "evaluation": {44    "eval_frame_ensemble": "concat",45    "eval_offload": true,46    "eval_x_only": false,47    "k_test": 12848  },49  "gradient_checkpointing": true,50  "inputs": {51    "batch_size": {52      "image": 64,53      "video": 6454    },55    "batch_size_test": {56      "image": 4,57      "video": 458    },59    "image_res": 224,60    "max_txt_l": {61      "image": 32,62      "video": 3263    },64    "video_input": {65      "num_frames": 8,66      "num_frames_test": 8,67      "random_aug": false,68      "sample_type": "rand",69      "sample_type_test": "middle"70    }71  },72  "jump_evaluate": false,73  "log_freq": 100,74  "max_txt_l": 32,75  "mode": "pt",76  "model": {77    "embed_dim": 512,78    "find_unused_parameters": false,79    "model_cls": "InternVideo2_Stage2",80    "multimodal": {81      "enable": true82    },83    "temp": 0.07,84    "text_encoder": "bert_large",85    "vision_encoder": {86      "checkpoint_num": 40,87      "clip_embed_dim": 768,88      "clip_input_resolution": 224,89      "clip_norm_type": "l2",90      "clip_return_layer": 6,91      "clip_student_return_interval": 1,92      "clip_teacher": null,93      "clip_teacher_embed_dim": 3200,94      "clip_teacher_final_dim": 768,95      "clip_teacher_return_interval": 1,96      "d_model": 1408,97      "image_mask_ratio": 0.5,98      "image_mask_type": "random",99      "img_size": 224,100      "keep_temporal": false,101      "name": "pretrain_internvideo2_1b_patch14_224",102      "num_frames": 8,103      "only_mask": true,104      "patch_size": 14,105      "pretrained": "/home/linanxi/InternVideo/checkpoints/InternVideo2-stage2_1b-224p-f4/InternVideo2-stage2_1b-224p-f4.pt",106      "sep_image_video_pos_embed": true,107      "tubelet_size": 1,108      "use_checkpoint": false,109      "use_flash_attn": true,110      "use_fused_mlp": true,111      "use_fused_rmsnorm": true,112      "video_mask_ratio": 0.8,113      "video_mask_type": "random"114    }115  },116  "model_type": "internvideo2",117  "num_frames": 8,118  "num_frames_test": 8,119  "num_workers": 6,120  "optimizer": {121    "different_lr": {122      "enable": false,123      "lr": 0.001,124      "module_names": []125    },126    "lr": 5e-05,127    "max_grad_norm": 3.0,128    "opt": "adamW",129    "opt_betas": [130      0.9,131      0.98132    ],133    "weight_decay": 0.05134  },135  "output_dir": null,136  "pretrained_path": "",137  "resume": false,138  "save_ckpt_iter": null,139  "save_latest": true,140  "scheduler": {141    "epochs": 10,142    "min_lr_multi": 0.01,143    "sched": "cosine",144    "warmup_epochs": 1145  },146  "seed": 42,147  "test_file": {148    "didemo_ret_test": "available_corpus[\"didemo_ret_test\"]",149    "msrvtt_1k_test": "available_corpus[\"msrvtt_1k_test\"]"150  },151  "test_types": [152    "msrvtt_1k_test",153    "didemo_ret_test"154  ],155  "text_enc": "bert_large",156  "tokenizer": null,157  "torch_dtype": "float16",158  "train_file": "available_corpus[\"pretrain_example_data_1B\"]",159  "transformers_version": "4.47.0",160  "use_bf16": true,161  "use_flash_sdp": false,162  "use_half_precision": false,163  "use_mem_efficient_sdp": false,164  "wandb": {165    "enable": false,166    "entity": "opengvlab",167    "project": "InternVideo2-Stage2"168  }169}170