simplecloud/VidChain-exercise
✏️ Data for VidChain Excercise VidChain: Chain-of-Tasks with Metric-based Direct Preference Optimization for Dense Video Captioning Ji Soo Lee*, Jongha Kim*, Jeehye Na, Jinyoung Park, Hyunwoo J. Kim†. AAAI 2025 🎯 Learning Objectives By working through this exercise, you will: Reproduce baseline behavior of a video-language model (VTimeLLM, CVPR 2024 Highlight). Observe the limitations of existing approaches in temporal… See the full description on the dataset page: https://huggingface.co/datasets/simplecloud/VidChain-exercise.
0150
1import torch2import torch.nn as nn3from typing import List, Optional, Tuple, Union4from transformers import AutoConfig, AutoModelForCausalLM5from .chatglm import ChatGLMConfig, ChatGLMModel, ChatGLMForConditionalGeneration6from .vtimellm_arch import VTimeLLMMetaModel, VTimeLLMMetaForCausalLM7 8class VTimeLLMChatGLMConfig(ChatGLMConfig):9 model_type = "VTimeLLM_ChatGLM"10 11class VTimeLLMChatGLMModel(ChatGLMModel, VTimeLLMMetaModel):12 config_class = VTimeLLMChatGLMConfig13 14 def __init__(self, config, empty_init=True, device=None):15 super(VTimeLLMChatGLMModel, self).__init__(config, empty_init=empty_init, device=device)16 17class VTimeLLMChatGLMForCausalLM(ChatGLMForConditionalGeneration, VTimeLLMMetaForCausalLM):18 config_class = VTimeLLMChatGLMConfig19 20 def __init__(self, config, empty_init=True, device=None):21 super(ChatGLMForConditionalGeneration, self).__init__(config)22 self.transformer = VTimeLLMChatGLMModel(config, empty_init=empty_init, device=device)23 self.max_sequence_length = config.max_length24 self.config = config25 self.quantized = False26 # Initialize weights and apply final processing27 self.post_init()28 29 def get_model(self):30 return self.transformer31 32 def forward(33 self,34 input_ids: torch.LongTensor = None,35 position_ids: Optional[torch.LongTensor] = None,36 attention_mask: Optional[torch.Tensor] = None,37 past_key_values: Optional[List[torch.FloatTensor]] = None,38 inputs_embeds: Optional[torch.FloatTensor] = None,39 labels: Optional[torch.LongTensor] = None,40 use_cache: Optional[bool] = None,41 output_attentions: Optional[bool] = None,42 output_hidden_states: Optional[bool] = None,43 return_dict: Optional[bool] = None,44 return_last_logit: Optional[bool] = False,45 images: Optional[torch.FloatTensor] = None,46 ):47 48 if inputs_embeds is None:49 (50 input_ids,51 position_ids,52 attention_mask,53 past_key_values,54 inputs_embeds,55 labels56 ) = self.prepare_inputs_labels_for_multimodal(57 input_ids,58 position_ids,59 attention_mask,60 past_key_values,61 labels,62 images63 )64 65 return super().forward(66 input_ids=input_ids,67 attention_mask=attention_mask,68 position_ids=position_ids,69 past_key_values=past_key_values,70 inputs_embeds=inputs_embeds,71 labels=labels,72 use_cache=use_cache,73 output_attentions=output_attentions,74 output_hidden_states=output_hidden_states,75 return_dict=return_dict76 )77 78 def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):79 images = kwargs.pop("images", None)80 _inputs = super().prepare_inputs_for_generation(81 input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs82 )83 if images is not None:84 _inputs['images'] = images85 return _inputs86 87AutoConfig.register("VTimeLLM_ChatGLM", VTimeLLMChatGLMConfig)88AutoModelForCausalLM.register(VTimeLLMChatGLMConfig, VTimeLLMChatGLMForCausalLM)89