XaviXva/Video-LLaVA
0
1o
2 ^I7e�* � @ s� d dl Z d dl mZ d dlmZ ddlmZ ddlmZ ddlm Z ddl3m Z dd l m
Z
dd4lmZ dd lmZ dd lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddl m Z ddl m Z ddl m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+ e%ee
ee d�Z,e'eeee d�Z-ee#ee+e d�Z.G dd � d ej/�Z0d d � Z1G d d � d ej/�Z2G d!d"� d"ej/�Z3G d#d$� d$ej/�Z4dS )%� N)�nn)�5AutoConfig� )�LanguageBindImageConfig)�LanguageBindImage)�LanguageBindImageTokenizer)�LanguageBindImageProcessor)�LanguageBindVideoConfig)�LanguageBindVideo)�LanguageBindVideoTokenizer)�LanguageBindVideoProcessor)�LanguageBindDepthConfig)�LanguageBindDepth)�LanguageBindDepthTokenizer)�LanguageBindDepthProcessor)�LanguageBindAudioConfig)�LanguageBindAudio)�LanguageBindAudioTokenizer)�LanguageBindAudioProcessor)�LanguageBindThermalConfig)�LanguageBindThermal)� LanguageBindThermalTokenizer)� LanguageBindThermalProcessor��thermal�image�video�depth�audio)r r r r r c s& e Zd Zd� fdd� Zdd� Z� ZS ) � LanguageBindr T� ./cache_dirc s� t t| ��� || _i | _i | _i | _i | _|D ]*}d|�� � �}t | j6||d�}|j | j|< |j | j|< |j
| j|< |j| j|< q|j| jd< |j| jd< t�| j�| _t�| j�| _d S )NzLanguageBind/LanguageBind_�� cache_dir�language)�superr �__init__�use_temp�modality_encoder�
modality_proj�modality_scaleZmodality_config�7capitalize�8model_dict�from_pretrained� vision_model�visual_projection� logit_scale�config�9text_model�text_projectionr �10ModuleDict)�selfZ clip_typer& r" �cZpretrained_ckpt�model�� __class__� �FD:\LLaVA-Video\llava\model\multimodal_encoder\languagebind\__init__.pyr% 7 s" zLanguageBind.__init__c C s| i }|� � D ]5\}}| j| di |��d }| j| |�}||jdddd� }| jr7|dkr7|| j| �� }|||< q|S )Nr � �����T)�p�dim�keepdimr# r9 )�itemsr' r( �normr& r) �exp)r4 �inputs�outputs�key�valuer9 r9 r: �forwardK s 11zLanguageBind.forward)r Tr ��__name__�12__module__� __qualname__r% rG �
__classcell__r9 r9 r7 r: r 6 s r c s � fdd�| � � D �}|S )Nc s i | ]13\}}||� � ��qS r9 )�to)�.0�k�v��devicer9 r: �14<dictcomp>X s z to_device.<locals>.<dictcomp>)r@ )�xrR Zout_dictr9 rQ r: � to_deviceW s rU c � e Zd Zd� fdd� Zdd� Zdd� Ze�� d d15� �Ze d d � �Z16e d
d� �Z e dd� �Z e dd� �Z
e dd� �Ze dd� �Z� ZS )�LanguageBindImageTowerFr c �Z t � �� d| _|| _|j| _t|dd�| _|| _|s!| � � d S t17j | j| jd�| _ d S �NF�mm_vision_select_feature�patchr! )
r$ r% � is_loaded�image_tower_name�mm_vision_select_layer� select_layer�getattr�select_featurer" �18load_modelr r, �cfg_only)r4 � image_tower�args�19delay_loadr" r7 r9 r: r% _ � 20 z LanguageBindImageTower.__init__c C s< t j| j| jd�}|j| _| j�d� t|j�| _ d| _21d S �Nr! FT) r r, r] r" r- rd �requires_grad_r r0 �image_processorr\ �r4 r6 r9 r9 r: rb p s22 23z!LanguageBindImageTower.load_modelc C sP |j | j }| jdkr|d d �dd �f }|S | jdkr |}|S td| j� ���)Nr[ r � cls_patch�Unexpected select feature: )�
hidden_statesr_ ra �24ValueError)r4 �image_forward_outs�image_featuresr9 r9 r: �feature_selecty s 2526��z%LanguageBindImageTower.feature_selectc C � t |�tu r.g }|D ]!}| j|j| j| jd��d�dd�}| �|��|j�}|�|� q27|S | j|j| j| jd�dd�}| �|��|j�}|S �N�rR �dtyper T)�output_hidden_states) �type�listrd rM rR rv � unsqueezerr �append)r4 �imagesrq r �image_forward_out�
image_featurerp r9 r9 r: rG � s " �z LanguageBindImageTower.forwardc C � t jd| j| j| jd�S �Nr ru ��torch�zeros� hidden_sizerR rv �r4 r9 r9 r: �
dummy_feature� � z$LanguageBindImageTower.dummy_featurec C � | j jjjS �N)rd �28embeddings�class_embeddingrv r� r9 r9 r: rv � � z LanguageBindImageTower.dtypec C r� r� )rd r� r� rR r� r9 r9 r: rR � r� z LanguageBindImageTower.devicec C � | j r| jjS | jS r� )r\ rd r0 rc r� r9 r9 r: r0 � � z LanguageBindImageTower.configc C � | j jS r� �r0 r� r� r9 r9 r: r� � � z"LanguageBindImageTower.hidden_sizec C � | j j| j j d S �Nr; �r0 �29image_size�30patch_sizer� r9 r9 r: � num_patches� � z"LanguageBindImageTower.num_patches�Fr �rI rJ rK r% rb rr r� �no_gradrG �propertyr� rv rR r0 r� r� rL r9 r9 r7 r: rW ^ s$ 31 323334353637rW c s$ e Zd Z� fdd�Zdd� Z� ZS )�38temp_modelc s t t| ��� d S r� )r$ r� r% r� r7 r9 r: r% � s ztemp_model.__init__c K s t �dddd�S )N� r � i )r� �randn)r4 �kwargsr9 r9 r: rG � s ztemp_model.forwardrH r9 r9 r7 r: r� � s r� c rV )�LanguageBindVideoTowerFr c rX rY )
r$ r% r\ �video_tower_namer^ r_ r` ra r" rb r r, rc )r4 � video_towerre rf r" r7 r9 r: r% � rg z LanguageBindVideoTower.__init__c C s< t j| j| jd�}t|j�| _|j| _| j� d� d| _39d S rh ) r40 r, r� r" r r0 Zvideo_processorr- r� ri r\ rk r9 r9 r: rb � s41 42z!LanguageBindVideoTower.load_modelc C s| |j | j }|j\}}}}| jdkr(|d d �d d �dd �f }|�|d|�}|S | jdkr6|�|d|�}|S td| j� ���)Nr[ r r< rl rm )rn r_ �shapera �reshapero )r4 �video_forward_outs�video_features�b�t�nr5 r9 r9 r: rr � s 4344��z%LanguageBindVideoTower.feature_selectc C rs rt ) rx ry r� rM rR rv rz rr r{ )r4 �videosr� r Zvideo_forward_outZ
video_featurer� r9 r9 r: rG � s " �z LanguageBindVideoTower.forwardc C r r� r� r� r9 r9 r: r� � r� z$LanguageBindVideoTower.dummy_featurec C r� r� )r� r� r� rv r� r9 r9 r: rv r� z LanguageBindVideoTower.dtypec C r� r� )r� r� r� rR r� r9 r9 r: rR r� z LanguageBindVideoTower.devicec C r� r� )r\ r� r0 rc r� r9 r9 r: r0 r� z LanguageBindVideoTower.configc C r� r� r� r� r9 r9 r: r� r� z"LanguageBindVideoTower.hidden_sizec C r� r� r� r� r9 r9 r: r� r� z"LanguageBindVideoTower.num_patchesr� r� r9 r9 r7 r: r� � s$ 454647484950r� )5r� r � transformersr Zimage.configuration_imager Zimage.modeling_imager Zimage.tokenization_imager Zimage.processing_imager Zvideo.configuration_videor Zvideo.modeling_videor51 Zvideo.tokenization_videor Zvideo.processing_videor Zdepth.configuration_depthr
Zdepth.modeling_depthr Zdepth.tokenization_depthr Zdepth.processing_depthr Zaudio.configuration_audior Zaudio.modeling_audior Zaudio.tokenization_audior Zaudio.processing_audior Z thermal.configuration_thermalr Zthermal.modeling_thermalr Z thermal.tokenization_thermalr Zthermal.processing_thermalr � config_dictr+ Ztransform_dict�Moduler rU rW r� r� r9 r9 r9 r: �<module> s\ ���!Q