Doubiiu/DynamiCrafter_interp_loop
165
1a
2 �fe! � @ s� d dl Z d dlmZ d dlZd dlZd dlmZ d dlm m3Z d dl Z
d dlmZmZ d dlmZ d dlmZ G dd� de
j�ZG d d4� d5ej j�ZdS ) � N)�contextmanager)� rearrange)�Encoder�Decoder)� DiagonalGaussianDistribution)�instantiate_from_configc s� e Zd Zdg dddddddf � fdd� Zdd� Ze� fd d6�Zd d � Zd
d� Zd dd�Z dd� Z7dd� Z dd� Z dd� Z
dd� Ze�� d!d d ��Zd d � Z� ZS )"�
AutoencoderKLN�imageF� c
s� t � �� || _tf i |��| _tf i |��| _t|�| _|d sFJ �t j8� d|d d| d�| _ t j9� ||d d�| _
|| _| | _| | _| | _|10| _|d ur�t|�tks�J �| �dt �d|dd�� |d ur�|| _|d ur�| j||d� | jr�| �� d S )NZdouble_z� Z11z_channels� �colorize� )� ignore_keys)�super�__init__� image_keyr �encoderr �decoderr �loss�torch�nn�Conv2d�12quant_conv�post_quant_conv� embed_dim� input_dim�test� test_args�logdir�type�int�register_buffer�randn�monitor�init_from_ckpt� init_test)
�self�ddconfigZ13lossconfigr � ckpt_pathr r Zcolorize_nlabelsr$ r r r r �� __class__� �t/apdcephfs_cq2/share_1290939/jinboxing/Experiments/DynamiCrafter/scripts/evaluation/../../lvdm/models/autoencoder.pyr s, 1415 zAutoencoderKL.__init__c C sH d| _ tj�| jd�}d| jv rXtj�| jj��d�d d| j � � }tj�||�| _16n|| _17d| jv r|tj�|| jj �| _18tj�| j19d�| _ tj�| j20d �| _
tj�| j21d22�| _tj| j23dd � | jjr�tj| j dd � | jjr�tj| j
dd � | jj�r tj| jdd � | jd u�s J �t| jd d �| _d| _i | _g | _d
| _d S )NTr �ckptz.ckptr �_epoch� test_subdir�zs�reconstructions�inputs)�exist_ok� test_maximumi )r �os�path�joinr r �basenamer. �split�24_cur_epoch�rootr0 Zroot_zsZroot_decZ root_inputs�makedirsZsave_zZsave_reconstructionZ25save_input�getattrr5 �countZ eval_metricsZdecodesZsave_decode_samples)r' �save_dir� ckpt_namer, r, r- r& 3 s0 26&2728zAutoencoderKL.init_testc C s� t j|dd�}z|d | _|d }W n d| _Y n0 t|�� �}|D ],}|D ]"}|�|�rPtd�|�� ||= qPqH| j|dd� td |� �� d S )29N�cpu)� map_location�epoch�30state_dict�nullz Deleting key {} from state_dict.F)�strictzRestored from ) r �loadr; �list�keys�31startswith�print�format�load_state_dict)r' r7 r �sdrJ �k�ikr, r, r- r% P s 32 3334z AutoencoderKL.init_from_ckptc K s | � |�}| �|�}t|�}|S �N)r r r )r' �x�kwargs�h�moments� posteriorr, r, r- �encodea s 3536zAutoencoderKL.encodec K s | � |�}| �|�}|S rR )r r )r' �zrT �decr, r, r- �decodeh s 3738zAutoencoderKL.decodeTc C s2 | � |�}|r|�� }n|�� }| �|�}||fS rR )rX �sample�moder[ )r' �inputZsample_posteriorrW rY rZ r, r, r- �forwardm s 394041zAutoencoderKL.forwardc C sH || }|� � dkrD| jdkrD|j\}}}}}|| _|| _t|d�}|S )N� r42 zb c t h w -> (b t) c h w)�dimr �shape�b�tr ) r' �batchrP rS rc �crd rU �wr, r, r- � get_inputv s 43zAutoencoderKL.get_inputc C s� | � || j�}| |�\}}|dkrr| j||||| j| �� dd�\}}| jd|ddddd� | j|ddddd� |S |dkr�| j||||| j| �� dd�\} }44| jd | ddddd� | j|45ddddd� | S d S )46Nr �train�Z47last_layerr: �aelossT)�prog_bar�logger�on_step�on_epochFr �discloss�rh r r � global_step�get_last_layer�log�log_dict) r' re � batch_idx�
optimizer_idxr3 r2 rW rk � log_dict_aerp �
log_dict_discr, r, r- �
training_step� s �48�49zAutoencoderKL.training_stepc 50 C s� | � || j�}| |�\}}| j|||d| j| �� dd�\}}| j|||d| j| �� dd�\}} | �d|d � | �|� | �| � | jS )Nr �valrj r z val/rec_lossrq )51r' re rv r3 r2 rW rk rx rp ry r, r, r- �validation_step� s �52�535455z AutoencoderKL.validation_stepc C sr | j }tjjt| j�� �t| j�� � t| j�� � t| j �� � |dd�}tjj| j56j �� |dd�}||gg fS )N)g �?g�������?)�lr�betas) �
learning_rater �optim�AdamrI r �57parametersr r r r Z
discriminator)r' r} Zopt_aeZopt_discr, r, r- �configure_optimizers� s � � ���z"AutoencoderKL.configure_optimizersc C s58 | j jjS rR )r �conv_out�weight)r' r, r, r- rs � s z AutoencoderKL.get_last_layerc K s� t � }| �|| j�}|�| j�}|s�| |�\}}|jd dkrd|jd dksPJ �| �|�}| �|�}| �t� |�59� ��|d< ||d< ||d< |S )Nr r �samplesr2 r3 ) �dictrh r �to�devicerb �to_rgbr[ r �60randn_liker\ )r' re � only_inputsrT rt rS ZxrecrW r, r, r- �61log_images� s 6263zAutoencoderKL.log_imagesc C st | j dksJ �t| d�s<| �dt�d|jd dd��|�� tj|| j d�}d||�64� |� � |�65� d }|S )N� segmentationr
r r )r� g @g �?) r �hasattrr" r r# rb r� �F�conv2dr
�min�max)r' rS r, r, r- r� � s 66$$zAutoencoderKL.to_rgb)T)F)�__name__�67__module__� __qualname__r r& rI r% rX r[ r_ rh rz r| r� rs r �no_gradr� r� �
__classcell__r, r, r* r- r
s. �% 68 69 r c sB e Zd Zdd�� fdd�70Zdd� Zdd� Zd d71� Zd d � Z� ZS )
�IdentityFirstStageF�� vq_interfacec s || _ t� �� d S rR )r� r r )r' r� �argsrT r* r, r- r � s zIdentityFirstStage.__init__c O s |S rR r, �r' rS r� rT r, r, r- rX � s zIdentityFirstStage.encodec O s |S rR r, r� r, r, r- r[ � s zIdentityFirstStage.decodec O s | j r|d g d�fS |S )N)NNNr� r� r, r, r- �quantize� s zIdentityFirstStage.quantizec O s |S rR r, r� r, r, r- r_ � s zIdentityFirstStage.forward) r� r� r� r rX r[ r� r_ r� r, r, r* r- r� � s72 r� )r6 �73contextlibr r �numpy�np�einopsr Ztorch.nn.functionalr �74functionalr� �pytorch_lightning�plZ lvdm.modules.networks.ae_modulesr r �lvdm.distributionsr � utils.utilsr �LightningModuler �Moduler� r, r, r, r- �<module> s >