Aditya02/IndicF5
4425
1o
2 3�gwB � @ s� d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dl m3Z4 d dl m Z d dl
mZ d dlmZmZ d dlmZmZmZ d d lmZ d d5lmZ d d lmZmZ d d l m Z m Z G d
d� d�Z dS )� )� annotationsN)� Accelerator)� DistributedDataParallelKwargs)�EMA)�AdamW)�LinearLR� SequentialLR)�6DataLoader�Dataset�SequentialSampler)�tqdm)�CFM)�DynamicBatchSampler�7collate_fn)�default�existsc @ sr e Zd Zdddddddddddd d8dd de� e� d d d d
fd0d!d"�Zed#d$� �Zd1d%d&�Zd'd(� Zd2d3d.d/�Z dS )4�Traineri N i� N� �sample� � �?�wandbz test_e2-ttsZtest_runF�vocos� �modelr
�batch_size_type�str�noise_scheduler�9str | None�duration_predictor�torch.nn.Module | None�logger�wandb_resume_id� log_samples�bool�accelerate_kwargs�dict�10ema_kwargs�
bnb_optimizer�
mel_spec_type�is_local_vocoder�local_vocoder_pathc C s� t dd�}|dkrtjjsd }td|� �� || _td|dkr!|nd |g|11d�|��| _|| _| jdkr_t |�rAdd||d�i}ndd|d�i}| jj12|||||||| |13| | jj | d �14d15� n| jd krsd d
l m
} | d|� �d�| _|| _| jr�t|fddi|��| _| j�| jj� || _|| _|| _t|||16 �| _t|d�| _|| _|| _ | | _ |17| _ | | _ || _ || _!|| _"| | _#|
| _$|r�d d l%} | j&j'|�(� |d�| _)n t*|�(� |d�| _)| j�+| j| j)�\| _| _)d S )NT)�find_unused_parametersr zUsing logger: )�log_with�kwargs_handlers�gradient_accumulation_steps�allow)�resume�name�id)r1 r2 )18�epochs�
learning_rate�num_warmup_updates�19batch_sizer � max_samples�grad_accumulation_steps�
max_grad_norm�gpusr )� project_name� init_kwargs�config� tensorboardr )�
SummaryWriterzruns/)�log_dirZinclude_online_modelFzckpts/test_e2-tts)�lr� ),r r �api�api_key�printr# r � acceleratorr! r �
init_trackers�
num_processesZtorch.utils.tensorboardr@ �writerr �is_mainr � ema_model�to�devicer4 r6 �save_per_updatesr �last_per_steps�checkpoint_pathr7 r r8 r9 r: � vocoder_namer* r+ r r � bitsandbytes�optimZ AdamW8bit�20parameters� optimizerr �prepare) �selfr r4 r5 r6 rO rQ r7 r r8 r9 r: r r r! Z
wandb_projectZwandb_run_namer" r# rP r% r' r( r) r* r+ Z21ddp_kwargsr= r@ �bnbrC rC �A/projects/data/ttsteam/repos/f5_hf/models/f5_tts/model/trainer.py�__init__ sv 22 ��23��24 zTrainer.__init__c C s | j jS )N)rG �is_main_process)rX rC rC rZ rK � s zTrainer.is_mainc C s� | j �� | jrXt| j �| j��� | j �| j��� | j�� | j �� |d�}t25j � | j
�s2t26�| j
� |rH| j �|| j
� d�� td|� �� d S | j �|| j
� d|� d�� d S d S )N)�model_state_dict�optimizer_state_dict�ema_model_state_dict�scheduler_state_dict�stepz/model_last.ptz Saved last checkpoint at step z/model_�.pt)rG �wait_for_everyonerK r& � unwrap_modelr �27state_dictrV rL � scheduler�os�pathr rQ �makedirs�saverF )rX ra �last�28checkpointrC rC rZ �save_checkpoint� s 29� �zTrainer.save_checkpointc C s� t | j�rtj� | j�rtdd� t�| j�D ��sdS | j�� dt�| j�v r+d}ntdd� t�| j�D �dd� d �d30 }t d | j� d |� �� t31j | j� d |� �d
dd�}dD ]
}||d v rf|d |= qY| j rr| j
�|d � d|v r�dD ]
}||d v r�|d |= qx| j�| j��|d � | j�| j��|d � | jr�| j�|d � ndd� |d �� D �|d< | j�| j��|d � d}~t�� |S )Nc s s � | ]}|� d �V qdS )rb N��endswith)�.0�filenamerC rC rZ � <genexpr>� s � z*Trainer.load_checkpoint.<locals>.<genexpr>r z
model_last.ptc S s g | ] }|� d �r|�qS )rb rn )rp �frC rC rZ �32<listcomp>� s z+Trainer.load_checkpoint.<locals>.<listcomp>c S s t d�ttj| ���S )Nr )�int�join�filterr �isdigit)�xrC rC rZ �<lambda>� s z)Trainer.load_checkpoint.<locals>.<lambda>)�key�����zLoading checkpoint from: �/T�cpu)� weights_only� map_location)z(ema_model.mel_spec.mel_stft.mel_scale.fbz.ema_model.mel_spec.mel_stft.spectrogram.windowr_ ra )z mel_spec.mel_stft.mel_scale.fbz$mel_spec.mel_stft.spectrogram.windowr] r^ r` c S s& i | ]\}}|d vr|� dd�|�qS ))�inittedra z33ema_model.r )�replace)rp �k�vrC rC rZ �34<dictcomp>� s35 �z+Trainer.load_checkpoint.<locals>.<dictcomp>)r rQ rg rh �any�listdirrG rc �sortedrF �torch�loadrK rL �load_state_dictrd r rV rf �items�gc�collect)rX Zlatest_checkpointrl r{ ra rC rC rZ �load_checkpoint� sT � ��36�� 37� 38��3940�zTrainer.load_checkpoint� �
train_datasetr41 �resumable_with_seedru c , C s< | j r/ddlm}m}m}m} || j| j| jd�}| j �42| j �j j
} | j� d�}43tj|44dd� t|�r=t�� } | �|� nd } | jdkrRt|t|dd| jd| d�} n+| jd krud45| j _t|�}
t|
| j| j |d46d �}t|t|dd|d �} nt d
| j� ���| j | j j }t d|� t!| �| j" | j# }|| }t$| j%dd|d�}t$| j%dd|d�}t&| j%||g|gd�| _'| j �(| | j'�\} | _'| �)� }|}t|�r�t!| �}t*|| �}|| }| j j+| |d�}nd}t,|| j"�D �]%}| j �-� t|��r||k�rt.|d|d � d| j"� �d| j j/ ||d�}nt.| d|d � d| j"� �d| j j/ d�}|D �]�} | j �0| j ��� | d } | d �1dd d�} | d } | j2d | j2d d k�rZ W d � �q(| j3d u�r{| j j/�r{| j3| | �4d �d �} | j j5d!| �6� i|d"� | j | | | | j7d#�\}!}"}#| j �8|!� | j9dk�r�| j j:�r�| j �;| j �<� | j9� | j%�=� | j'�=� | j%�>� W d � n 1 �s�w Y | j?�r�| j@�A� |d7 }| j j/�r| j j5|!�6� | j'�B� d d$�|d"� | jCd%k�r| jD�Ed&|!�6� |� | jD�Ed'| j'�B� d |� |jFtG|�|!�6� d(� || jH| j# dk�r�| �I|� | j �r�| j j/�r�| d }$| d tJ| d tK��r:d)gnd) | d g}%t�L� �x | j �47| j �jM| d d |$� �Nd�|%|$d |||d*�\}&}'|&�OtjP�}&|&d d �|$d �d d �f �1dd d��O| j jQ�}(| d d �Nd�})| jd+k�r�|�R|(��S� }*|�R|)��S� }+n| jd,k�r�||(��Td��S� }*||)��Td��S� }+W d � n 1 �s�w Y tU�V|48� d-|� d.�|*| � tU�V|49� d-|� d/�|+| � || jW dk�r�| jI|dd0� �q(t tjX�Y� d1 d2� t tjX�Z� d1 d3� tjX�[� t\�]� q�| jI|dd0� | j �^� d S )4Nr )� cfg_strength� load_vocoder�nfe_step�sway_sampling_coef)rR �is_local�50local_pathz/samplesT)�exist_okr )r � num_workers�51pin_memory�persistent_workersr7 �shuffle� generator�frameF)r8 � random_seed� drop_last)r r� r� r� �
batch_samplerzAbatch_size_type must be either 'sample' or 'frame', but received zWarm Up steps are: g:�0�yE>r )� start_factor�52end_factor� total_iters)�53schedulers�54milestones)� num_batcheszEpoch r r} ra )�desc�unit�disable�initial�total)r� r� r� �text�mel� � mel_lengthsip� � durations)�lensz
duration loss)ra )r� r� r )�lossrB r? r� rB )ra r� � )�condr� �duration�stepsr� r� r �bigvganz/step_z_gen.wavz_ref.wav)rk g e��Az GB allocatedz GB reserved)_r# �f5_tts.infer.utils_inferr� r� r� r� rR r* r+ rG rd r �mel_spec�target_sample_raterQ rg ri r r� � Generator� manual_seedr r r r7 � even_batchesr r r8 �55ValueErrorr6 rI rF �lenr4 r9 r rV r rf rW r� ru �skip_first_batches�range�trainr �is_local_main_process�56accumulate�permute�shaper �get�log�itemr �backwardr: �sync_gradients�clip_grad_norm_rU ra � zero_gradrK rL �update� get_last_lrr! rJ �57add_scalar� set_postfixr rO rm �58isinstance�list�inference_moder � unsqueezerM �float32rN �decoder~ �squeeze�59torchaudiorj rP �cuda�memory_allocated�memory_reserved� empty_cacher� r� � end_training),rX r� r� r� r� r� r� r� �vocoderr� Zlog_samples_pathr� Ztrain_dataloader�samplerr� � warmup_steps� total_stepsZ decay_stepsZwarmup_schedulerZdecay_scheduler�60start_step� global_stepZorig_epoch_stepZ
skipped_epochZ
skipped_batchZskipped_dataloader�epoch� progress_bar�batch� text_inputsr� r� Zdur_lossr� r� �pred�
ref_audio_lenZ61infer_text� generated�_Z gen_mel_specZ ref_mel_specZ gen_audio� ref_audiorC rC rZ r� � s � 62�63 64�� �65�66� 67� �68�69 � 7071 �7273$ 74(�7576� . � ��7778z
Trainer.train)r r
r r r r r r r! r r" r r# r$ r% r& r' r& r( r$ r) r r* r$ r+ r )F)r� N)r� r79 r� ru )80�__name__�81__module__� __qualname__r& r[ �propertyrK rm r� r� rC rC rC rZ r s: �j82836r ) �84__future__r r� rg r� r� r �85accelerater �accelerate.utilsr Z ema_pytorchr � torch.optimr Ztorch.optim.lr_schedulerr r �torch.utils.datar r86 r r � f5_tts.modelr
Zf5_tts.model.datasetr r �f5_tts.model.utilsr r r rC rC rC rZ �<module> s"