RASMUS/Finnish-ASR-Canary-v2

1model:2  # Every name/path here starting with 'pretrained' is used to initialize the model weights.3  pretrained_llm: TinyLlama/TinyLlama_v1.14  pretrained_audio_codec: ???  # to be released5  pretrained_asr: stt_en_fastconformer_hybrid_large_streaming_80ms6  scoring_asr: stt_en_fastconformer_transducer_large  # used only in validation/evaluation7 8  pretrained_weights: True  # When False, we use pretrained_name to load the architecture, but with random init9 10  # Regexp (re.compile) patterns matching parameters to be frozen.11  freeze_params:12    - "^audio_codec\\..+$"  # Keep audio codec frozen as it only provides supervision for training.13  prevent_freeze_params: []  # Use to make specific submodules trainable; overrides freeze_params14 15  audio_loss_weight: 416  text_loss_weight: 317 18  # Note: Uncomment the block below to enable LoRA on LLM via HuggingFace PEFT library.19  #   It will automatically freeze LLM parameters even if freeze_params was unused,20  #   and prevent freezing any parameter that has the string '.lora_' in its name.21  # lora:22  #   task_type: CAUSAL_LM23  #   r: 824  #   lora_alpha: 3225  #   lora_dropout: 0.126 27  perception:28     target:  nemo.collections.speechlm2.modules.perception.AudioPerceptionModule29     modality_adapter:30       _target_: nemo.collections.asr.modules.ConformerEncoder31       feat_in: 51232       feat_out: -1 # you may set it if you need different output size other than the default d_model33       n_layers: 234       d_model: 51235       subsampling: dw_striding # vggnet, striding, stacking or stacking_norm, dw_striding36       subsampling_factor: 1 # must be power of 2 for striding and vggnet37       subsampling_conv_channels: 256 # set to -1 to make it equal to the d_model38       causal_downsampling: true39       ff_expansion_factor: 440       self_attention_model: rel_pos # rel_pos or abs_pos41       n_heads: 8 # may need to be lower for smaller d_models42       # [left, right] specifies the number of steps to be seen from left and right of each step in self-attention43       att_context_size: [70, 1] # -1 means unlimited context44       att_context_style: chunked_limited # regular or chunked_limited45       xscaling: true # scales up the input embeddings by sqrt(d_model)46       untie_biases: true # unties the biases of the TransformerXL layers47       pos_emb_max_len: 500048       conv_kernel_size: 949       conv_norm_type: layer_norm # batch_norm or layer_norm or groupnormN (N specifies the number of groups)50       # conv_context_size can be"causal" or a list of two integers while conv_context_size[0]+conv_context_size[1]+1==conv_kernel_size51       # null means [(kernel_size-1)//2, (kernel_size-1)//2], and 'causal' means [(kernel_size-1), 0]52       conv_context_size: causal53       ### regularization54       dropout: 0 # The dropout used in most of the Conformer Modules55       dropout_pre_encoder: 0 # The dropout used before the encoder56       dropout_emb: 0.0 # The dropout used for embeddings57       dropout_att: 0 # The dropout for multi-headed attention modules58 59  speech_decoder:60    n_layers: 1261    d_model: 76862    d_ffn: 307263    sa_n_heads: 1264    kernel_size: 365    p_dropout: 0.166    p_dropout_out: 0.067    has_xattn: false68    xa_d_memory: 76869    xa_n_heads: 1270    is_causal: true71    apply_norm_to_cond: true72    apply_norm_out: true73    max_length_causal_mask: 500074    cond_on_prev_audio_tokens: True75    detach_input: False76    use_learnable_pos_emb: True77 78  optimizer:79    _target_: torch.optim.AdamW80    lr: 3e-481    betas: [0.9, 0.98]82    weight_decay: 083    foreach: true # set to false if having issues with tensor-parallelism84 85  lr_scheduler:86#    _target_: nemo.core.optim.lr_scheduler.InverseSquareRootAnnealing87    _target_: nemo.core.optim.lr_scheduler.CosineAnnealing88    warmup_steps: 0  #250089    min_lr: 1e-690    max_steps: ${trainer.max_steps}91 92trainer:93  devices: -194  accelerator: gpu95  num_nodes: 196  precision: bf16-true97  logger: False # logger provided by exp_manager98  enable_checkpointing: False99  use_distributed_sampler: False100  max_steps: 1000000101  limit_train_batches: 100  # "epoch" size102  val_check_interval: ${trainer.limit_train_batches}103  limit_val_batches: 10104  log_every_n_steps: 10105  num_sanity_val_steps: 1106  gradient_clip_val: 1.0107  accumulate_grad_batches: 1108  strategy:109    # Replace DDPStrategy with ModelParallelStrategy to enable model parallelism110    _target_: lightning.pytorch.strategies.DDPStrategy111    gradient_as_bucket_view: true112    find_unused_parameters: true113    # _target_: lightning.pytorch.strategies.ModelParallelStrategy114    # tensor_parallel_size: 1115    # data_parallel_size: 2116 117data:118  frame_length: 0.08119  source_sample_rate: 16000120  target_sample_rate: 22050121  input_roles: ["user", "User"]122  output_roles: ["agent", "Assistant"]123 124  train_ds:125    sample_rate: ${data.target_sample_rate}126    input_cfg:127      - type: lhotse_shar128        shar_path: ???129    seed: 42130    shard_seed: "randomized"131    num_workers: 2132    batch_size: 4133    # Optional bucketing:134    # batch_size: null135    # batch_duration: 100136    # bucket_duration_bins: [8.94766,10.1551,11.64118,19.30376,42.85]137    # use_bucketing: true138    # num_buckets: 5139    # bucket_buffer_size: 5000140 141  validation_ds:142    # The entries under 'datasets' are a list of separate dataloaders.143    # The structure is <dataset-name>: {<dataloader-dict-config>}144    # They inherit all settings from validation_ds, but can individually override them.145    datasets:146      val_set_0:  # rename to your dataset name, add more as needed147        shar_path: ???148    sample_rate: ${data.target_sample_rate}149    batch_size: 1150    seed: 42151    shard_seed: "randomized"152 153exp_manager:154   exp_dir: null155   explicit_log_dir: s2s_sdv2_results/156   name: speechlm2157   create_tensorboard_logger: false158   create_checkpoint_callback: true159   use_datetime_version: true160   max_time_per_run: 00:03:50:00161 162   resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc.163   # you need to set these two to True to continue the training164   resume_if_exists: true165   resume_ignore_no_checkpoint: true166 167   # You may use this section to create a W&B logger168   create_wandb_logger: false169   wandb_logger_kwargs:170     name: development-run171     project: speechlm2_speech_decoder172     resume: true173 174   checkpoint_callback_params:175     filename: "{step}"176     monitor: val_asr_bleu177     mode: max178     every_n_train_steps: null179     every_n_epochs: 1180     save_top_k: 1181     always_save_nemo: false182     save_nemo_on_train_end: false183