LittleDesignSolution/Voxtral-4B-TTS-2603

1{2  "dim": 3072,3  "n_layers": 26,4  "head_dim": 128,5  "hidden_dim": 9216,6  "n_heads": 32,7  "n_kv_heads": 8,8  "fp8_matmul": false,9  "use_biases": false,10  "causal": true,11  "rope_theta": 1000000.0,12  "norm_eps": 1e-05,13  "init": "NO_INIT",14  "dropout": 0.0,15  "vocab_size": 131072,16  "model_parallel": 1,17  "is_sequence_parallel": false,18  "context_parallel": 1,19  "tied_embeddings": true,20  "shard_on_vocab_dim": false,21  "model_pipelining": 1,22  "virtual_model_pipelining": 1,23  "fused_rms_norm": true,24  "checkpoint": false,25  "use_cache": false,26  "max_concurrent_tokens": 65536,27  "learnable_sinks": false,28  "rms_norm": "PRE",29  "cust_bwd": false,30  "recompute_w1_every": 0,31  "recompute_w3_every": 0,32  "recompute_attn_every": 0,33  "freeze_nonembedding": false,34  "fsdp2": true,35  "dp_replicate_size": 1,36  "zero2": true,37  "fsdp_optimize_backward_concat_if_pp": true,38  "attention_type": "FLASH_ATTN_3",39  "multimodal": {40    "bos_token_id": 1,41    "audio_model_args": {42      "semantic_codebook_size": 8192,43      "acoustic_codebook_size": 21,44      "n_acoustic_codebook": 36,45      "audio_encoding_args": {46        "codebook_pattern": "parallel",47        "interleave_audio_tokens_per_segment": 8192,48        "interleave_text_tokens_per_segment": 8192,49        "single_trailing_segment": false,50        "num_codebooks": 37,51        "sampling_rate": 24000,52        "frame_rate": 12.553      },54      "audio_token_id": 24,55      "begin_audio_token_id": 25,56      "input_embedding_concat_type": "sum",57      "acoustic_transformer_args": {58        "input_dim": 3072,59        "dim": 3072,60        "n_layers": 3,61        "head_dim": 128,62        "hidden_dim": 9216,63        "n_heads": 32,64        "n_kv_heads": 8,65        "use_biases": false,66        "rope_theta": 10000.0,67        "sigma": 1e-05,68        "sigma_max": 1.069      },70      "p_uncond": 0.0,71      "text_feature_bugged": false,72      "condition_dropped_token_id": 4273    },74    "audio_tokenizer_args": {75      "channels": 1,76      "sampling_rate": 24000,77      "pretransform_patch_size": 240,78      "patch_proj_kernel_size": 7,79      "semantic_codebook_size": 8192,80      "semantic_dim": 256,81      "acoustic_codebook_size": 21,82      "acoustic_dim": 36,83      "conv_weight_norm": true,84      "causal": true,85      "attn_sliding_window_size": 16,86      "half_attn_window_upon_downsampling": true,87      "dim": 1024,88      "hidden_dim": 4096,89      "head_dim": 128,90      "n_heads": 8,91      "n_kv_heads": 8,92      "qk_norm_eps": 1e-06,93      "qk_norm": true,94      "use_biases": false,95      "norm_eps": 0.01,96      "layer_scale": true,97      "layer_scale_init": 0.01,98      "decoder_transformer_lengths_str": "2,2,2,2",99      "decoder_convs_kernels_str": "3,4,4,4",100      "decoder_convs_strides_str": "1,2,2,2",101      "voice": {102        "casual_female": 0,103        "casual_male": 1,104        "cheerful_female": 2,105        "neutral_female": 3,106        "neutral_male": 4,107        "pt_male": 5,108        "pt_female": 6,109        "nl_male": 7,110        "nl_female": 8,111        "it_male": 9,112        "it_female": 10,113        "fr_male": 11,114        "fr_female": 12,115        "es_male": 13,116        "es_female": 14,117        "de_male": 15,118        "de_female": 16,119        "ar_male": 17,120        "hi_male": 18,121        "hi_female": 19122      }123    }124  },125  "torch_compile_swiglu_noncust_bwd": false,126  "override_parameters_str": "",127  "max_seq_len": 65536,128  "model_type": "voxtral_tts",129  "max_position_embeddings": 128000130}