dots-studio/dots3-note-prev

1{2  "architectures": [3    "Dots3NoteForCausalLM"4  ],5  "apply_mla_qkv_lora_rescale": true,6  "attention_bias": false,7  "attention_dropout": 0.0,8  "attention_gate_type": "headwise",9  "bos_token_id": 151643,10  "eos_token_id": 151668,11  "first_k_dense_replace": 1,12  "hidden_act": "silu",13  "hidden_size": 5120,14  "index_head_dim": 128,15  "index_n_heads": 64,16  "index_topk": 2048,17  "initializer_range": 0.02,18  "intermediate_size": 13824,19  "kv_lora_rank": 512,20  "layer_types": [21    "full_attention",22    "full_attention",23    "sliding_attention",24    "sliding_attention",25    "sliding_attention",26    "full_attention",27    "sliding_attention",28    "sliding_attention",29    "sliding_attention",30    "full_attention",31    "sliding_attention",32    "sliding_attention",33    "sliding_attention",34    "full_attention",35    "sliding_attention",36    "sliding_attention",37    "sliding_attention",38    "full_attention",39    "sliding_attention",40    "sliding_attention",41    "sliding_attention",42    "full_attention",43    "sliding_attention",44    "sliding_attention",45    "sliding_attention",46    "full_attention",47    "sliding_attention",48    "sliding_attention",49    "sliding_attention",50    "full_attention",51    "sliding_attention",52    "sliding_attention",53    "sliding_attention",54    "full_attention",55    "sliding_attention",56    "sliding_attention",57    "sliding_attention",58    "full_attention",59    "sliding_attention",60    "sliding_attention",61    "sliding_attention",62    "full_attention",63    "sliding_attention",64    "sliding_attention",65    "sliding_attention",66    "full_attention"67  ],68  "max_position_embeddings": 524288,69  "model_type": "dots3_note",70  "moe_intermediate_size": 1536,71  "moe_layer_freq": 1,72  "n_routed_experts": 256,73  "n_shared_experts": 1,74  "norm_topk_prob": true,75  "num_attention_heads": 128,76  "num_experts_per_tok": 8,77  "num_hidden_layers": 46,78  "num_key_value_heads": 128,79  "pretraining_tp": 1,80  "q_lora_rank": 1024,81  "qk_nope_head_dim": 128,82  "qk_rope_head_dim": 64,83  "rms_norm_eps": 1e-05,84  "rope_scaling": null,85  "rope_theta": 80000000.0,86  "routed_scaling_factor": 1.0,87  "scoring_func": "sigmoid",88  "sliding_window_size": 513,89  "swa_attention_gate_type": "headwise",90  "swa_kv_lora_rank": 1024,91  "swa_num_attention_heads": 64,92  "swa_num_key_value_heads": 64,93  "swa_q_lora_rank": 1024,94  "swa_qk_nope_head_dim": 192,95  "swa_qk_rope_head_dim": 64,96  "swa_rope_theta": 50000.0,97  "swa_v_head_dim": 128,98  "tie_word_embeddings": false,99  "topk_method": "noaux_tc",100  "torch_dtype": "bfloat16",101  "transformers_version": "4.36.0",102  "use_cache": true,103  "v_head_dim": 128,104  "vocab_size": 152064,105  "vision_config": {106    "embed_dim": 1536,107    "hidden_size": 5120,108    "intermediate_size": 4224,109    "moe_intermediate_size": 2112,110    "num_hidden_layers": 42,111    "num_attention_heads": 24,112    "num_channels": 3,113    "patch_size": 14,114    "spatial_merge_size": 2,115    "temporal_patch_size": 1,116    "rms_norm_eps": 1e-05,117    "use_bias": false,118    "use_qk_norm": true,119    "attn_implementation": "flash_attention_3",120    "initializer_range": 0.02,121    "is_causal": false,122    "post_norm": true,123    "pre_pixel_shuffle": true,124    "pyramid_num_routed": [125      -1,126      -1,127      -1,128      -1,129      -1,130      -1,131      -1,132      -1,133      -1,134      -1,135      -1,136      -1,137      -1,138      -1,139      -1,140      -1,141      -1,142      -1,143      -1,144      -1,145      -1,146      -1,147      -1,148      -1,149      -1,150      4,151      8,152      12,153      16,154      20,155      24,156      28,157      32,158      36,159      40,160      44,161      48,162      52,163      56,164      60,165      64,166      64167    ],168    "capacity_factor": 2,169    "router_scoring_func": "sigmoid",170    "router_scale": 1.0,171    "adapter_type": "patch_merger",172    "adapter_in_dim": 1536,173    "adapter_out_dim": 5120,174    "adapter_merge_size": 2175  },176  "audio_config": {177    "encoder_type": "dots",178    "whisper_config": {179      "d_model": 1280,180      "encoder_attention_heads": 20,181      "encoder_ffn_dim": 5120,182      "encoder_layers": 32,183      "num_mel_bins": 128,184      "max_source_positions": 6000,185      "activation_function": "swiglu"186    },187    "use_conv2d_stem": true,188    "use_rope": true,189    "use_rms_norm": true,190    "use_causal": false,191    "downsample_hidden_size": 480,192    "conv_chunksize": 500,193    "conv_stem_gradient_checkpointing": true,194    "conv_bucket_step": 10,195    "conv_bucket_max_elements": 20000,196    "rope_parameters": {197      "partial_rotary_factor": 0.5,198      "rope_theta": 10000.0199    },200    "merge_factor": 1,201    "chunk_seconds": 60,202    "whisper_adapter_in_dim": 1280,203    "whisper_adapter_out_dim": 5120,204    "sampling_rate": 16000,205    "audio_comp_start": "<|audio_comp_start|>",206    "audio_comp_span": "<|audio_comp_pad|>",207    "audio_comp_end": "<|audio_comp_end|>"208  }209}210