stepfun-ai/Step-3.5-Flash

1{2  "architectures": [3    "Step3p5ForCausalLM"4  ],5  "model_type": "step3p5",6  "auto_map": {7    "AutoConfig": "configuration_step3p5.Step3p5Config",8    "AutoModelForCausalLM": "modeling_step3p5.Step3p5ForCausalLM"9  },10  "rope_scaling": {11    "rope_type": "llama3",12    "factor": 2.0,13    "original_max_position_embeddings": 131072,14    "low_freq_factor": 1.0,15    "high_freq_factor": 32.016  },17  "yarn_only_types": ["full_attention"],18  "hidden_size": 4096,19  "intermediate_size": 11264,20  "num_hidden_layers": 45,21  "max_seq_len": 262144,22  "vocab_size": 128896,23  "torch_dtype": "bfloat16",24  "use_qk_norm": true,25  "moe_layers_enum": "3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44",26  "num_attention_heads": 64,27  "num_attention_groups": 8,28  "head_dim": 128,29  "use_moe": true,30  "moe_num_experts": 288,31  "moe_top_k": 8,32  "moe_intermediate_size": 1280,33  "share_expert_dim": 1280,34  "moe_layer_offset": 0,35  "moe_every_n_layer": 1,36  "norm_expert_weight": true,37  "moe_router_activation": "sigmoid",38  "moe_router_scaling_factor": 3.0,39  "att_impl_type": "GQA",40  "tie_word_embeddings": false,41  "rope_theta": [42    5000000.0,43    10000.0,44    10000.0,45    10000.0,46    5000000.0,47    10000.0,48    10000.0,49    10000.0,50    5000000.0,51    10000.0,52    10000.0,53    10000.0,54    5000000.0,55    10000.0,56    10000.0,57    10000.0,58    5000000.0,59    10000.0,60    10000.0,61    10000.0,62    5000000.0,63    10000.0,64    10000.0,65    10000.0,66    5000000.0,67    10000.0,68    10000.0,69    10000.0,70    5000000.0,71    10000.0,72    10000.0,73    10000.0,74    5000000.0,75    10000.0,76    10000.0,77    10000.0,78    5000000.0,79    10000.0,80    10000.0,81    10000.0,82    5000000.0,83    10000.0,84    10000.0,85    10000.0,86    5000000.0,87    10000.0,88    10000.0,89    10000.090  ],91  "use_head_wise_attn_gate": true,92  "sliding_window": 512,93  "use_moe_router_bias": true,94  "need_fp32_gate": true,95  "sink": false,96  "layer_types": [97    "full_attention",98    "sliding_attention",99    "sliding_attention",100    "sliding_attention",101    "full_attention",102    "sliding_attention",103    "sliding_attention",104    "sliding_attention",105    "full_attention",106    "sliding_attention",107    "sliding_attention",108    "sliding_attention",109    "full_attention",110    "sliding_attention",111    "sliding_attention",112    "sliding_attention",113    "full_attention",114    "sliding_attention",115    "sliding_attention",116    "sliding_attention",117    "full_attention",118    "sliding_attention",119    "sliding_attention",120    "sliding_attention",121    "full_attention",122    "sliding_attention",123    "sliding_attention",124    "sliding_attention",125    "full_attention",126    "sliding_attention",127    "sliding_attention",128    "sliding_attention",129    "full_attention",130    "sliding_attention",131    "sliding_attention",132    "sliding_attention",133    "full_attention",134    "sliding_attention",135    "sliding_attention",136    "sliding_attention",137    "full_attention",138    "sliding_attention",139    "sliding_attention",140    "sliding_attention",141    "full_attention",142    "sliding_attention",143    "sliding_attention",144    "sliding_attention"145  ],146  "use_rope_layers": [],147  "num_nextn_predict_layers": 3,148  "partial_rotary_factors": [149    0.5,150    1.0,151    1.0,152    1.0,153    0.5,154    1.0,155    1.0,156    1.0,157    0.5,158    1.0,159    1.0,160    1.0,161    0.5,162    1.0,163    1.0,164    1.0,165    0.5,166    1.0,167    1.0,168    1.0,169    0.5,170    1.0,171    1.0,172    1.0,173    0.5,174    1.0,175    1.0,176    1.0,177    0.5,178    1.0,179    1.0,180    1.0,181    0.5,182    1.0,183    1.0,184    1.0,185    0.5,186    1.0,187    1.0,188    1.0,189    0.5,190    1.0,191    1.0,192    1.0,193    0.5,194    1.0,195    1.0,196    1.0197  ],198  "eos_token_id": [199    1,200    2,201    128007202  ],203  "bos_token_id": 0,204  "attention_other_setting": {205    "attention_type": "sliding_attention",206    "num_attention_heads": 96,207    "num_attention_groups": 8,208    "head_dim": 128,209    "true_head_dim": 128210  },211  "swiglu_limits": [212    0.0,213    0.0,214    0.0,215    0.0,216    0.0,217    0.0,218    0.0,219    0.0,220    0.0,221    0.0,222    0.0,223    0.0,224    0.0,225    0.0,226    0.0,227    0.0,228    0.0,229    0.0,230    0.0,231    0.0,232    0.0,233    0.0,234    0.0,235    0.0,236    0.0,237    0.0,238    0.0,239    0.0,240    0.0,241    0.0,242    0.0,243    0.0,244    0.0,245    0.0,246    0.0,247    0.0,248    0.0,249    0.0,250    0.0,251    0.0,252    0.0,253    0.0,254    0.0,255    7,256    7,257    0.0,258    0.0,259    0.0260  ],261  "swiglu_limits_shared": [262    0.0,263    0.0,264    0.0,265    0.0,266    0.0,267    0.0,268    0.0,269    0.0,270    0.0,271    0.0,272    0.0,273    0.0,274    0.0,275    0.0,276    0.0,277    0.0,278    0.0,279    0.0,280    0.0,281    0.0,282    0.0,283    0.0,284    0.0,285    0.0,286    0.0,287    0.0,288    0.0,289    0.0,290    0.0,291    0.0,292    0.0,293    0.0,294    0.0,295    0.0,296    0.0,297    0.0,298    0.0,299    0.0,300    0.0,301    0.0,302    0.0,303    0.0,304    0.0,305    0.0,306    16,307    0.0,308    0.0,309    0.0310  ],311  "zero_centered": true,312  "max_position_embeddings": 262144313}