Azfarhashmi/adaption_market_analysis_reasoning

1{2  "architectures": [3    "Llama4ForCausalLM"4  ],5  "attention_bias": false,6  "attention_chunk_size": 8192,7  "attention_dropout": 0.0,8  "attn_scale": 0.1,9  "attn_temperature_tuning": true,10  "bos_token_id": 200000,11  "cache_implementation": "hybrid",12  "dtype": "bfloat16",13  "eos_token_id": 200008,14  "floor_scale": 8192,15  "for_llm_compressor": false,16  "head_dim": 128,17  "hidden_act": "silu",18  "hidden_size": 5120,19  "initializer_range": 0.02,20  "interleave_moe_layer_step": 1,21  "intermediate_size": 8192,22  "intermediate_size_mlp": 16384,23  "layer_types": [24    "chunked_attention",25    "chunked_attention",26    "chunked_attention",27    "full_attention",28    "chunked_attention",29    "chunked_attention",30    "chunked_attention",31    "full_attention",32    "chunked_attention",33    "chunked_attention",34    "chunked_attention",35    "full_attention",36    "chunked_attention",37    "chunked_attention",38    "chunked_attention",39    "full_attention",40    "chunked_attention",41    "chunked_attention",42    "chunked_attention",43    "full_attention",44    "chunked_attention",45    "chunked_attention",46    "chunked_attention",47    "full_attention",48    "chunked_attention",49    "chunked_attention",50    "chunked_attention",51    "full_attention",52    "chunked_attention",53    "chunked_attention",54    "chunked_attention",55    "full_attention",56    "chunked_attention",57    "chunked_attention",58    "chunked_attention",59    "full_attention",60    "chunked_attention",61    "chunked_attention",62    "chunked_attention",63    "full_attention",64    "chunked_attention",65    "chunked_attention",66    "chunked_attention",67    "full_attention",68    "chunked_attention",69    "chunked_attention",70    "chunked_attention",71    "full_attention"72  ],73  "max_position_embeddings": 10485760,74  "model_type": "llama4_text",75  "moe_layers": [76    0,77    1,78    2,79    3,80    4,81    5,82    6,83    7,84    8,85    9,86    10,87    11,88    12,89    13,90    14,91    15,92    16,93    17,94    18,95    19,96    20,97    21,98    22,99    23,100    24,101    25,102    26,103    27,104    28,105    29,106    30,107    31,108    32,109    33,110    34,111    35,112    36,113    37,114    38,115    39,116    40,117    41,118    42,119    43,120    44,121    45,122    46,123    47124  ],125  "no_rope_layer_interval": 4,126  "no_rope_layers": [127    1,128    1,129    1,130    0,131    1,132    1,133    1,134    0,135    1,136    1,137    1,138    0,139    1,140    1,141    1,142    0,143    1,144    1,145    1,146    0,147    1,148    1,149    1,150    0,151    1,152    1,153    1,154    0,155    1,156    1,157    1,158    0,159    1,160    1,161    1,162    0,163    1,164    1,165    1,166    0,167    1,168    1,169    1,170    0,171    1,172    1,173    1,174    0175  ],176  "num_attention_heads": 40,177  "num_experts_per_tok": 1,178  "num_hidden_layers": 48,179  "num_key_value_heads": 8,180  "num_local_experts": 16,181  "output_router_logits": false,182  "pad_token_id": 200018,183  "quantization_config": {184    "_load_in_4bit": true,185    "_load_in_8bit": false,186    "bnb_4bit_compute_dtype": "bfloat16",187    "bnb_4bit_quant_storage": "bfloat16",188    "bnb_4bit_quant_type": "nf4",189    "bnb_4bit_use_double_quant": false,190    "llm_int8_enable_fp32_cpu_offload": false,191    "llm_int8_has_fp16_weight": false,192    "llm_int8_skip_modules": null,193    "llm_int8_threshold": 6.0,194    "load_in_4bit": true,195    "load_in_8bit": false,196    "quant_method": "bitsandbytes"197  },198  "rms_norm_eps": 1e-05,199  "rope_parameters": {200    "factor": 16.0,201    "high_freq_factor": 1.0,202    "low_freq_factor": 1.0,203    "original_max_position_embeddings": 8192,204    "rope_theta": 500000.0,205    "rope_type": "llama3"206  },207  "router_aux_loss_coef": 0.001,208  "router_jitter_noise": 0.0,209  "tie_word_embeddings": false,210  "transformers_version": "5.10.1",211  "use_cache": false,212  "use_qk_norm": true,213  "vocab_size": 202048,214  "torch_dtype": "bfloat16"215}216