Defetya/math-kaggle-comp

1{2  "_name_or_path": "microsoft/phi-2",3  "architectures": [4    "PhiForCausalLM"5  ],6  "attention_dropout": 0.0,7  "attention_partition_spec": [8    [9      "dp",10      "fsdp"11    ],12    "sp",13    "tp",14    null15  ],16  "attn_mechanism": "normal",17  "auto_map": {18    "AutoConfig": "microsoft/phi-2--configuration_phi.PhiConfig",19    "AutoModelForCausalLM": "microsoft/phi-2--modeling_phi.PhiForCausalLM"20  },21  "axis_dims": [22    1,23    -1,24    1,25    126  ],27  "axis_names": [28    "dp",29    "fsdp",30    "tp",31    "sp"32  ],33  "backend": null,34  "bias_partition_spec": [35    [36      "dp",37      "fsdp"38    ],39    null,40    null,41    null42  ],43  "bits": null,44  "block_b": 1,45  "block_k": 128,46  "block_k_dkv": 128,47  "block_k_dq": 128,48  "block_k_major": 128,49  "block_k_major_dkv": 128,50  "block_k_major_dq": 128,51  "block_q": 128,52  "block_q_dkv": 128,53  "block_q_dq": 128,54  "block_q_major_dkv": 128,55  "bos_token_id": 50256,56  "easy_method": "train",57  "embd_pdrop": 0.0,58  "eos_token_id": 50256,59  "generation_attention_partition_spec": [60    [61      "dp",62      "fsdp"63    ],64    null,65    "tp",66    null67  ],68  "generation_bias_partition_spec": [69    [70      "dp",71      "fsdp"72    ],73    null,74    null,75    null76  ],77  "generation_query_partition_spec": [78    [79      "dp",80      "fsdp"81    ],82    "sp",83    null,84    null85  ],86  "hidden_act": "gelu_new",87  "initializer_range": 0.02,88  "intermediate_size": 10240,89  "key_partition_spec": [90    [91      "dp",92      "fsdp"93    ],94    "sp",95    "tp",96    null97  ],98  "layer_norm_eps": 1e-05,99  "model_type": "phi",100  "n_embd": 2560,101  "n_positions": 2048,102  "num_attention_heads": 32,103  "num_hidden_layers": 32,104  "num_key_value_heads": 32,105  "partial_rotary_factor": 0.4,106  "qk_layernorm": false,107  "query_partition_spec": [108    [109      "dp",110      "fsdp"111    ],112    "sp",113    "tp",114    null115  ],116  "resid_pdrop": 0.1,117  "rope_scaling": null,118  "rope_theta": 10000.0,119  "scan_attention_layers": false,120  "scan_mlp_chunk_size": 1024,121  "scan_ring_attention": false,122  "shard_attention_computation": true,123  "tie_word_embeddings": false,124  "torch_dtype": "float32",125  "transformers_version": "4.38.0",126  "use_cache": true,127  "use_scan_mlp": true,128  "use_sharded_kv_caching": true,129  "use_sharding_constraint": false,130  "value_partition_spec": [131    [132      "dp",133      "fsdp"134    ],135    "sp",136    "tp",137    null138  ],139  "vocab_size": 51200140}141