BaptisteL/m2-reference-model

1{2  "_name_or_path": "./output/untrained_model",3  "activation_fn_name": "swish",4  "architectures": [5    "OpenELMForCausalLM"6  ],7  "auto_map": {8    "AutoConfig": "configuration_openelm.OpenELMConfig",9    "AutoModelForCausalLM": "modeling_openelm.OpenELMForCausalLM"10  },11  "bos_token_id": 1,12  "eos_token_id": 2,13  "ffn_dim_divisor": 256,14  "ffn_multipliers": [15    0.5,16    0.73,17    0.97,18    1.2,19    1.43,20    1.67,21    1.9,22    2.13,23    2.37,24    2.6,25    2.83,26    3.07,27    3.3,28    3.53,29    3.77,30    4.031  ],32  "ffn_with_glu": true,33  "head_dim": 64,34  "initializer_range": 0.02,35  "max_context_length": 2048,36  "model_dim": 1280,37  "model_type": "openelm",38  "normalization_layer_name": "rms_norm",39  "normalize_qk_projections": true,40  "num_gqa_groups": 4,41  "num_kv_heads": [42    3,43    3,44    3,45    3,46    3,47    4,48    4,49    4,50    4,51    4,52    4,53    4,54    5,55    5,56    5,57    558  ],59  "num_query_heads": [60    12,61    12,62    12,63    12,64    12,65    16,66    16,67    16,68    16,69    16,70    16,71    16,72    20,73    20,74    20,75    2076  ],77  "num_transformer_layers": 16,78  "qkv_multipliers": [79    0.5,80    1.081  ],82  "rope_freq_constant": 10000,83  "rope_max_length": 4096,84  "share_input_output_layers": true,85  "torch_dtype": "float32",86  "transformers_version": "4.40.2",87  "use_cache": true,88  "vocab_size": 3200089}90