amritansh/replit_3B

1{2  "architectures": [3    "MPTForCausalLM"4  ],5  "attn_config": {6    "alibi": true,7    "alibi_bias_max": 8,8    "attn_impl": "torch",9    "attn_pdrop": 0,10    "attn_type": "multihead_attention",11    "attn_uses_sequence_id": false,12    "clip_qkv": null,13    "prefix_lm": false,14    "qk_ln": false,15    "softmax_scale": null16  },17  "auto_map": {18    "AutoConfig": "configuration_mpt.MPTConfig",19    "AutoModelForCausalLM": "modeling_mpt.MPTForCausalLM"20  },21  "d_model": 2560,22  "emb_pdrop": 0,23  "embedding_fraction": 1.0,24  "expansion_ratio": 4,25  "init_config": {26    "emb_init_std": null,27    "emb_init_uniform_lim": null,28    "fan_mode": "fan_in",29    "init_div_is_residual": true,30    "init_gain": 0,31    "init_nonlinearity": "relu",32    "init_std": 0.02,33    "name": "kaiming_normal_",34    "verbose": 035  },36  "init_device": "cpu",37  "learned_pos_emb": true,38  "logit_scale": null,39  "max_seq_len": 2048,40  "model_type": "mpt",41  "n_heads": 32,42  "n_layers": 32,43  "no_bias": true,44  "norm_type": "low_precision_layernorm",45  "resid_pdrop": 0,46  "tokenizer_name": "teknium/Replit-v2-CodeInstruct-3B",47  "torch_dtype": "float16",48  "transformers_version": "4.29.2",49  "use_cache": false,50  "verbose": 0,51  "vocab_size": 3276852}