matorus/replit-coder

1{2  "architectures": [3    "MPTForCausalLM"4  ],5  "attn_config": {6    "alibi": true,7    "alibi_bias_max": 8,8    "attn_impl": "torch",9    "attn_pdrop": 0,10    "attn_type": "multihead_attention",11    "attn_uses_sequence_id": false,12    "clip_qkv": null,13    "prefix_lm": false,14    "qk_ln": false,15    "softmax_scale": null16  },17  "auto_map": {18    "AutoConfig": "configuration_mpt.MPTConfig",19    "AutoModelForCausalLM": "modeling_mpt.MPTForCausalLM"20  },21  "d_model": 2560,22  "emb_pdrop": 0,23  "embedding_fraction": 1.0,24  "eos_token_id": 1,25  "expansion_ratio": 4,26  "init_config": {27    "emb_init_std": null,28    "emb_init_uniform_lim": null,29    "fan_mode": "fan_in",30    "init_div_is_residual": true,31    "init_gain": 0,32    "init_nonlinearity": "relu",33    "init_std": 0.02,34    "name": "kaiming_normal_",35    "verbose": 036  },37  "init_device": "cpu",38  "learned_pos_emb": true,39  "logit_scale": null,40  "max_seq_len": 2048,41  "model_type": "mpt",42  "n_heads": 32,43  "n_layers": 32,44  "no_bias": true,45  "norm_type": "low_precision_layernorm",46  "resid_pdrop": 0,47  "tokenizer_name": "replit/replit-code-v1-3b",48  "torch_dtype": "bfloat16",49  "transformers_version": "4.30.2",50  "use_cache": true,51  "verbose": 0,52  "vocab_size": 3276853}54