Nethermind/Mpt-Instruct-DotNet-XS

1{2  "alibi": true,3  "alibi_bias_max": 8,4  "architectures": [5    "MosaicGPT"6  ],7  "attn_clip_qkv": null,8  "attn_impl": "torch",9  "attn_pdrop": 0,10  "attn_qk_ln": true,11  "attn_uses_sequence_id": false,12  "auto_map": {13    "AutoConfig": "configuration_mosaic_gpt.MosaicGPTConfig",14    "AutoModelForCausalLM": "mosaic_gpt.MosaicGPT"15  },16  "d_model": 2048,17  "emb_init_std": null,18  "emb_init_uniform_lim": null,19  "emb_pdrop": 0,20  "embedding_fraction": 1.0,21  "fan_mode": "fan_in",22  "init_device": "cpu",23  "init_div_is_residual": true,24  "init_gain": 0,25  "init_nonlinearity": "relu",26  "init_std": 0.02,27  "logit_scale": null,28  "low_precision_layernorm": true,29  "max_seq_len": 2048,30  "mlp_ratio": 4,31  "model_type": "mosaic_gpt",32  "n_heads": 16,33  "n_layers": 24,34  "no_bias": true,35  "param_init_fn": "kaiming_normal_",36  "prefix_lm": false,37  "resid_pdrop": 0,38  "softmax_scale": null,39  "tokenizer_name": "EleutherAI/gpt-neox-20b",40  "torch_dtype": "bfloat16",41  "transformers_version": "4.33.0",42  "use_cache": false,43  "verbose": 0,44  "vocab_size": 5043245}46