FarziBuilder/fastInferenceTry6

1{2  "_name_or_path": "Birchlabs/mosaicml-mpt-7b-chat-qlora",3  "architectures": [4    "MPTForCausalLM"5  ],6  "attn_config": {7    "alibi": true,8    "alibi_bias_max": 8,9    "attn_impl": "torch",10    "attn_pdrop": 0,11    "attn_type": "multihead_attention",12    "attn_uses_sequence_id": false,13    "clip_qkv": null,14    "prefix_lm": false,15    "qk_ln": false,16    "softmax_scale": null17  },18  "auto_map": {19    "AutoConfig": "Birchlabs/mosaicml-mpt-7b-chat-qlora--configuration_mpt.MPTConfig",20    "AutoModelForCausalLM": "Birchlabs/mosaicml-mpt-7b-chat-qlora--modeling_mpt.MPTForCausalLM"21  },22  "d_model": 4096,23  "emb_pdrop": 0,24  "embedding_fraction": 1.0,25  "expansion_ratio": 4,26  "init_config": {27    "emb_init_std": null,28    "emb_init_uniform_lim": null,29    "fan_mode": "fan_in",30    "init_div_is_residual": true,31    "init_gain": 0,32    "init_nonlinearity": "relu",33    "init_std": 0.02,34    "name": "kaiming_normal_",35    "verbose": 036  },37  "init_device": "cpu",38  "learned_pos_emb": true,39  "logit_scale": null,40  "max_seq_len": 2048,41  "model_type": "mpt",42  "n_heads": 32,43  "n_layers": 32,44  "no_bias": true,45  "norm_type": "low_precision_layernorm",46  "resid_pdrop": 0,47  "tokenizer_name": "sam-mosaic/gpt-neox-20b-chatml",48  "torch_dtype": "float16",49  "transformers_version": "4.31.0.dev0",50  "use_cache": false,51  "verbose": 0,52  "vocab_size": 5043253}54