refactai/codify_3b_multi

1{2  "E": 2560,3  "L": 32,4  "T": 2048,5  "_mup": true,6  "alt_pw_klass": {7    "type": ""8  },9  "alt_rel_klass": {10    "fused": true,11    "type": "alibi"12  },13  "alt_sa_klass": {14    "triton": true,15    "type": "flash",16    "use_rotary_emb": null17  },18  "architectures": [19    "CodifyForCausalLM"20  ],21  "attn_a_reach": 2048,22  "attn_b_reach": 2048,23  "attn_heads": 40,24  "attn_ra_nbasis": 64,25  "attn_seq": [26    "d"27  ],28  "attn_sparse_layout_seq": null,29  "auto_map": {30    "AutoConfig": "configuration_codify.CodifyConfig",31    "AutoModel": "modeling_codify.CodifyForCausalLM"32  },33  "backcheck_pw": "inside",34  "backcheck_sa": "none",35  "bos_token_id": 1,36  "dtype_acts": "torch.float16",37  "dtype_weights": "torch.float16",38  "enc_name": "openai_programming_v2",39  "eos_token_id": 2,40  "init_scale": 1,41  "initializer_range": 0.02,42  "layer_norm_epsilon": 1e-05,43  "mlp_mult": 4,44  "model_type": "codify",45  "moe": null,46  "mup_optimal_lr": 0.0005,47  "mup_shapes_file": "lean_former/mup/alibi_32l/shapes.json",48  "posemb": false,49  "rescale_embeddings": false,50  "tie_word_embeddings": false,51  "torch_dtype": "float32",52  "transformers_version": "4.24.0",53  "tune": [54    3,55    3,56    3,57    358  ],59  "unembedding_shared": false,60  "use_cache": true,61  "use_res_scale": false,62  "vocab_size": 5130563}64