tsss1/expressvpn_embeddingmodel

1{2  "_name_or_path": "nomic-ai/nomic-embed-text-v2-moe",3  "activation_function": "gelu",4  "add_pooling_layer": false,5  "architectures": [6    "NomicBertModel"7  ],8  "attn_pdrop": 0.0,9  "auto_map": {10    "AutoConfig": "nomic-ai/nomic-bert-2048--configuration_hf_nomic_bert.NomicBertConfig",11    "AutoModel": "nomic-ai/nomic-bert-2048--modeling_hf_nomic_bert.NomicBertModel",12    "AutoModelForMaskedLM": "nomic-ai/nomic-bert-2048--modeling_hf_nomic_bert.NomicBertForPreTraining",13    "AutoModelForMultipleChoice": "nomic-ai/nomic-bert-2048--modeling_hf_nomic_bert.NomicBertForMultipleChoice",14    "AutoModelForQuestionAnswering": "nomic-ai/nomic-bert-2048--modeling_hf_nomic_bert.NomicBertForQuestionAnswering",15    "AutoModelForSequenceClassification": "nomic-ai/nomic-bert-2048--modeling_hf_nomic_bert.NomicBertForSequenceClassification",16    "AutoModelForTokenClassification": "nomic-ai/nomic-bert-2048--modeling_hf_nomic_bert.NomicBertForTokenClassification"17  },18  "bos_token_id": null,19  "causal": false,20  "dense_seq_output": true,21  "embd_pdrop": 0.1,22  "eos_token_id": null,23  "expert_choice_router": false,24  "ffn_div": 1,25  "fused_bias_fc": true,26  "fused_dropout_add_ln": true,27  "initializer_range": 0.02,28  "layer_norm_epsilon": 1e-05,29  "max_trained_positions": 2048,30  "mlp_fc1_bias": true,31  "mlp_fc2_bias": true,32  "model_type": "nomic_bert",33  "moe_every_n_layers": 2,34  "moe_impl": "megablocks",35  "moe_normalize_expert_weights": false,36  "moe_resid_pdrop": 0.0,37  "moe_top_k": 2,38  "n_embd": 768,39  "n_head": 12,40  "n_inner": 3072,41  "n_layer": 12,42  "n_positions": 2048,43  "num_experts": 8,44  "num_shared_experts": 0,45  "pad_token_id": 1,46  "pad_vocab_size_multiple": 64,47  "parallel_block": false,48  "parallel_block_tied_norm": false,49  "prenorm": false,50  "qkv_proj_bias": true,51  "reorder_and_upcast_attn": false,52  "resid_pdrop": 0.0,53  "rotary_emb_base": 10000,54  "rotary_emb_fraction": 1.0,55  "rotary_emb_interleaved": false,56  "rotary_emb_scale_base": null,57  "rotary_scaling_factor": null,58  "router_aux_loss_coef": 0.1,59  "scale_attn_by_inverse_layer_idx": false,60  "scale_attn_weights": true,61  "summary_activation": null,62  "summary_first_dropout": 0.1,63  "summary_proj_to_labels": true,64  "summary_type": "cls_index",65  "summary_use_proj": true,66  "torch_dtype": "float32",67  "transformers_version": "4.48.3",68  "type_vocab_size": 1,69  "use_cache": true,70  "use_flash_attn": true,71  "use_rms_norm": null,72  "use_xentropy": true,73  "vocab_size": 25004874}75