JBrightmanAI/GEmbedder

1{2  "_sliding_window_pattern": 6,3  "architectures": [4    "Gemma3TextModel"5  ],6  "attention_bias": false,7  "attention_dropout": 0.0,8  "attn_logit_softcapping": null,9  "bos_token_id": 2,10  "dtype": "bfloat16",11  "eos_token_id": 1,12  "final_logit_softcapping": null,13  "head_dim": 128,14  "hidden_activation": "gelu_pytorch_tanh",15  "hidden_size": 5376,16  "initializer_range": 0.02,17  "intermediate_size": 21504,18  "layer_types": [19    "sliding_attention",20    "sliding_attention",21    "sliding_attention",22    "sliding_attention",23    "sliding_attention",24    "full_attention",25    "sliding_attention",26    "sliding_attention",27    "sliding_attention",28    "sliding_attention",29    "sliding_attention",30    "full_attention",31    "sliding_attention",32    "sliding_attention",33    "sliding_attention",34    "sliding_attention",35    "sliding_attention",36    "full_attention",37    "sliding_attention",38    "sliding_attention",39    "sliding_attention",40    "sliding_attention",41    "sliding_attention",42    "full_attention",43    "sliding_attention",44    "sliding_attention",45    "sliding_attention",46    "sliding_attention",47    "sliding_attention",48    "full_attention",49    "sliding_attention",50    "sliding_attention",51    "sliding_attention",52    "sliding_attention",53    "sliding_attention",54    "full_attention",55    "sliding_attention",56    "sliding_attention",57    "sliding_attention",58    "sliding_attention",59    "sliding_attention",60    "full_attention",61    "sliding_attention",62    "sliding_attention",63    "sliding_attention",64    "sliding_attention",65    "sliding_attention",66    "full_attention",67    "sliding_attention",68    "sliding_attention",69    "sliding_attention",70    "sliding_attention",71    "sliding_attention",72    "full_attention",73    "sliding_attention",74    "sliding_attention",75    "sliding_attention",76    "sliding_attention",77    "sliding_attention",78    "full_attention",79    "sliding_attention",80    "sliding_attention"81  ],82  "max_position_embeddings": 131072,83  "model_type": "gemma3_text",84  "num_attention_heads": 32,85  "num_hidden_layers": 62,86  "num_key_value_heads": 16,87  "pad_token_id": 0,88  "query_pre_attn_scalar": 168,89  "rms_norm_eps": 1e-06,90  "rope_local_base_freq": 10000.0,91  "rope_scaling": {92    "factor": 8.0,93    "rope_type": "linear"94  },95  "rope_theta": 1000000.0,96  "sliding_window": 1024,97  "transformers_version": "4.57.6",98  "use_bidirectional_attention": false,99  "use_cache": false,100  "vocab_size": 262208101}102