tiny-random/gemma-4-dense

1{2  "architectures": [3    "Gemma4ForConditionalGeneration"4  ],5  "audio_config": null,6  "audio_token_id": 258881,7  "boa_token_id": 256000,8  "boi_token_id": 255999,9  "dtype": "bfloat16",10  "eoa_token_id": 258883,11  "eoa_token_index": 258883,12  "eoi_token_id": 258882,13  "eos_token_id": [14    1,15    10616  ],17  "image_token_id": 258880,18  "initializer_range": 0.02,19  "model_type": "gemma4",20  "text_config": {21    "attention_bias": false,22    "attention_dropout": 0.0,23    "attention_k_eq_v": true,24    "bos_token_id": 2,25    "dtype": "bfloat16",26    "enable_moe_block": false,27    "eos_token_id": 1,28    "expert_intermediate_size": null,29    "final_logit_softcapping": 30.0,30    "global_head_dim": 64,31    "head_dim": 32,32    "hidden_activation": "gelu_pytorch_tanh",33    "hidden_size": 8,34    "hidden_size_per_layer_input": 0,35    "initializer_range": 0.02,36    "intermediate_size": 64,37    "layer_types": [38      "sliding_attention",39      "full_attention",40      "sliding_attention",41      "full_attention"42    ],43    "max_position_embeddings": 262144,44    "model_type": "gemma4_text",45    "moe_intermediate_size": null,46    "num_attention_heads": 8,47    "num_experts": null,48    "num_global_key_value_heads": 4,49    "num_hidden_layers": 4,50    "num_key_value_heads": 4,51    "num_kv_shared_layers": 0,52    "pad_token_id": 0,53    "rms_norm_eps": 1e-06,54    "rope_parameters": {55      "full_attention": {56        "partial_rotary_factor": 0.25,57        "rope_theta": 1000000.0,58        "rope_type": "proportional"59      },60      "sliding_attention": {61        "rope_theta": 10000.0,62        "rope_type": "default"63      }64    },65    "sliding_window": 1024,66    "tie_word_embeddings": true,67    "top_k_experts": null,68    "use_bidirectional_attention": "vision",69    "use_cache": true,70    "use_double_wide_mlp": false,71    "vocab_size": 262144,72    "vocab_size_per_layer_input": 26214473  },74  "tie_word_embeddings": true,75  "transformers_version": "5.9.0",76  "video_token_id": 258884,77  "vision_config": {78    "_name_or_path": "",79    "architectures": null,80    "attention_bias": false,81    "attention_dropout": 0.0,82    "chunk_size_feed_forward": 0,83    "default_output_length": 280,84    "dtype": "bfloat16",85    "global_head_dim": 32,86    "head_dim": 32,87    "hidden_activation": "gelu_pytorch_tanh",88    "hidden_size": 8,89    "id2label": {90      "0": "LABEL_0",91      "1": "LABEL_1"92    },93    "initializer_range": 0.02,94    "intermediate_size": 64,95    "is_encoder_decoder": false,96    "label2id": {97      "LABEL_0": 0,98      "LABEL_1": 199    },100    "max_position_embeddings": 131072,101    "model_type": "gemma4_vision",102    "num_attention_heads": 4,103    "num_hidden_layers": 2,104    "num_key_value_heads": 4,105    "output_attentions": false,106    "output_hidden_states": false,107    "patch_size": 16,108    "pooling_kernel_size": 3,109    "position_embedding_size": 10240,110    "problem_type": null,111    "return_dict": true,112    "rms_norm_eps": 1e-06,113    "rope_parameters": {114      "rope_theta": 100.0,115      "rope_type": "default"116    },117    "standardize": true,118    "use_clipped_linears": false119  },120  "vision_soft_tokens_per_image": 280121}122