trl-internal-testing/tiny-Gemma4ForConditionalGeneration

1{2  "architectures": [3    "Gemma4ForConditionalGeneration"4  ],5  "audio_config": null,6  "audio_token_id": 258881,7  "boa_token_id": 256000,8  "boi_token_id": 255999,9  "dtype": "bfloat16",10  "eoa_token_id": 258883,11  "eoa_token_index": 258883,12  "eoi_token_id": 258882,13  "eos_token_id": [14    1,15    10616  ],17  "image_token_id": 258880,18  "initializer_range": 0.02,19  "model_type": "gemma4",20  "text_config": {21    "attention_bias": false,22    "attention_dropout": 0.0,23    "attention_k_eq_v": false,24    "bos_token_id": 2,25    "dtype": "bfloat16",26    "enable_moe_block": false,27    "eos_token_id": 1,28    "expert_intermediate_size": null,29    "final_logit_softcapping": 30.0,30    "global_head_dim": 8,31    "head_dim": 256,32    "hidden_activation": "gelu_pytorch_tanh",33    "hidden_size": 16,34    "hidden_size_per_layer_input": 16,35    "initializer_range": 0.02,36    "intermediate_size": 32,37    "layer_types": [38      "sliding_attention",39      "full_attention"40    ],41    "max_position_embeddings": 131072,42    "model_type": "gemma4_text",43    "moe_intermediate_size": null,44    "num_attention_heads": 4,45    "num_experts": null,46    "num_global_key_value_heads": null,47    "num_hidden_layers": 2,48    "num_key_value_heads": 2,49    "num_kv_shared_layers": 0,50    "pad_token_id": 0,51    "rms_norm_eps": 1e-06,52    "rope_parameters": {53      "full_attention": {54        "partial_rotary_factor": 0.25,55        "rope_theta": 1000000.0,56        "rope_type": "proportional"57      },58      "sliding_attention": {59        "rope_theta": 10000.0,60        "rope_type": "default"61      }62    },63    "sliding_window": 512,64    "tie_word_embeddings": true,65    "top_k_experts": null,66    "use_bidirectional_attention": null,67    "use_cache": true,68    "use_double_wide_mlp": true,69    "vocab_size": 262144,70    "vocab_size_per_layer_input": 26214471  },72  "tie_word_embeddings": true,73  "transformers_version": "5.5.0",74  "video_token_id": 258884,75  "vision_config": {76    "_name_or_path": "",77    "architectures": null,78    "attention_bias": false,79    "attention_dropout": 0.0,80    "chunk_size_feed_forward": 0,81    "default_output_length": 70,82    "dtype": "bfloat16",83    "embed_dim": 64,84    "global_head_dim": 64,85    "head_dim": 64,86    "hidden_activation": "gelu_pytorch_tanh",87    "hidden_size": 16,88    "id2label": {89      "0": "LABEL_0",90      "1": "LABEL_1"91    },92    "initializer_range": 0.02,93    "intermediate_size": 32,94    "is_encoder_decoder": false,95    "label2id": {96      "LABEL_0": 0,97      "LABEL_1": 198    },99    "max_position_embeddings": 131072,100    "model_type": "gemma4_vision",101    "num_attention_heads": 4,102    "num_hidden_layers": 2,103    "num_key_value_heads": 2,104    "output_attentions": false,105    "output_hidden_states": false,106    "patch_size": 16,107    "pooling_kernel_size": 3,108    "position_embedding_size": 630,109    "problem_type": null,110    "return_dict": true,111    "rms_norm_eps": 1e-06,112    "rope_parameters": {113      "rope_theta": 100.0,114      "rope_type": "default"115    },116    "standardize": false,117    "use_clipped_linears": true118  },119  "vision_soft_tokens_per_image": 280120}121