LetheanNetwork/lemma-pt

1{2  "architectures": [3    "Gemma4ForConditionalGeneration"4  ],5  "audio_config": {6    "_name_or_path": "",7    "architectures": null,8    "attention_chunk_size": 12,9    "attention_context_left": 13,10    "attention_context_right": 0,11    "attention_invalid_logits_value": -1000000000.0,12    "attention_logit_cap": 50.0,13    "chunk_size_feed_forward": 0,14    "conv_kernel_size": 5,15    "dtype": "bfloat16",16    "gradient_clipping": 10000000000.0,17    "hidden_act": "silu",18    "hidden_size": 1024,19    "id2label": {20      "0": "LABEL_0",21      "1": "LABEL_1"22    },23    "initializer_range": 0.02,24    "is_encoder_decoder": false,25    "label2id": {26      "LABEL_0": 0,27      "LABEL_1": 128    },29    "model_type": "gemma4_audio",30    "num_attention_heads": 8,31    "num_hidden_layers": 12,32    "output_attentions": false,33    "output_hidden_states": false,34    "output_proj_dims": 1536,35    "problem_type": null,36    "residual_weight": 0.5,37    "return_dict": true,38    "rms_norm_eps": 1e-06,39    "subsampling_conv_channels": [40      128,41      3242    ],43    "use_clipped_linears": true44  },45  "audio_token_id": 258881,46  "boa_token_id": 256000,47  "boi_token_id": 255999,48  "dtype": "bfloat16",49  "eoa_token_id": 258883,50  "eoa_token_index": 258883,51  "eoi_token_id": 258882,52  "image_token_id": 258880,53  "initializer_range": 0.02,54  "model_type": "gemma4",55  "text_config": {56    "attention_bias": false,57    "attention_dropout": 0.0,58    "attention_k_eq_v": false,59    "bos_token_id": 2,60    "dtype": "bfloat16",61    "enable_moe_block": false,62    "eos_token_id": 1,63    "expert_intermediate_size": null,64    "final_logit_softcapping": 30.0,65    "global_head_dim": 512,66    "head_dim": 256,67    "hidden_activation": "gelu_pytorch_tanh",68    "hidden_size": 2560,69    "hidden_size_per_layer_input": 256,70    "initializer_range": 0.02,71    "intermediate_size": 10240,72    "layer_types": [73      "sliding_attention",74      "sliding_attention",75      "sliding_attention",76      "sliding_attention",77      "sliding_attention",78      "full_attention",79      "sliding_attention",80      "sliding_attention",81      "sliding_attention",82      "sliding_attention",83      "sliding_attention",84      "full_attention",85      "sliding_attention",86      "sliding_attention",87      "sliding_attention",88      "sliding_attention",89      "sliding_attention",90      "full_attention",91      "sliding_attention",92      "sliding_attention",93      "sliding_attention",94      "sliding_attention",95      "sliding_attention",96      "full_attention",97      "sliding_attention",98      "sliding_attention",99      "sliding_attention",100      "sliding_attention",101      "sliding_attention",102      "full_attention",103      "sliding_attention",104      "sliding_attention",105      "sliding_attention",106      "sliding_attention",107      "sliding_attention",108      "full_attention",109      "sliding_attention",110      "sliding_attention",111      "sliding_attention",112      "sliding_attention",113      "sliding_attention",114      "full_attention"115    ],116    "max_position_embeddings": 131072,117    "model_type": "gemma4_text",118    "num_attention_heads": 8,119    "num_experts": null,120    "num_global_key_value_heads": null,121    "num_hidden_layers": 42,122    "num_key_value_heads": 2,123    "num_kv_shared_layers": 18,124    "pad_token_id": 0,125    "rms_norm_eps": 1e-06,126    "rope_parameters": {127      "full_attention": {128        "partial_rotary_factor": 0.25,129        "rope_theta": 1000000.0,130        "rope_type": "proportional"131      },132      "sliding_attention": {133        "rope_theta": 10000.0,134        "rope_type": "default"135      }136    },137    "sliding_window": 512,138    "tie_word_embeddings": true,139    "top_k_experts": null,140    "use_bidirectional_attention": null,141    "use_cache": true,142    "use_double_wide_mlp": false,143    "vocab_size": 262144,144    "vocab_size_per_layer_input": 262144145  },146  "tie_word_embeddings": true,147  "transformers_version": "5.5.0.dev0",148  "video_token_id": 258884,149  "vision_config": {150    "_name_or_path": "",151    "architectures": null,152    "attention_bias": false,153    "attention_dropout": 0.0,154    "chunk_size_feed_forward": 0,155    "default_output_length": 280,156    "dtype": "bfloat16",157    "global_head_dim": 64,158    "head_dim": 64,159    "hidden_activation": "gelu_pytorch_tanh",160    "hidden_size": 768,161    "id2label": {162      "0": "LABEL_0",163      "1": "LABEL_1"164    },165    "initializer_range": 0.02,166    "intermediate_size": 3072,167    "is_encoder_decoder": false,168    "label2id": {169      "LABEL_0": 0,170      "LABEL_1": 1171    },172    "max_position_embeddings": 131072,173    "model_type": "gemma4_vision",174    "num_attention_heads": 12,175    "num_hidden_layers": 16,176    "num_key_value_heads": 12,177    "output_attentions": false,178    "output_hidden_states": false,179    "patch_size": 16,180    "pooling_kernel_size": 3,181    "position_embedding_size": 10240,182    "problem_type": null,183    "return_dict": true,184    "rms_norm_eps": 1e-06,185    "rope_parameters": {186      "rope_theta": 100.0,187      "rope_type": "default"188    },189    "standardize": false,190    "use_clipped_linears": true191  },192  "vision_soft_tokens_per_image": 280193}194