LetheanNetwork/lemma-mlx

1{2    "architectures": [3        "Gemma4ForConditionalGeneration"4    ],5    "audio_config": {6        "_name_or_path": "",7        "architectures": null,8        "attention_chunk_size": 12,9        "attention_context_left": 13,10        "attention_context_right": 0,11        "attention_invalid_logits_value": -1000000000.0,12        "attention_logit_cap": 50.0,13        "chunk_size_feed_forward": 0,14        "conv_kernel_size": 5,15        "dtype": "bfloat16",16        "gradient_clipping": 10000000000.0,17        "hidden_act": "silu",18        "hidden_size": 1024,19        "id2label": {20            "0": "LABEL_0",21            "1": "LABEL_1"22        },23        "initializer_range": 0.02,24        "is_encoder_decoder": false,25        "label2id": {26            "LABEL_0": 0,27            "LABEL_1": 128        },29        "model_type": "gemma4_audio",30        "num_attention_heads": 8,31        "num_hidden_layers": 12,32        "output_attentions": false,33        "output_hidden_states": false,34        "output_proj_dims": 1536,35        "problem_type": null,36        "residual_weight": 0.5,37        "return_dict": true,38        "rms_norm_eps": 1e-06,39        "subsampling_conv_channels": [40            128,41            3242        ],43        "use_clipped_linears": true44    },45    "audio_token_id": 258881,46    "boa_token_id": 256000,47    "boi_token_id": 255999,48    "dtype": "bfloat16",49    "eoa_token_id": 258883,50    "eoa_token_index": 258883,51    "eoi_token_id": 258882,52    "eos_token_id": [53        1,54        106,55        5056    ],57    "image_token_id": 258880,58    "initializer_range": 0.02,59    "model_type": "gemma4",60    "quantization": {61        "group_size": 64,62        "bits": 4,63        "mode": "affine"64    },65    "quantization_config": {66        "group_size": 64,67        "bits": 4,68        "mode": "affine"69    },70    "text_config": {71        "attention_bias": false,72        "attention_dropout": 0.0,73        "attention_k_eq_v": false,74        "bos_token_id": 2,75        "dtype": "bfloat16",76        "enable_moe_block": false,77        "eos_token_id": 1,78        "expert_intermediate_size": null,79        "final_logit_softcapping": 30.0,80        "global_head_dim": 512,81        "head_dim": 256,82        "hidden_activation": "gelu_pytorch_tanh",83        "hidden_size": 2560,84        "hidden_size_per_layer_input": 256,85        "initializer_range": 0.02,86        "intermediate_size": 10240,87        "layer_types": [88            "sliding_attention",89            "sliding_attention",90            "sliding_attention",91            "sliding_attention",92            "sliding_attention",93            "full_attention",94            "sliding_attention",95            "sliding_attention",96            "sliding_attention",97            "sliding_attention",98            "sliding_attention",99            "full_attention",100            "sliding_attention",101            "sliding_attention",102            "sliding_attention",103            "sliding_attention",104            "sliding_attention",105            "full_attention",106            "sliding_attention",107            "sliding_attention",108            "sliding_attention",109            "sliding_attention",110            "sliding_attention",111            "full_attention",112            "sliding_attention",113            "sliding_attention",114            "sliding_attention",115            "sliding_attention",116            "sliding_attention",117            "full_attention",118            "sliding_attention",119            "sliding_attention",120            "sliding_attention",121            "sliding_attention",122            "sliding_attention",123            "full_attention",124            "sliding_attention",125            "sliding_attention",126            "sliding_attention",127            "sliding_attention",128            "sliding_attention",129            "full_attention"130        ],131        "max_position_embeddings": 131072,132        "model_type": "gemma4_text",133        "num_attention_heads": 8,134        "num_experts": null,135        "num_global_key_value_heads": null,136        "num_hidden_layers": 42,137        "num_key_value_heads": 2,138        "num_kv_shared_layers": 18,139        "pad_token_id": 0,140        "rms_norm_eps": 1e-06,141        "rope_parameters": {142            "full_attention": {143                "partial_rotary_factor": 0.25,144                "rope_theta": 1000000.0,145                "rope_type": "proportional"146            },147            "sliding_attention": {148                "rope_theta": 10000.0,149                "rope_type": "default"150            }151        },152        "sliding_window": 512,153        "tie_word_embeddings": true,154        "top_k_experts": null,155        "use_bidirectional_attention": null,156        "use_cache": true,157        "use_double_wide_mlp": false,158        "vocab_size": 262144,159        "vocab_size_per_layer_input": 262144160    },161    "tie_word_embeddings": true,162    "transformers_version": "5.5.0.dev0",163    "video_token_id": 258884,164    "vision_soft_tokens_per_image": 280165}