trl-internal-testing/tiny-MuseGlimmerForConditionalGeneration

1{2  "architectures": [3    "MuseGlimmerForConditionalGeneration"4  ],5  "dtype": "bfloat16",6  "image_token_id": 200092,7  "model_type": "muse_glimmer",8  "out_hidden_size": 64,9  "projector_hidden_act": "gelu",10  "projector_hidden_size": 32,11  "text_config": {12    "attention_bias": false,13    "attention_dropout": 0.0,14    "bos_token_id": 200000,15    "eos_token_id": 200001,16    "final_logit_softcapping": 20.0,17    "head_dim": 8,18    "hidden_activation": "silu",19    "hidden_size": 16,20    "initializer_range": 0.02,21    "intermediate_size": 32,22    "layer_rope_theta": [23      500000.0,24      025    ],26    "layer_types": [27      "sliding_attention",28      "full_attention"29    ],30    "max_position_embeddings": 131072,31    "model_type": "muse_glimmer_text",32    "num_attention_heads": 4,33    "num_hidden_layers": 2,34    "num_key_value_heads": 2,35    "output_multiplier": 0.19611613513818404,36    "pad_token_id": null,37    "post_norm_eps": 1e-08,38    "qk_scale_factor": 3.87,39    "rms_norm_eps": 1e-05,40    "rope_parameters": {41      "rope_theta": 500000.0,42      "rope_type": "default"43    },44    "sliding_window": 2048,45    "tie_word_embeddings": false,46    "use_cache": true,47    "vocab_size": 20204848  },49  "transformers_version": "5.15.0",50  "video_token_id": 200091,51  "vision_config": {52    "hidden_act": "gelu",53    "hidden_size": 16,54    "intermediate_size": 32,55    "layer_norm_eps": 1e-05,56    "layer_types": [57      "window_attention",58      "full_attention"59    ],60    "max_position_embeddings": 1024,61    "merge_size": 2,62    "model_type": "muse_glimmer_vision",63    "num_attention_heads": 4,64    "num_hidden_layers": 2,65    "patch_size": 14,66    "patch_temporal": 2,67    "pos_emb_height": 32,68    "pos_emb_width": 32,69    "rope_parameters": {70      "rope_theta": 10000.0,71      "rope_type": "default"72    }73  }74}75