tyfeld/MMaDA-Parallel-M

1{2  "_name_or_path": "/mnt/bn/strategy-mllm-train/user/tianye/pretrained_models/MMaDA0000",3  "activation_type": "silu",4  "alibi": false,5  "alibi_bias_max": 8.0,6  "architectures": [7    "MMadaModelLM"8  ],9  "attention_dropout": 0.0,10  "attention_layer_norm": false,11  "attention_layer_norm_with_affine": true,12  "auto_map": {13    "AutoConfig": "configuration_llada.LLaDAConfig",14    "AutoModel": "modeling_llada.LLaDAModelLM",15    "AutoModelForCausalLM": "modeling_llada.LLaDAModelLM"16  },17  "bias_for_layer_norm": false,18  "block_group_size": 1,19  "block_type": "llama",20  "codebook_size": 8192,21  "d_model": 4096,22  "embedding_dropout": 0.0,23  "embedding_size": 134656,24  "eos_token_id": 126081,25  "flash_attention": false,26  "include_bias": false,27  "include_qkv_bias": false,28  "init_cutoff_factor": null,29  "init_device": "meta",30  "init_fn": "mitchell",31  "init_std": 0.02,32  "input_emb_norm": false,33  "layer_norm_type": "rms",34  "layer_norm_with_affine": true,35  "llm_vocab_size": 126464,36  "mask_token_id": 126336,37  "max_sequence_length": 4096,38  "mlp_hidden_size": 12288,39  "mlp_ratio": 4,40  "model_type": "mmada",41  "multi_query_attention": null,42  "n_heads": 32,43  "n_kv_heads": 32,44  "n_layers": 32,45  "new_vocab_size": 134656,46  "num_new_special_tokens": 0,47  "num_vq_tokens": 256,48  "pad_token_id": 126081,49  "precision": "amp_bf16",50  "pretrained_model_path": "/data_storage/shared/pretrained_models/LLaDA-8B-Instruct",51  "residual_dropout": 0.0,52  "rms_norm_eps": 1e-05,53  "rope": true,54  "rope_full_precision": true,55  "rope_theta": 500000.0,56  "scale_logits": false,57  "tie_word_embeddings": false,58  "torch_dtype": "bfloat16",59  "transformers_version": "4.46.0",60  "use_cache": false,61  "vocab_size": 134656,62  "w_clip_vit": false,63  "weight_tying": false64}65