chris0809/tinyLLM-0.51B-VLM

1{
2  "return_dict": true,
3  "output_hidden_states": false,
4  "output_attentions": false,
5  "torchscript": false,
6  "torch_dtype": null,
7  "use_bfloat16": false,
8  "tf_legacy_loss": false,
9  "pruned_heads": {},
10  "tie_word_embeddings": true,
11  "chunk_size_feed_forward": 0,
12  "is_encoder_decoder": false,
13  "is_decoder": true,
14  "cross_attention_hidden_size": null,
15  "add_cross_attention": false,
16  "tie_encoder_decoder": false,
17  "max_length": 20,
18  "min_length": 0,
19  "do_sample": false,
20  "early_stopping": false,
21  "num_beams": 1,
22  "num_beam_groups": 1,
23  "diversity_penalty": 0.0,
24  "temperature": 1.0,
25  "top_k": 50,
26  "top_p": 1.0,
27  "typical_p": 1.0,
28  "repetition_penalty": 1.0,
29  "length_penalty": 1.0,
30  "no_repeat_ngram_size": 0,
31  "encoder_no_repeat_ngram_size": 0,
32  "bad_words_ids": null,
33  "num_return_sequences": 1,
34  "output_scores": false,
35  "return_dict_in_generate": false,
36  "forced_bos_token_id": null,
37  "forced_eos_token_id": null,
38  "remove_invalid_values": false,
39  "exponential_decay_length_penalty": null,
40  "suppress_tokens": null,
41  "begin_suppress_tokens": null,
42  "architectures": [
43    "TinyLLM"
44  ],
45  "finetuning_task": null,
46  "id2label": {
47    "0": "LABEL_0",
48    "1": "LABEL_1"
49  },
50  "label2id": {
51    "LABEL_0": 0,
52    "LABEL_1": 1
53  },
54  "tokenizer_class": null,
55  "prefix": null,
56  "bos_token_id": 1,
57  "pad_token_id": 73440,
58  "eos_token_id": 73440,
59  "sep_token_id": null,
60  "decoder_start_token_id": null,
61  "task_specific_params": null,
62  "problem_type": null,
63  "_name_or_path": "",
64  "_commit_hash": null,
65  "_attn_implementation_internal": null,
66  "_attn_implementation_autoset": false,
67  "transformers_version": null,
68  "vision_use_rmsnorm": true,
69  "qformer_num_layers": 4,
70  "qformer_mlp_ratio": 2.0,
71  "vocab_size": 73448,
72  "hidden_size": 1280,
73  "num_hidden_layers": 24,
74  "num_attention_heads": 20,
75  "num_key_value_heads": 4,
76  "max_position_embeddings": 8192,
77  "RoPE_base": 10000.0,
78  "rope_type": "yarn",
79  "dropout": 0.0,
80  "train_maxlength": 8192,
81  "rms_norm_eps": 1e-05,
82  "qkrms_norm_eps": 1e-05,
83  "use_qk_norm": true,
84  "kv_cache_dtype": "auto",
85  "use_moe": false,
86  "adapter_type": "none",
87  "use_ssm": false,
88  "ssm_layers": [],
89  "embeddingdropout": 0.0,
90  "learnable_temp": true,
91  "use_affine": true,
92  "use_sampleatt": false,
93  "use_swiGLU": true,
94  "mlp_ratio": 4.5,
95  "ignore_index": -100,
96  "loss_reduction": "hybrid",
97  "sample_mean_alpha": 0.75,
98  "drop_path": 0.0,
99  "residual_dropout": 0.0,
100  "moe_layers": [],
101  "num_expert": 4,
102  "moe_use_detach": false,
103  "moe_cap_factor": 1.5,
104  "moe_aux_weight": 5.0,
105  "use_checkpoint": false,
106  "checkpoint_use_reentrant": false,
107  "use_adaptive_softmax": true,
108  "adaptive_cutoffs": [
109    20000,
110    60000
111  ],
112  "adaptive_div": 4.0,
113  "adaptive_calibrate_every": 200,
114  "use_tcn": false,
115  "tcn_layers": [],
116  "drop_high_loss": 0.0,
117  "mlp_ratio_front": 3.5,
118  "mlp_ratio_mid": 4.0,
119  "mlp_ratio_back": 4.5,
120  "mlp_mid_start": 8,
121  "mlp_back_start": 16,
122  "mlp_ratio_overrides": {
123    "21": 5.0,
124    "22": 5.0,
125    "23": 5.0
126  },
127  "eso_loss_radio": 1.0,
128  "use_vision": true,
129  "vision_feature_dim": 1024,
130  "max_vision_tokens": 0,
131  "vision_mlp_ratio": 4.0,
132  "vision_use_swiglu": true,
133  "vision_dropout": 0.0,
134  "num_query_tokens": 128,
135  "auto_map": {
136    "AutoConfig": "configuration_tinyllm.Config",
137    "AutoModelForCausalLM": "modeling_tinyllm.TinyLLM"
138  },
139  "vision_tower_name_or_path": "OpenGVLab/InternViT-300M-448px-V2_5"
140}
141