pythonstudentiam/tinyllm

1{2  "config": {3    "project": "tinyllm",4    "hub": {5      "user": "pythonstudentiam",6      "model_repo_suffix": "tinyllm",7      "ckpt_repo_suffix": "tinyllm-checkpoints",8      "model_repo": "pythonstudentiam/tinyllm",9      "ckpt_repo": "pythonstudentiam/tinyllm-checkpoints"10    },11    "tokenizer": {12      "vocab_size": 8192,13      "model_type": "bpe",14      "character_coverage": 1.0,15      "train_sentences": 400000,16      "max_sentence_length": 8192,17      "unk_id": 0,18      "bos_id": 1,19      "eos_id": 2,20      "pad_id": 3,21      "unk_piece": "<unk>",22      "bos_piece": "<s>",23      "eos_piece": "</s>",24      "pad_piece": "<pad>",25      "im_start": "<|im_start|>",26      "im_end": "<|im_end|>",27      "user_defined_symbols": [28        "<|im_start|>",29        "<|im_end|>"30      ],31      "chat_template": "{% for message in messages %}{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}"32    },33    "model": {34      "hidden_size": 384,35      "num_hidden_layers": 8,36      "num_attention_heads": 6,37      "num_key_value_heads": 2,38      "intermediate_size": 1024,39      "vocab_size": 8192,40      "max_position_embeddings": 512,41      "rope_theta": 10000.0,42      "rms_norm_eps": 1e-05,43      "tie_word_embeddings": true,44      "attention_bias": false,45      "mlp_bias": false,46      "initializer_range": 0.02,47      "head_dim": 64,48      "kv_dim": 128,49      "n_rep": 3,50      "n_params": 1573516851    },52    "data": {53      "dataset_id": "roneneldan/TinyStories",54      "instruct_dataset_id": "roneneldan/TinyStoriesInstruct",55      "train_split": "train",56      "val_split": "validation",57      "seq_len": 512,58      "val_tokens": 1000000,59      "shard_tokens": 25000000,60      "seed": 133761    },62    "train": {63      "micro_batch_size": 32,64      "grad_accum_steps": 4,65      "max_steps": 2500,66      "learning_rate": 0.0006,67      "min_lr_ratio": 0.1,68      "warmup_steps": 200,69      "weight_decay": 0.1,70      "beta1": 0.9,71      "beta2": 0.95,72      "grad_clip": 1.0,73      "dtype": "fp16",74      "compile_model": false,75      "eval_every": 250,76      "eval_batches": 40,77      "sample_every": 500,78      "log_every": 10,79      "checkpoint_every": 500,80      "keep_last_n_checkpoints": 2,81      "seed": 1337,82      "smoke_max_steps": 50,83      "smoke_stories": 2000,84      "tokens_per_step": 65536,85      "total_tokens": 163840000,86      "min_lr": 5.9999999999999995e-0587    },88    "sft": {89      "micro_batch_size": 16,90      "grad_accum_steps": 4,91      "max_steps": 1500,92      "learning_rate": 0.0001,93      "min_lr_ratio": 0.1,94      "warmup_steps": 50,95      "weight_decay": 0.0,96      "beta1": 0.9,97      "beta2": 0.95,98      "grad_clip": 1.0,99      "seq_len": 512,100      "ignore_index": -100,101      "eval_every": 200,102      "log_every": 10,103      "checkpoint_every": 500,104      "seed": 1337105    },106    "gen": {107      "max_new_tokens": 256,108      "temperature": 0.8,109      "top_p": 0.95,110      "top_k": 40,111      "repetition_penalty": 1.1,112      "eval_prompts": [113        "Once upon a time, there was a little girl named Lily.",114        "Tom and Sara went to the park. They saw a big",115        "The cat was very hungry, so it"116      ],117      "eval_instructions": [118        "Write a story about a lost puppy who finds its way home.",119        "Write a short story using the words: ball, tree, happy.",120        "Tell me a story about a brave little boat."121      ]122    },123    "quant": {124      "levels": [125        "Q8_0",126        "Q5_K_M",127        "Q4_K_M"128      ],129      "perplexity_ctx": 512,130      "perplexity_chunks": 40131    },132    "serve": {133      "host": "127.0.0.1",134      "port": 8080,135      "threads": 4,136      "ctx_size": 512,137      "served_model_name": "tinyllm",138      "default_quant": "Q8_0",139      "llamacpp_build": "b10107",140      "llamacpp_asset": "llama-b10107-bin-win-cpu-x64.zip",141      "base_url": "http://127.0.0.1:8080/v1",142      "llamacpp_url": "https://github.com/ggml-org/llama.cpp/releases/download/b10107/llama-b10107-bin-win-cpu-x64.zip"143    },144    "derived": {145      "head_dim": 64,146      "kv_dim": 128,147      "n_params": 15735168,148      "param_breakdown": {149        "embedding": 3145728,150        "attention": 3145728,151        "mlp": 9437184,152        "layernorms": 6528,153        "lm_head": 0,154        "per_layer": 1573632,155        "blocks_total": 12589056,156        "non_embedding": 12589440,157        "total": 15735168158      },159      "tokens_per_step": 65536,160      "total_tokens": 163840000,161      "flops_per_token": 94411008162    }163  }164}