hugging-science/CodVa-2-SFT

1{2  "vocab_size": 50304,3  "d_model": 896,4  "n_layers": 18,5  "n_heads": 14,6  "n_kv_heads": 2,7  "max_len": 2048,8  "rope_theta": 500000.0,9  "window_size": 512,10  "pattern_mult": 2.5,11  "reason_mult": 0.75,12  "reason_depth": 2,13  "gate_hidden": 64,14  "gate_init": 0.0,15  "diff_lambda_init": 0.8,16  "tie_embeddings": true,17  "muon_lr": 0.002,18  "muon_min_lr": 0.0002,19  "muon_mom": 0.95,20  "adamw_lr": 5e-05,21  "adamw_min_lr": 5e-06,22  "beta1": 0.9,23  "beta2": 0.95,24  "wd": 0.05,25  "grad_clip": 1.0,26  "warmup": 100,27  "total_steps": 8000,28  "micro_batch": 8,29  "grad_accum": 8,30  "log_every": 10,31  "eval_every": 200,32  "ckpt_every": 1000,33  "step": 8000,34  "epoch": 0,35  "loss": 1.4803357672691346,36  "arch": "CodVa-2-SFT"37}