AMFORGE/cofos

1{2  "vocab_size": 16384,3  "dim": 384,4  "n_layers": 8,5  "n_heads": 8,6  "max_seq_len": 512,7  "channel_top_k": 128,8  "token_top_k": 96,9  "ffn_mult": 4,10  "mem_dim": 128,11  "mem_size": 20000,12  "mem_k": 8,13  "mem_threshold": 0.4,14  "memory_layers": [15    2,16    517  ],18  "use_memory": true,19  "n_neuron_types": 7,20  "target_stem_ratio": 0.1,21  "min_stem_ratio": 0.08,22  "stem_plasticity": 0.012,23  "reversion_rate": 0.012,24  "min_age_before_revert": 8,25  "update_interval": 10,26  "baseline_revert_ratio": 0.5,27  "inhibition_strength": 0.08,28  "modulation_strength": 0.1,29  "excitation_strength": 0.3,30  "dropout": 0.1,31  "batch_size": 24,32  "lr": 0.0003,33  "max_steps": 8000,34  "warmup": 800,35  "log_every": 100,36  "eval_every": 500,37  "patience": 8,38  "use_diversity": true,39  "pad_id": 0,40  "eos_id": 2,41  "unk_id": 3,42  "architecture": "SparseMind",43  "model_name": "Cofos Code"44}