nkkbr/Mini-K3-1H-kda-kernel-64-v2

1{2  "world_size": 1,3  "tensor_parallel_size": 1,4  "expert_parallel_size": 1,5  "expert_tensor_parallel_size": 1,6  "pipeline_parallel_size": 1,7  "experts_per_ep_rank": 64,8  "sequence_length_first_stage": 4096,9  "micro_batch_sequences_per_ep_data_row": 4,10  "gradient_accumulation_steps": 8,11  "global_tokens_per_optimizer_step": 131072,12  "activation_checkpointing": "every_decoder_layer_and_attnres",13  "parameter_dtype": "bfloat16",14  "gradient_reduction_dtype": "bfloat16",15  "optimizer_state_dtype": "float32",16  "kda_sensitive_state_dtype": "float32",17  "notes": [18    "The complete model, all 64 routed experts, optimizer state, gradients, and activations fit on one H200.",19    "No tensor, expert, pipeline, or data-parallel communication is required for one experiment.",20    "Benchmark micro-batch size before formal runs; the value here is a conservative starting point, not a memory guarantee.",21    "QB statistics are local to the optimizer batch and correction biases are updated after each optimizer step."22  ]23}24