NextTokenAI/NextSearch-1-M

1{2  "architectures": [3    "InklingForConditionalGeneration"4  ],5  "model_type": "inkling_mm_model",6  "eos_token_id": 200006,7  "text_config": {8    "model_max_length": 1048576,9    "torch_dtype": "bfloat16",10    "hidden_size": 4096,11    "num_hidden_layers": 42,12    "vocab_size": 201024,13    "num_attention_heads": 32,14    "num_key_value_heads": 8,15    "head_dim": 128,16    "d_rel": 16,17    "rel_extent": 1024,18    "q_bias": false,19    "o_bias": false,20    "log_scaling_n_floor": 128000,21    "log_scaling_alpha": 0.1,22    "rms_norm_eps": 1e-06,23    "use_embed_norm": true,24    "local_layer_ids": [25      0,26      1,27      2,28      3,29      4,30      6,31      7,32      8,33      9,34      10,35      12,36      13,37      14,38      15,39      16,40      18,41      19,42      20,43      21,44      22,45      24,46      25,47      26,48      27,49      28,50      30,51      31,52      32,53      33,54      34,55      36,56      37,57      38,58      39,59      4060    ],61    "dense_mlp_idx": 2,62    "use_sconv": true,63    "sconv_kernel_size": 4,64    "unpadded_vocab_size": 200058,65    "logits_mup_width_multiplier": 16.0,66    "final_logit_softcapping": null,67    "swa_head_dim": 128,68    "swa_num_attention_heads": 32,69    "swa_num_key_value_heads": 8,70    "sliding_window_size": 512,71    "n_routed_experts": 256,72    "num_experts_per_tok": 6,73    "n_shared_experts": 2,74    "shared_expert_sink": true,75    "dense_intermediate_size": 16384,76    "intermediate_size": 2048,77    "route_scale": 8.0,78    "use_gate_bias": true,79    "gate_activation": "sigmoid",80    "norm_after_topk": true,81    "use_global_scale": true82  },83  "audio_config": {84    "decoder_dmodel": 4096,85    "n_mel_bins": 80,86    "mel_vocab_size": 16,87    "bias": false,88    "dmel_min_value": -7.0,89    "dmel_max_value": 2.0,90    "use_audio_norm": true,91    "audio_mode": "dmel"92  },93  "vision_config": {94    "vision_encoder_type": "hmlp",95    "decoder_dmodel": 4096,96    "patch_size": 40,97    "temporal_patch_size": 2,98    "n_channels": 3,99    "n_layers": 4,100    "use_vision_norm": true101  },102  "mtp_config": {103    "num_nextn_predict_layers": 8,104    "chain_hidden_post_norm": false,105    "local_layer_ids": [106      0,107      2,108      4,109      5,110      6,111      7112    ]113  }114}115