Cactus-Compute/needle3

1{2  "model_type": "needle",3  "engine_version": "3.0.0",4  "architectures": [5    "NeedleForToolCalling"6  ],7  "name_or_path": "Cactus-Compute/needle3",8  "vocab_size": 8192,9  "output_vocab_size": 8192,10  "hidden_size": 768,11  "num_hidden_layers": 20,12  "num_attention_heads": 12,13  "num_key_value_heads": 2,14  "qk_head_dim": 48,15  "value_head_dim": 64,16  "max_position_embeddings": 8192,17  "rope_theta": 100000.0,18  "tie_word_embeddings": true,19  "total_parameters": 121021910,20  "sliding_window": 1024,21  "attention": "gqa",22  "mlp": "hadamard",23  "extras": {24    "engram": {25      "sites": [26        3,27        7,28        11,29        15,30        1931      ],32      "orders": [33        2,34        335      ],36      "slots": 18432,37      "sub_dim": 12838    },39    "global_attention_layers": [40      4,41      9,42      14,43      1944    ],45    "qkv_conv_taps": 3,46    "mhc_lanes": 4,47    "heads": [48      "confidence"49    ],50    "modalities": [51      "text"52    ]53  },54  "quantization": {55    "method": "cactus-quants",56    "scheme": "embedding=4,mhc=4,default=2",57    "group_size": 128,58    "kv_cache_bits": 8,59    "activation_bits": 860  },61  "deployment": {62    "format": "cact",63    "runtime": "self-contained C++ / WebAssembly, no dependencies"64  },65  "kv_window": 25666}67