cwenzi/neuroflow-cpp
1
1/**2 * NeuroFlow MultiModal Tests3 */4 5#include <iostream>6#include <cassert>7#include <cmath>8#include <chrono>9#include "../include/neuroflow/multimodal.hpp"10#include "../include/neuroflow/multimodal_model.hpp"11 12using namespace neuroflow;13 14void test_patch_embedding() {15 std::cout << "Testing PatchEmbedding..." << std::endl;16 17 PatchEmbedding patch_emb(64, 8, 3, 64); // 64x64 image, 8x8 patch18 19 std::cout << " num_patches: " << patch_emb.num_patches << std::endl;20 assert(patch_emb.num_patches == (64/8) * (64/8)); // 64 patches21 22 // 创建模拟图像数据23 Tensor image({2, 3, 64, 64});24 float* img = image.as_fp32();25 for (size_t i = 0; i < image.numel(); ++i) img[i] = 0.1f * i;26 27 Tensor embedded = patch_emb.forward(image);28 std::cout << " embedded shape: [" << embedded.shape_[0] << ", " 29 << embedded.shape_[1] << ", " << embedded.shape_[2] << "]" << std::endl;30 31 assert(embedded.shape_[0] == 2); // batch32 assert(embedded.shape_[1] == 64); // num_patches33 assert(embedded.shape_[2] == 64); // embed_dim34 35 std::cout << " PASSED: PatchEmbedding" << std::endl;36}37 38void test_vision_encoder() {39 std::cout << "Testing VisionEncoder..." << std::endl;40 41 VisionEncoder encoder(64, 8, 64, 4, 2); // 小尺寸测试42 43 std::cout << " embed_dim: " << encoder.embed_dim << std::endl;44 std::cout << " num_layers: " << encoder.num_layers << std::endl;45 46 // 创建模拟图像47 Tensor image({2, 3, 64, 64});48 float* img = image.as_fp32();49 for (size_t i = 0; i < image.numel(); ++i) img[i] = 0.1f * i;50 51 Tensor vision_feat = encoder.forward(image);52 53 std::cout << " vision_feat shape: [" << vision_feat.shape_[0] 54 << ", " << vision_feat.shape_[1] << "]" << std::endl;55 56 assert(vision_feat.shape_[0] == 2); // batch57 assert(vision_feat.shape_[1] == 64); // embed_dim58 59 std::cout << " PASSED: VisionEncoder" << std::endl;60}61 62void test_cross_modal_fusion() {63 std::cout << "Testing CrossModalFusion..." << std::endl;64 65 CrossModalFusion fusion(64, 64, 64);66 67 Tensor text_feat({2, 64});68 Tensor image_feat({2, 64});69 70 float* t = text_feat.as_fp32();71 float* i = image_feat.as_fp32();72 for (size_t j = 0; j < 64; ++j) {73 t[j] = 0.5f;74 t[64 + j] = 0.3f;75 i[j] = 0.5f; // 相似76 i[64 + j] = 0.1f; // 不同77 }78 79 auto output = fusion.forward(text_feat, image_feat);80 81 std::cout << " fused shape: [" << output.fused.shape_[0] 82 << ", " << output.fused.shape_[1] << "]" << std::endl;83 std::cout << " text_feat shape: [" << output.text_feat.shape_[0] 84 << ", " << output.text_feat.shape_[1] << "]" << std::endl;85 std::cout << " similarity scores: ";86 float* sim = output.similarity.as_fp32();87 std::cout << sim[0] << ", " << sim[1] << std::endl;88 89 assert(output.fused.shape_[0] == 2);90 assert(output.fused.shape_[1] == 64);91 92 std::cout << " PASSED: CrossModalFusion" << std::endl;93}94 95void test_multimodal_attention() {96 std::cout << "Testing MultiModalAttention..." << std::endl;97 98 MultiModalAttention attn(64, 64, 4);99 100 Tensor text({2, 64});101 Tensor image({2, 64});102 103 for (size_t j = 0; j < 64; ++j) {104 text.as_fp32()[j] = 0.1f * j;105 text.as_fp32()[64 + j] = 0.2f * j;106 image.as_fp32()[j] = 0.3f * j;107 image.as_fp32()[64 + j] = 0.4f * j;108 }109 110 Tensor text_enhanced = attn.text_attend_image(text, image);111 112 std::cout << " text_enhanced shape: [" << text_enhanced.shape_[0] 113 << ", " << text_enhanced.shape_[1] << "]" << std::endl;114 115 assert(text_enhanced.shape_[0] == 2);116 assert(text_enhanced.shape_[1] == 64);117 118 std::cout << " PASSED: MultiModalAttention" << std::endl;119}120 121void test_multimodal_model_creation() {122 std::cout << "Testing NeuroFlowMultiModal creation..." << std::endl;123 124 NeuroFlowMultiModal::Config cfg;125 cfg.text_dim = 64;126 cfg.image_size = 64;127 cfg.patch_size = 8;128 cfg.vision_dim = 32;129 cfg.fusion_dim = 32;130 cfg.hidden_dim = 32;131 cfg.output_dim = 5;132 cfg.memory_dim = 16;133 cfg.memory_slots = 8;134 cfg.num_layers = 1;135 cfg.num_associations = 2;136 cfg.vision_layers = 2;137 cfg.vision_heads = 2;138 cfg.use_mla = false;139 140 NeuroFlowMultiModal model(cfg);141 142 auto stats = model.get_stats();143 std::cout << " Total params: " << stats.total_params << std::endl;144 std::cout << " Vision params: " << stats.vision_params << std::endl;145 std::cout << " Fusion params: " << stats.fusion_params << std::endl;146 std::cout << " Brain params: " << stats.brain_params << std::endl;147 std::cout << " Memory (KB): " << stats.memory_bytes / 1024.0 << std::endl;148 149 assert(stats.total_params > 0);150 151 std::cout << " PASSED: NeuroFlowMultiModal creation" << std::endl;152}153 154void test_multimodal_forward_text() {155 std::cout << "Testing multimodal forward (text only)..." << std::endl;156 157 NeuroFlowMultiModal::Config cfg;158 cfg.text_dim = 64;159 cfg.image_size = 64;160 cfg.patch_size = 8;161 cfg.vision_dim = 32;162 cfg.fusion_dim = 32;163 cfg.hidden_dim = 32;164 cfg.output_dim = 5;165 cfg.memory_dim = 16;166 cfg.memory_slots = 8;167 cfg.num_layers = 1;168 cfg.num_associations = 2;169 cfg.vision_layers = 2;170 cfg.vision_heads = 2;171 172 NeuroFlowMultiModal model(cfg);173 174 Tensor text_input({2, cfg.text_dim});175 for (size_t i = 0; i < text_input.numel(); ++i) 176 text_input.as_fp32()[i] = 0.1f * i;177 178 auto output = model.forward_text(text_input);179 180 std::cout << " output shape: [" << output.output.shape_[0] 181 << ", " << output.output.shape_[1] << "]" << std::endl;182 std::cout << " decision shape: [" << output.decision.shape_[0] 183 << ", " << output.decision.shape_[1] << "]" << std::endl;184 185 assert(output.output.shape_[0] == 2);186 assert(output.output.shape_[1] == cfg.output_dim);187 188 std::cout << " PASSED: multimodal forward (text)" << std::endl;189}190 191void test_multimodal_forward_with_image() {192 std::cout << "Testing multimodal forward (text + image)..." << std::endl;193 194 NeuroFlowMultiModal::Config cfg;195 cfg.text_dim = 32;196 cfg.image_size = 32;197 cfg.patch_size = 4;198 cfg.vision_dim = 16;199 cfg.fusion_dim = 16;200 cfg.hidden_dim = 16;201 cfg.output_dim = 5;202 cfg.memory_dim = 8;203 cfg.memory_slots = 4;204 cfg.num_layers = 1;205 cfg.num_associations = 2;206 cfg.vision_layers = 1;207 cfg.vision_heads = 2;208 209 NeuroFlowMultiModal model(cfg);210 211 // 文本输入212 Tensor text_input({2, cfg.text_dim});213 for (size_t i = 0; i < text_input.numel(); ++i) 214 text_input.as_fp32()[i] = 0.1f * i;215 216 // 图像输入217 Tensor image_input({2, 3, cfg.image_size, cfg.image_size});218 for (size_t i = 0; i < image_input.numel(); ++i) 219 image_input.as_fp32()[i] = 0.05f * i;220 221 auto output = model.forward_multimodal(text_input, image_input);222 223 std::cout << " output shape: [" << output.output.shape_[0] 224 << ", " << output.output.shape_[1] << "]" << std::endl;225 std::cout << " vision_feat shape: [" << output.vision_feat.shape_[0] 226 << ", " << output.vision_feat.shape_[1] << "]" << std::endl;227 std::cout << " fused_feat shape: [" << output.fused_feat.shape_[0] 228 << ", " << output.fused_feat.shape_[1] << "]" << std::endl;229 std::cout << " text-image similarity: " << output.text_image_sim.as_fp32()[0] 230 << ", " << output.text_image_sim.as_fp32()[1] << std::endl;231 232 assert(output.output.shape_[0] == 2);233 assert(output.output.shape_[1] == cfg.output_dim);234 assert(output.vision_feat.shape_[1] == cfg.vision_dim);235 236 std::cout << " PASSED: multimodal forward (text+image)" << std::endl;237}238 239void test_multimodal_image_only() {240 std::cout << "Testing multimodal forward (image only)..." << std::endl;241 242 NeuroFlowMultiModal::Config cfg;243 cfg.text_dim = 32;244 cfg.image_size = 32;245 cfg.patch_size = 4;246 cfg.vision_dim = 16;247 cfg.fusion_dim = 16;248 cfg.hidden_dim = 16;249 cfg.output_dim = 5;250 cfg.memory_dim = 8;251 cfg.memory_slots = 4;252 cfg.num_layers = 1;253 cfg.num_associations = 2;254 cfg.vision_layers = 1;255 cfg.vision_heads = 2;256 257 NeuroFlowMultiModal model(cfg);258 259 // 图像输入260 Tensor image_input({1, 3, cfg.image_size, cfg.image_size});261 for (size_t i = 0; i < image_input.numel(); ++i) 262 image_input.as_fp32()[i] = 0.05f * i;263 264 auto output = model.forward_image_only(image_input);265 266 std::cout << " output shape: [" << output.output.shape_[0] 267 << ", " << output.output.shape_[1] << "]" << std::endl;268 std::cout << " vision_feat shape: [" << output.vision_feat.shape_[0] 269 << ", " << output.vision_feat.shape_[1] << "]" << std::endl;270 271 assert(output.output.shape_[0] == 1);272 assert(output.output.shape_[1] == cfg.output_dim);273 274 std::cout << " PASSED: multimodal forward (image only)" << std::endl;275}276 277void test_multimodal_quantization() {278 std::cout << "Testing multimodal quantization..." << std::endl;279 280 NeuroFlowMultiModal::Config cfg;281 cfg.text_dim = 32;282 cfg.image_size = 32;283 cfg.patch_size = 4;284 cfg.vision_dim = 16;285 cfg.fusion_dim = 16;286 cfg.hidden_dim = 16;287 cfg.output_dim = 5;288 cfg.use_quantization = true;289 290 NeuroFlowMultiModal model(cfg);291 model.quantize();292 293 auto stats = model.get_stats();294 std::cout << " Total params after quant: " << stats.total_params << std::endl;295 296 // 测试量化后仍能运行297 Tensor text_input({1, cfg.text_dim});298 Tensor image_input({1, 3, cfg.image_size, cfg.image_size});299 300 auto output = model.forward_multimodal(text_input, image_input);301 302 std::cout << " Output after quant: [" << output.output.shape_[0] 303 << ", " << output.output.shape_[1] << "]" << std::endl;304 305 assert(output.output.shape_[1] == cfg.output_dim);306 307 std::cout << " PASSED: multimodal quantization" << std::endl;308}309 310void test_multimodal_performance() {311 std::cout << "Testing multimodal performance..." << std::endl;312 313 NeuroFlowMultiModal::Config cfg;314 cfg.text_dim = 128;315 cfg.image_size = 64;316 cfg.patch_size = 8;317 cfg.vision_dim = 64;318 cfg.fusion_dim = 64;319 cfg.hidden_dim = 64;320 cfg.output_dim = 10;321 cfg.memory_dim = 32;322 cfg.memory_slots = 16;323 cfg.num_layers = 1;324 cfg.num_associations = 4;325 cfg.vision_layers = 2;326 cfg.vision_heads = 4;327 328 NeuroFlowMultiModal model(cfg);329 330 auto stats = model.get_stats();331 std::cout << " Full model params: " << stats.total_params << std::endl;332 333 // Lite版本334 NeuroFlowMultiModal::Config lite_cfg;335 lite_cfg.text_dim = 64;336 lite_cfg.image_size = 32;337 lite_cfg.patch_size = 4;338 lite_cfg.vision_dim = 32;339 lite_cfg.fusion_dim = 32;340 lite_cfg.hidden_dim = 32;341 lite_cfg.output_dim = 10;342 lite_cfg.memory_dim = 16;343 lite_cfg.memory_slots = 8;344 lite_cfg.num_layers = 1;345 lite_cfg.num_associations = 2;346 lite_cfg.vision_layers = 1;347 lite_cfg.vision_heads = 2;348 lite_cfg.use_quantization = true;349 350 NeuroFlowMultiModal lite(lite_cfg);351 352 auto lite_stats = lite.get_stats();353 std::cout << " Lite model params: " << lite_stats.total_params << std::endl;354 std::cout << " Size reduction: " << (1.0 - (double)lite_stats.total_params / stats.total_params) * 100 << "%" << std::endl;355 356 // 性能测试357 Tensor text({4, cfg.text_dim});358 Tensor image({4, 3, cfg.image_size, cfg.image_size});359 360 for (size_t i = 0; i < text.numel(); ++i) text.as_fp32()[i] = 0.1f * i;361 for (size_t i = 0; i < image.numel(); ++i) image.as_fp32()[i] = 0.05f * i;362 363 // 预热364 model.forward_multimodal(text, image);365 366 // Full模型367 auto start = std::chrono::high_resolution_clock::now();368 for (int i = 0; i < 10; ++i) {369 model.forward_multimodal(text, image);370 }371 auto end = std::chrono::high_resolution_clock::now();372 auto full_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count() / 1000.0 / 10;373 374 // Lite模型375 Tensor lite_text({4, lite_cfg.text_dim});376 Tensor lite_image({4, 3, lite_cfg.image_size, lite_cfg.image_size});377 378 start = std::chrono::high_resolution_clock::now();379 for (int i = 0; i < 10; ++i) {380 lite.forward_multimodal(lite_text, lite_image);381 }382 end = std::chrono::high_resolution_clock::now();383 auto lite_time = std::chrono::duration_cast<std::chrono::microseconds>(end - start).count() / 1000.0 / 10;384 385 std::cout << " Full model time: " << full_time << " ms" << std::endl;386 std::cout << " Lite model time: " << lite_time << " ms" << std::endl;387 std::cout << " Speedup: " << (full_time / lite_time) << "x" << std::endl;388 389 std::cout << " PASSED: multimodal performance" << std::endl;390}391 392int main(int argc, char** argv) {393 std::cout << "========================================" << std::endl;394 std::cout << "NeuroFlow MultiModal Tests" << std::endl;395 std::cout << "========================================" << std::endl;396 397 test_patch_embedding();398 test_vision_encoder();399 test_cross_modal_fusion();400 test_multimodal_attention();401 test_multimodal_model_creation();402 test_multimodal_forward_text();403 test_multimodal_forward_with_image();404 test_multimodal_image_only();405 test_multimodal_quantization();406 test_multimodal_performance();407 408 std::cout << "========================================" << std::endl;409 std::cout << "All MultiModal tests PASSED!" << std::endl;410 std::cout << "========================================" << std::endl;411 412 return 0;413}